* Move (de)compression helpers to akka-stream #21395 * Move compression and decompression -related classes from akka-http-experimental to akka-stream * Add Compression helper object with functions to create decompressing Flows * Add a short cookbook entry * =str move compression impl classes into their own directory (and change visibility) * =str also expose gzip/deflate compression flows * Fix formatting of plural ByteStrings in cookbook * =str #21395 make compressor call Deflater.end in postStop to release resources Also simplified the creation of the flow given a compressor. * =str #21395 decompressors call Inflater.end in postStop to release resources * =str #21395 smallish Scaladoc fixes
This commit is contained in:
parent
d7af58bafa
commit
658b46e1cc
14 changed files with 618 additions and 0 deletions
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Copyright (C) 2016 Lightbend Inc. <http://www.lightbend.com/>
|
||||
*/
|
||||
package docs.stream.javadsl.cookbook;
|
||||
|
||||
import akka.NotUsed;
|
||||
import akka.actor.ActorSystem;
|
||||
import akka.stream.ActorMaterializer;
|
||||
import akka.stream.Materializer;
|
||||
import akka.stream.javadsl.Compression;
|
||||
import akka.stream.javadsl.Sink;
|
||||
import akka.stream.javadsl.Source;
|
||||
import akka.testkit.JavaTestKit;
|
||||
import akka.util.ByteString;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
public class RecipeDecompress extends RecipeTest {
|
||||
|
||||
static ActorSystem system;
|
||||
static Materializer mat;
|
||||
|
||||
@BeforeClass
|
||||
public static void setup() {
|
||||
system = ActorSystem.create("RecipeDecompress");
|
||||
mat = ActorMaterializer.create(system);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
JavaTestKit.shutdownActorSystem(system);
|
||||
system = null;
|
||||
mat = null;
|
||||
}
|
||||
|
||||
private ByteString gzip(final String s) throws IOException {
|
||||
final ByteArrayOutputStream buf = new ByteArrayOutputStream();
|
||||
final GZIPOutputStream out = new GZIPOutputStream(buf);
|
||||
try {
|
||||
out.write(s.getBytes(StandardCharsets.UTF_8));
|
||||
} finally {
|
||||
out.close();
|
||||
}
|
||||
return ByteString.fromArray(buf.toByteArray());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseLines() throws Exception {
|
||||
final Source<ByteString, NotUsed> compressed = Source.single(gzip("Hello World"));
|
||||
|
||||
//#decompress-gzip
|
||||
final Source<String, NotUsed> uncompressed = compressed
|
||||
.via(Compression.gunzip(100))
|
||||
.map(b -> b.utf8String());
|
||||
//#decompress-gzip
|
||||
|
||||
uncompressed.runWith(Sink.head(), mat).toCompletableFuture().get(1, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -102,6 +102,17 @@ The :class:`Framing` helper class contains a convenience method to parse message
|
|||
|
||||
.. includecode:: ../code/docs/stream/javadsl/cookbook/RecipeParseLines.java#parse-lines
|
||||
|
||||
Dealing with compressed data streams
|
||||
------------------------------------
|
||||
|
||||
**Situation:** A gzipped stream of bytes is given as a stream of ``ByteString`` s, for example from a ``FileIO`` source.
|
||||
|
||||
The :class:`Compression` helper class contains convenience methods for decompressing data streams compressed with
|
||||
Gzip or Deflate.
|
||||
|
||||
.. includecode:: ../code/docs/stream/javadsl/cookbook/RecipeDecompress.java#decompress-gzip
|
||||
|
||||
|
||||
Implementing reduce-by-key
|
||||
--------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Copyright (C) 2016 Lightbend Inc. <http://www.lightbend.com/>
|
||||
*/
|
||||
package docs.stream.cookbook
|
||||
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.zip.GZIPOutputStream
|
||||
|
||||
import akka.stream.impl.io.compression.GzipCompressor
|
||||
import akka.stream.scaladsl.Sink
|
||||
import akka.stream.scaladsl.Source
|
||||
import akka.util.ByteString
|
||||
|
||||
import scala.annotation.tailrec
|
||||
import scala.concurrent.Await
|
||||
import scala.concurrent.duration._
|
||||
|
||||
class RecipeDecompress extends RecipeSpec {
|
||||
def gzip(s: String): ByteString = {
|
||||
val buf = new ByteArrayOutputStream()
|
||||
val out = new GZIPOutputStream(buf)
|
||||
try out.write(s.getBytes(StandardCharsets.UTF_8)) finally out.close()
|
||||
ByteString(buf.toByteArray)
|
||||
}
|
||||
|
||||
"Recipe for decompressing a Gzip stream" must {
|
||||
"work" in {
|
||||
val compressed = Source.single(gzip("Hello World"))
|
||||
|
||||
//#decompress-gzip
|
||||
import akka.stream.scaladsl.Compression
|
||||
val uncompressed = compressed.via(Compression.gunzip())
|
||||
.map(_.utf8String)
|
||||
//#decompress-gzip
|
||||
|
||||
Await.result(uncompressed.runWith(Sink.head), 3.seconds) should be("Hello World")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -100,6 +100,16 @@ The :class:`Framing` helper object contains a convenience method to parse messag
|
|||
|
||||
.. includecode:: ../code/docs/stream/cookbook/RecipeParseLines.scala#parse-lines
|
||||
|
||||
Dealing with compressed data streams
|
||||
------------------------------------
|
||||
|
||||
**Situation:** A gzipped stream of bytes is given as a stream of ``ByteString`` s, for example from a ``FileIO`` source.
|
||||
|
||||
The :class:`Compression` helper object contains convenience methods for decompressing data streams compressed with
|
||||
Gzip or Deflate.
|
||||
|
||||
.. includecode:: ../code/docs/stream/cookbook/RecipeDecompress.scala#decompress-gzip
|
||||
|
||||
Implementing reduce-by-key
|
||||
--------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
/**
|
||||
* Copyright (C) 2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.scaladsl
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
import akka.stream.impl.io.compression.{ DeflateCompressor, GzipCompressor }
|
||||
import akka.stream.testkit.StreamSpec
|
||||
import akka.stream.testkit.scaladsl.TestSink
|
||||
import akka.stream.{ ActorMaterializer, ActorMaterializerSettings }
|
||||
import akka.util.ByteString
|
||||
|
||||
class CompressionSpec extends StreamSpec {
|
||||
val settings = ActorMaterializerSettings(system)
|
||||
implicit val materializer = ActorMaterializer(settings)
|
||||
|
||||
def gzip(s: String): ByteString = new GzipCompressor().compressAndFinish(ByteString(s))
|
||||
|
||||
def deflate(s: String): ByteString = new DeflateCompressor().compressAndFinish(ByteString(s))
|
||||
|
||||
val data = "hello world"
|
||||
|
||||
"Gzip decompression" must {
|
||||
"be able to decompress a gzipped stream" in {
|
||||
Source.single(gzip(data))
|
||||
.via(Compression.gunzip())
|
||||
.map(_.decodeString(StandardCharsets.UTF_8))
|
||||
.runWith(TestSink.probe)
|
||||
.requestNext(data)
|
||||
.expectComplete()
|
||||
}
|
||||
}
|
||||
|
||||
"Deflate decompression" must {
|
||||
"be able to decompress a deflated stream" in {
|
||||
Source.single(deflate(data))
|
||||
.via(Compression.inflate())
|
||||
.map(_.decodeString(StandardCharsets.UTF_8))
|
||||
.runWith(TestSink.probe)
|
||||
.requestNext(data)
|
||||
.expectComplete()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import akka.NotUsed
|
||||
import akka.stream.{ Attributes, FlowShape }
|
||||
import akka.stream.impl.fusing.GraphStages.SimpleLinearGraphStage
|
||||
import akka.stream.scaladsl.Flow
|
||||
import akka.stream.stage.{ GraphStage, GraphStageLogic, InHandler, OutHandler }
|
||||
import akka.util.ByteString
|
||||
|
||||
/** INTERNAL API */
|
||||
private[stream] object CompressionUtils {
|
||||
/**
|
||||
* Creates a flow from a compressor constructor.
|
||||
*/
|
||||
def compressorFlow(newCompressor: () ⇒ Compressor): Flow[ByteString, ByteString, NotUsed] =
|
||||
Flow.fromGraph {
|
||||
new SimpleLinearGraphStage[ByteString] {
|
||||
override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) with InHandler with OutHandler {
|
||||
val compressor = newCompressor()
|
||||
|
||||
override def onPush(): Unit = {
|
||||
val data = compressor.compress(grab(in))
|
||||
if (data.nonEmpty) push(out, data)
|
||||
else pull(in)
|
||||
}
|
||||
|
||||
override def onPull(): Unit = pull(in)
|
||||
|
||||
override def onUpstreamFinish(): Unit = {
|
||||
val data = compressor.finish()
|
||||
if (data.nonEmpty) emit(out, data)
|
||||
completeStage()
|
||||
}
|
||||
|
||||
override def postStop(): Unit = compressor.close()
|
||||
|
||||
setHandlers(in, out, this)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import akka.util.ByteString
|
||||
|
||||
/**
|
||||
* INTERNAL API
|
||||
*
|
||||
* A stateful object representing ongoing compression.
|
||||
*/
|
||||
private[akka] abstract class Compressor {
|
||||
/**
|
||||
* Compresses the given input and returns compressed data. The implementation
|
||||
* can and will choose to buffer output data to improve compression. Use
|
||||
* `flush` or `compressAndFlush` to make sure that all input data has been
|
||||
* compressed and pending output data has been returned.
|
||||
*/
|
||||
def compress(input: ByteString): ByteString
|
||||
|
||||
/**
|
||||
* Flushes any output data and returns the currently remaining compressed data.
|
||||
*/
|
||||
def flush(): ByteString
|
||||
|
||||
/**
|
||||
* Closes this compressed stream and return the remaining compressed data. After
|
||||
* calling this method, this Compressor cannot be used any further.
|
||||
*/
|
||||
def finish(): ByteString
|
||||
|
||||
/** Combines `compress` + `flush` */
|
||||
def compressAndFlush(input: ByteString): ByteString
|
||||
/** Combines `compress` + `finish` */
|
||||
def compressAndFinish(input: ByteString): ByteString
|
||||
|
||||
/** Make sure any resources have been released */
|
||||
def close(): Unit
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import java.util.zip.Deflater
|
||||
|
||||
import akka.util.{ ByteString, ByteStringBuilder }
|
||||
|
||||
import scala.annotation.tailrec
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] class DeflateCompressor extends Compressor {
|
||||
import DeflateCompressor._
|
||||
|
||||
protected lazy val deflater = new Deflater(Deflater.BEST_COMPRESSION, false)
|
||||
|
||||
override final def compressAndFlush(input: ByteString): ByteString = {
|
||||
val buffer = newTempBuffer(input.size)
|
||||
|
||||
compressWithBuffer(input, buffer) ++ flushWithBuffer(buffer)
|
||||
}
|
||||
override final def compressAndFinish(input: ByteString): ByteString = {
|
||||
val buffer = newTempBuffer(input.size)
|
||||
|
||||
compressWithBuffer(input, buffer) ++ finishWithBuffer(buffer)
|
||||
}
|
||||
override final def compress(input: ByteString): ByteString = compressWithBuffer(input, newTempBuffer())
|
||||
override final def flush(): ByteString = flushWithBuffer(newTempBuffer())
|
||||
override final def finish(): ByteString = finishWithBuffer(newTempBuffer())
|
||||
|
||||
protected def compressWithBuffer(input: ByteString, buffer: Array[Byte]): ByteString = {
|
||||
require(deflater.needsInput())
|
||||
deflater.setInput(input.toArray)
|
||||
drainDeflater(deflater, buffer)
|
||||
}
|
||||
protected def flushWithBuffer(buffer: Array[Byte]): ByteString = {
|
||||
val written = deflater.deflate(buffer, 0, buffer.length, Deflater.SYNC_FLUSH)
|
||||
ByteString.fromArray(buffer, 0, written)
|
||||
}
|
||||
protected def finishWithBuffer(buffer: Array[Byte]): ByteString = {
|
||||
deflater.finish()
|
||||
val res = drainDeflater(deflater, buffer)
|
||||
deflater.end()
|
||||
res
|
||||
}
|
||||
|
||||
def close(): Unit = deflater.end()
|
||||
|
||||
private def newTempBuffer(size: Int = 65536): Array[Byte] = {
|
||||
// The default size is somewhat arbitrary, we'd like to guess a better value but Deflater/zlib
|
||||
// is buffering in an unpredictable manner.
|
||||
// `compress` will only return any data if the buffered compressed data has some size in
|
||||
// the region of 10000-50000 bytes.
|
||||
// `flush` and `finish` will return any size depending on the previous input.
|
||||
// This value will hopefully provide a good compromise between memory churn and
|
||||
// excessive fragmentation of ByteStrings.
|
||||
// We also make sure that buffer size stays within a reasonable range, to avoid
|
||||
// draining deflator with too small buffer.
|
||||
new Array[Byte](math.max(size, MinBufferSize))
|
||||
}
|
||||
}
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] object DeflateCompressor {
|
||||
val MinBufferSize = 1024
|
||||
|
||||
@tailrec
|
||||
def drainDeflater(deflater: Deflater, buffer: Array[Byte], result: ByteStringBuilder = new ByteStringBuilder()): ByteString = {
|
||||
val len = deflater.deflate(buffer)
|
||||
if (len > 0) {
|
||||
result ++= ByteString.fromArray(buffer, 0, len)
|
||||
drainDeflater(deflater, buffer, result)
|
||||
} else {
|
||||
require(deflater.needsInput())
|
||||
result.result()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import java.util.zip.Inflater
|
||||
|
||||
import akka.stream.Attributes
|
||||
import akka.stream.impl.io.ByteStringParser
|
||||
import akka.stream.impl.io.ByteStringParser.{ ParseResult, ParseStep }
|
||||
import akka.util.ByteString
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] class DeflateDecompressor(maxBytesPerChunk: Int = DeflateDecompressorBase.MaxBytesPerChunkDefault)
|
||||
extends DeflateDecompressorBase(maxBytesPerChunk) {
|
||||
|
||||
override def createLogic(attr: Attributes) = new DecompressorParsingLogic {
|
||||
override val inflater: Inflater = new Inflater()
|
||||
|
||||
override val inflateState = new Inflate(true) {
|
||||
override def onTruncation(): Unit = completeStage()
|
||||
}
|
||||
|
||||
override def afterInflate = inflateState
|
||||
override def afterBytesRead(buffer: Array[Byte], offset: Int, length: Int): Unit = {}
|
||||
|
||||
startWith(inflateState)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import java.util.zip.Inflater
|
||||
|
||||
import akka.stream.impl.io.ByteStringParser
|
||||
import akka.stream.impl.io.ByteStringParser.{ ParseResult, ParseStep }
|
||||
import akka.util.ByteString
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] abstract class DeflateDecompressorBase(maxBytesPerChunk: Int = DeflateDecompressorBase.MaxBytesPerChunkDefault)
|
||||
extends ByteStringParser[ByteString] {
|
||||
|
||||
abstract class DecompressorParsingLogic extends ParsingLogic {
|
||||
val inflater: Inflater
|
||||
def afterInflate: ParseStep[ByteString]
|
||||
def afterBytesRead(buffer: Array[Byte], offset: Int, length: Int): Unit
|
||||
val inflateState: Inflate
|
||||
|
||||
abstract class Inflate(noPostProcessing: Boolean) extends ParseStep[ByteString] {
|
||||
override def canWorkWithPartialData = true
|
||||
override def parse(reader: ByteStringParser.ByteReader): ParseResult[ByteString] = {
|
||||
inflater.setInput(reader.remainingData.toArray)
|
||||
|
||||
val buffer = new Array[Byte](maxBytesPerChunk)
|
||||
val read = inflater.inflate(buffer)
|
||||
|
||||
reader.skip(reader.remainingSize - inflater.getRemaining)
|
||||
|
||||
if (read > 0) {
|
||||
afterBytesRead(buffer, 0, read)
|
||||
val next = if (inflater.finished()) afterInflate else this
|
||||
ParseResult(Some(ByteString.fromArray(buffer, 0, read)), next, noPostProcessing)
|
||||
} else {
|
||||
if (inflater.finished()) ParseResult(None, afterInflate, noPostProcessing)
|
||||
else throw ByteStringParser.NeedMoreData
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override def postStop(): Unit = inflater.end()
|
||||
}
|
||||
}
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] object DeflateDecompressorBase {
|
||||
final val MaxBytesPerChunkDefault = 64 * 1024
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import java.util.zip.{ CRC32, Deflater }
|
||||
|
||||
import akka.util.ByteString
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] class GzipCompressor extends DeflateCompressor {
|
||||
override protected lazy val deflater = new Deflater(Deflater.BEST_COMPRESSION, true)
|
||||
private val checkSum = new CRC32 // CRC32 of uncompressed data
|
||||
private var headerSent = false
|
||||
private var bytesRead = 0L
|
||||
|
||||
override protected def compressWithBuffer(input: ByteString, buffer: Array[Byte]): ByteString = {
|
||||
updateCrc(input)
|
||||
header() ++ super.compressWithBuffer(input, buffer)
|
||||
}
|
||||
override protected def flushWithBuffer(buffer: Array[Byte]): ByteString = header() ++ super.flushWithBuffer(buffer)
|
||||
override protected def finishWithBuffer(buffer: Array[Byte]): ByteString = header() ++ super.finishWithBuffer(buffer) ++ trailer()
|
||||
|
||||
private def updateCrc(input: ByteString): Unit = {
|
||||
checkSum.update(input.toArray)
|
||||
bytesRead += input.length
|
||||
}
|
||||
private def header(): ByteString =
|
||||
if (!headerSent) {
|
||||
headerSent = true
|
||||
GzipDecompressor.Header
|
||||
} else ByteString.empty
|
||||
|
||||
private def trailer(): ByteString = {
|
||||
def int32(i: Int): ByteString = ByteString(i, i >> 8, i >> 16, i >> 24)
|
||||
val crc = checkSum.getValue.toInt
|
||||
val tot = bytesRead.toInt // truncated to 32bit as specified in https://tools.ietf.org/html/rfc1952#section-2
|
||||
val trailer = int32(crc) ++ int32(tot)
|
||||
|
||||
trailer
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.impl.io.compression
|
||||
|
||||
import java.util.zip.{ CRC32, Inflater, ZipException }
|
||||
|
||||
import akka.stream.Attributes
|
||||
import akka.stream.impl.io.ByteStringParser
|
||||
import akka.stream.impl.io.ByteStringParser.{ ParseResult, ParseStep }
|
||||
import akka.util.ByteString
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] class GzipDecompressor(maxBytesPerChunk: Int = DeflateDecompressorBase.MaxBytesPerChunkDefault)
|
||||
extends DeflateDecompressorBase(maxBytesPerChunk) {
|
||||
|
||||
override def createLogic(attr: Attributes) = new DecompressorParsingLogic {
|
||||
override val inflater: Inflater = new Inflater(true)
|
||||
override def afterInflate: ParseStep[ByteString] = ReadTrailer
|
||||
override def afterBytesRead(buffer: Array[Byte], offset: Int, length: Int): Unit =
|
||||
crc32.update(buffer, offset, length)
|
||||
|
||||
trait Step extends ParseStep[ByteString] {
|
||||
override def onTruncation(): Unit = failStage(new ZipException("Truncated GZIP stream"))
|
||||
}
|
||||
override val inflateState = new Inflate(false) with Step
|
||||
startWith(ReadHeaders)
|
||||
|
||||
/** Reading the header bytes */
|
||||
case object ReadHeaders extends Step {
|
||||
override def parse(reader: ByteStringParser.ByteReader): ParseResult[ByteString] = {
|
||||
import reader._
|
||||
if (readByte() != 0x1F || readByte() != 0x8B) fail("Not in GZIP format") // check magic header
|
||||
if (readByte() != 8) fail("Unsupported GZIP compression method") // check compression method
|
||||
val flags = readByte()
|
||||
skip(6) // skip MTIME, XFL and OS fields
|
||||
if ((flags & 4) > 0) skip(readShortLE()) // skip optional extra fields
|
||||
if ((flags & 8) > 0) skipZeroTerminatedString() // skip optional file name
|
||||
if ((flags & 16) > 0) skipZeroTerminatedString() // skip optional file comment
|
||||
if ((flags & 2) > 0 && crc16(fromStartToHere) != readShortLE()) fail("Corrupt GZIP header")
|
||||
|
||||
inflater.reset()
|
||||
crc32.reset()
|
||||
ParseResult(None, inflateState, false)
|
||||
}
|
||||
}
|
||||
var crc32: CRC32 = new CRC32
|
||||
private def fail(msg: String) = throw new ZipException(msg)
|
||||
|
||||
/** Reading the trailer */
|
||||
case object ReadTrailer extends Step {
|
||||
override def parse(reader: ByteStringParser.ByteReader): ParseResult[ByteString] = {
|
||||
import reader._
|
||||
if (readIntLE() != crc32.getValue.toInt) fail("Corrupt data (CRC32 checksum error)")
|
||||
if (readIntLE() != inflater.getBytesWritten.toInt /* truncated to 32bit */ )
|
||||
fail("Corrupt GZIP trailer ISIZE")
|
||||
ParseResult(None, ReadHeaders, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
private def crc16(data: ByteString) = {
|
||||
val crc = new CRC32
|
||||
crc.update(data.toArray)
|
||||
crc.getValue.toInt & 0xFFFF
|
||||
}
|
||||
}
|
||||
|
||||
/** INTERNAL API */
|
||||
private[akka] object GzipDecompressor {
|
||||
// RFC 1952: http://tools.ietf.org/html/rfc1952 section 2.2
|
||||
private[impl] val Header = ByteString(
|
||||
0x1F, // ID1
|
||||
0x8B, // ID2
|
||||
8, // CM = Deflate
|
||||
0, // FLG
|
||||
0, // MTIME 1
|
||||
0, // MTIME 2
|
||||
0, // MTIME 3
|
||||
0, // MTIME 4
|
||||
0, // XFL
|
||||
0 // OS
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/**
|
||||
* Copyright (C) 2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.javadsl
|
||||
|
||||
import akka.NotUsed
|
||||
import akka.stream.scaladsl
|
||||
import akka.util.ByteString
|
||||
|
||||
object Compression {
|
||||
/**
|
||||
* Creates a Flow that decompresses gzip-compressed stream of data.
|
||||
*
|
||||
* @param maxBytesPerChunk Maximum length of the output [[ByteString]] chunk.
|
||||
*/
|
||||
def gunzip(maxBytesPerChunk: Int): Flow[ByteString, ByteString, NotUsed] =
|
||||
scaladsl.Compression.gunzip(maxBytesPerChunk).asJava
|
||||
|
||||
/**
|
||||
* Creates a Flow that decompresses deflate-compressed stream of data.
|
||||
*
|
||||
* @param maxBytesPerChunk Maximum length of the output [[ByteString]] chunk.
|
||||
*/
|
||||
def inflate(maxBytesPerChunk: Int): Flow[ByteString, ByteString, NotUsed] =
|
||||
scaladsl.Compression.inflate(maxBytesPerChunk).asJava
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Copyright (C) 2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.stream.scaladsl
|
||||
|
||||
import akka.NotUsed
|
||||
import akka.stream.impl.io.compression._
|
||||
import akka.util.ByteString
|
||||
|
||||
object Compression {
|
||||
/**
|
||||
* Creates a flow that gzip-compresses a stream of ByteStrings. Note that the compressor
|
||||
* will SYNC_FLUSH after every [[ByteString]] so that it is guaranteed that every [[ByteString]]
|
||||
* coming out of the flow can be fully decompressed without waiting for additional data. This may
|
||||
* come at a compression performance cost for very small chunks.
|
||||
*
|
||||
* FIXME: should compression level / strategy / flush mode be configurable? See https://github.com/akka/akka/issues/21849
|
||||
*/
|
||||
def gzip: Flow[ByteString, ByteString, NotUsed] =
|
||||
CompressionUtils.compressorFlow(() ⇒ new GzipCompressor)
|
||||
|
||||
/**
|
||||
* Creates a Flow that decompresses a gzip-compressed stream of data.
|
||||
*
|
||||
* @param maxBytesPerChunk Maximum length of an output [[ByteString]] chunk.
|
||||
*/
|
||||
def gunzip(maxBytesPerChunk: Int = DeflateDecompressorBase.MaxBytesPerChunkDefault): Flow[ByteString, ByteString, NotUsed] =
|
||||
Flow[ByteString].via(new GzipDecompressor(maxBytesPerChunk))
|
||||
.named("gunzip")
|
||||
|
||||
/**
|
||||
* Creates a flow that deflate-compresses a stream of ByteString. Note that the compressor
|
||||
* will SYNC_FLUSH after every [[ByteString]] so that it is guaranteed that every [[ByteString]]
|
||||
* coming out of the flow can be fully decompressed without waiting for additional data. This may
|
||||
* come at a compression performance cost for very small chunks.
|
||||
*
|
||||
* FIXME: should compression level / strategy / flush mode be configurable? See https://github.com/akka/akka/issues/21849
|
||||
*/
|
||||
def deflate: Flow[ByteString, ByteString, NotUsed] =
|
||||
CompressionUtils.compressorFlow(() ⇒ new DeflateCompressor)
|
||||
|
||||
/**
|
||||
* Creates a Flow that decompresses a deflate-compressed stream of data.
|
||||
*
|
||||
* @param maxBytesPerChunk Maximum length of an output [[ByteString]] chunk.
|
||||
*/
|
||||
def inflate(maxBytesPerChunk: Int = DeflateDecompressorBase.MaxBytesPerChunkDefault): Flow[ByteString, ByteString, NotUsed] =
|
||||
Flow[ByteString].via(new DeflateDecompressor(maxBytesPerChunk))
|
||||
.named("inflate")
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue