raboof/pekko
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions 10

!str #19005 make groupBy et al return a SubFlow

Browse source 
A SubFlow (or SubSource) is not a Graph, it is an unfinished builder
that accepts transformations. This allows us to capture the substreams’
transformations before materializing the flow, which will be very
helpful in fully fusing all operators.

Another change is that groupBy now requires a maxSubstreams parameter in
order to bound its resource usage. In exchange the matching merge can be
unbounded. This trades silent deadlock for explicit stream failure.

This commit also changes all uses of Predef.identity to use `conforms`
and removes the HTTP impl.util.identityFunc.
This commit is contained in:
Roland Kuhn 2015-11-25 19:58:48 +01:00
parent 654fa41443
commit 1500d1f36d
56 changed files with 3484 additions and 720 deletions
Download patch file Download diff file Expand all files Collapse all files

139
akka-stream/src/main/scala/akka/stream/javadsl/Flow.scala
View file

@ -10,11 +10,11 @@ import akka.stream.impl.{ ConstantFun, StreamLayout }
import akka.stream.{ scaladsl, _ }
import akka.stream.stage.Stage
import org.reactivestreams.Processor
import scala.annotation.unchecked.uncheckedVariance
import scala.collection.immutable
import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration
import akka.japi.Util
object Flow {
@ -78,7 +78,6 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
/**
* Transform this [[Flow]] by appending the given processing steps.
*
* {{{
* +----------------------------+
* | Resulting Flow |
@ -90,10 +89,9 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* | +------+ +------+ |
* +----------------------------+
* }}}
*
* The materialized value of the combined [[Flow]] will be the materialized
* value of the current flow (ignoring the other Flow’s value), use
* [[Flow#viaMat viaMat]] if a different strategy is needed.
* `viaMat` if a different strategy is needed.
*/
def via[T, M](flow: Graph[FlowShape[Out, T], M]): javadsl.Flow[In, T, Mat] =
new Flow(delegate.via(flow))
@ -119,24 +117,70 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
/**
* Connect this [[Flow]] to a [[Sink]], concatenating the processing steps of both.
* {{{
* +----------------------------+
* | Resulting Sink |
* | |
* | +------+ +------+ |
* | | | | | |
* In ~~> | flow | ~Out~> | sink | |
* | | | | | |
* | +------+ +------+ |
* +----------------------------+
* }}}
* The materialized value of the combined [[Sink]] will be the materialized
* value of the current flow (ignoring the given Sink’s value), use
* `toMat` if a different strategy is needed.
*/
def to(sink: Graph[SinkShape[Out], _]): javadsl.Sink[In, Mat] =
new Sink(delegate.to(sink))
/**
* Connect this [[Flow]] to a [[Sink]], concatenating the processing steps of both.
* {{{
* +----------------------------+
* | Resulting Sink |
* | |
* | +------+ +------+ |
* | | | | | |
* In ~~> | flow | ~Out~> | sink | |
* | | | | | |
* | +------+ +------+ |
* +----------------------------+
* }}}
* The `combine` function is used to compose the materialized values of this flow and that
* Sink into the materialized value of the resulting Sink.
*/
def toMat[M, M2](sink: Graph[SinkShape[Out], M], combine: function.Function2[Mat, M, M2]): javadsl.Sink[In, M2] =
new Sink(delegate.toMat(sink)(combinerToScala(combine)))
/**
* Join this [[Flow]] to another [[Flow]], by cross connecting the inputs and outputs, creating a [[RunnableGraph]]
* Join this [[Flow]] to another [[Flow]], by cross connecting the inputs and outputs, creating a [[RunnableGraph]].
* {{{
* +------+ +-------+
* | | ~Out~> | |
* | this | | other |
* | | <~In~ | |
* +------+ +-------+
* }}}
* The materialized value of the combined [[Flow]] will be the materialized
* value of the current flow (ignoring the other Flow’s value), use
* `joinMat` if a different strategy is needed.
*/
def join[M](flow: Graph[FlowShape[Out, In], M]): javadsl.RunnableGraph[Mat] =
RunnableGraph.fromGraph(delegate.join(flow))
/**
* Join this [[Flow]] to another [[Flow]], by cross connecting the inputs and outputs, creating a [[RunnableGraph]]
* {{{
* +------+ +-------+
* | | ~Out~> | |
* | this | | other |
* | | <~In~ | |
* +------+ +-------+
* }}}
* The `combine` function is used to compose the materialized values of this flow and that
* Flow into the materialized value of the resulting Flow.
*/
def joinMat[M, M2](flow: Graph[FlowShape[Out, In], M], combine: function.Function2[Mat, M, M2]): javadsl.RunnableGraph[M2] =
RunnableGraph.fromGraph(delegate.joinMat(flow)(combinerToScala(combine)))
@ -231,13 +275,7 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* '''Cancels when''' downstream cancels
*/
def mapConcat[T](f: function.Function[Out, java.lang.Iterable[T]]): javadsl.Flow[In, T, Mat] =
new Flow(delegate.mapConcat { elem ⇒
val scalaIterable = new immutable.Iterable[T] {
import collection.JavaConverters._
override def iterator: Iterator[T] = f(elem).iterator().asScala
}
scalaIterable
})
new Flow(delegate.mapConcat { elem ⇒ Util.immutableSeq(f(elem)) })
/**
* Transform this stream by applying the given function to each of the elements
@ -577,7 +615,6 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* '''Completes when''' upstream completes or upstream failed with exception pf can handle
*
* '''Cancels when''' downstream cancels
*
*/
def recover[T >: Out](pf: PartialFunction[Throwable, T]): javadsl.Flow[In, T, Mat] =
new Flow(delegate.recover(pf))
@ -706,7 +743,7 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
new Flow(delegate.transform(() ⇒ mkStage.create()))
/**
* Takes up to `n` elements from the stream (less than `n` only if the upstream completes before emitting `n` elements)
* Takes up to `n` elements from the stream (less than `n` if the upstream completes before emitting `n` elements)
* and returns a pair containing a strict sequence of the taken element
* and a stream representing the remaining elements. If ''n'' is zero or negative, then this will return a pair
* of an empty collection and a stream containing the whole upstream unchanged.
@ -733,12 +770,20 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* This operation demultiplexes the incoming stream into separate output
* streams, one for each element key. The key is computed for each element
* using the given function. When a new key is encountered for the first time
* it is emitted to the downstream subscriber together with a fresh
* flow that will eventually produce all the elements of the substream
* for that key. Not consuming the elements from the created streams will
* stop this processor from processing more elements, therefore you must take
* care to unblock (or cancel) all of the produced streams even if you want
* to consume only one of them.
* a new substream is opened and subsequently fed with all elements belonging to
* that key.
*
* The object returned from this method is not a normal [[Flow]],
* it is a [[SubFlow]]. This means that after this combinator all transformations
* are applied to all encountered substreams in the same fashion. Substream mode
* is exited either by closing the substream (i.e. connecting it to a [[Sink]])
* or by merging the substreams back together; see the `to` and `mergeBack` methods
* on [[SubFlow]] for more information.
*
* It is important to note that the substreams also propagate back-pressure as
* any other stream, which means that blocking one substream will block the `groupBy`
* operator itself—and thereby all substreams—once all internal or
* explicit buffers are filled.
*
* If the group by function `f` throws an exception and the supervision decision
* is [[akka.stream.Supervision#stop]] the stream and substreams will be completed
@ -757,9 +802,11 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
*
* '''Cancels when''' downstream cancels and all substreams cancel
*
* @param maxSubstreams configures the maximum number of substreams (keys)
* that are supported; if more distinct keys are encountered then the stream fails
*/
def groupBy[K](f: function.Function[Out, K]): javadsl.Flow[In, akka.japi.Pair[K, javadsl.Source[Out @uncheckedVariance, Unit]], Mat] =
new Flow(delegate.groupBy(f.apply).map { case (k, p) ⇒ akka.japi.Pair(k, p.asJava) }) // TODO optimize to one step
def groupBy[K](maxSubstreams: Int, f: function.Function[Out, K]): SubFlow[In, Out @uncheckedVariance, Mat] =
new SubFlow(delegate.groupBy(maxSubstreams, f.apply))
/**
* This operation applies the given predicate to all incoming elements and
@ -782,6 +829,18 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* true, false // subsequent substreams operate the same way
* }}}
*
* The object returned from this method is not a normal [[Flow]],
* it is a [[SubFlow]]. This means that after this combinator all transformations
* are applied to all encountered substreams in the same fashion. Substream mode
* is exited either by closing the substream (i.e. connecting it to a [[Sink]])
* or by merging the substreams back together; see the `to` and `mergeBack` methods
* on [[SubFlow]] for more information.
*
* It is important to note that the substreams also propagate back-pressure as
* any other stream, which means that blocking one substream will block the `splitWhen`
* operator itself—and thereby all substreams—once all internal or
* explicit buffers are filled.
*
* If the split predicate `p` throws an exception and the supervision decision
* is [[akka.stream.Supervision#stop]] the stream and substreams will be completed
* with failure.
@ -802,8 +861,8 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
*
* See also [[Flow.splitAfter]].
*/
def splitWhen(p: function.Predicate[Out]): javadsl.Flow[In, Source[Out, Unit], Mat] =
new Flow(delegate.splitWhen(p.test).map(_.asJava))
def splitWhen(p: function.Predicate[Out]): SubFlow[In, Out, Mat] =
new SubFlow(delegate.splitWhen(p.test))
/**
* This operation applies the given predicate to all incoming elements and
@ -817,6 +876,18 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* false, false, true // elements go into third substream
* }}}
*
* The object returned from this method is not a normal [[Flow]],
* it is a [[SubFlow]]. This means that after this combinator all transformations
* are applied to all encountered substreams in the same fashion. Substream mode
* is exited either by closing the substream (i.e. connecting it to a [[Sink]])
* or by merging the substreams back together; see the `to` and `mergeBack` methods
* on [[SubFlow]] for more information.
*
* It is important to note that the substreams also propagate back-pressure as
* any other stream, which means that blocking one substream will block the `splitAfter`
* operator itself—and thereby all substreams—once all internal or
* explicit buffers are filled.
*
* If the split predicate `p` throws an exception and the supervision decision
* is [[akka.stream.Supervision.Stop]] the stream and substreams will be completed
* with failure.
@ -837,8 +908,8 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
*
* See also [[Flow.splitWhen]].
*/
def splitAfter[U >: Out](p: function.Predicate[Out]): javadsl.Flow[In, Source[Out, Unit], Mat] =
new Flow(delegate.splitAfter(p.test).map(_.asJava))
def splitAfter[U >: Out](p: function.Predicate[Out]): SubFlow[In, Out, Mat] =
new SubFlow(delegate.splitAfter(p.test))
/**
* Transform each input element into a `Source` of output elements that is
@ -853,8 +924,8 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
*
* '''Cancels when''' downstream cancels
*/
def flatMapConcat[T, M](f: function.Function[Out, Source[T, M]]): Flow[In, T, Mat] =
new Flow(delegate.flatMapConcat[T, M](x ⇒ f(x).asScala))
def flatMapConcat[T, M](f: function.Function[Out, _ <: Graph[SourceShape[T], M]]): Flow[In, T, Mat] =
new Flow(delegate.flatMapConcat[T, M](x ⇒ f(x)))
/**
* Transform each input element into a `Source` of output elements that is
@ -869,8 +940,8 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
*
* '''Cancels when''' downstream cancels
*/
def flatMapMerge[T, M](breadth: Int, f: function.Function[Out, Source[T, M]]): Flow[In, T, Mat] =
new Flow(delegate.flatMapMerge(breadth, o ⇒ f(o).asScala))
def flatMapMerge[T, M](breadth: Int, f: function.Function[Out, _ <: Graph[SourceShape[T], M]]): Flow[In, T, Mat] =
new Flow(delegate.flatMapMerge(breadth, o ⇒ f(o)))
/**
* Concatenate the given [[Source]] to this [[Flow]], meaning that once this
@ -1226,6 +1297,14 @@ final class Flow[-In, +Out, +Mat](delegate: scaladsl.Flow[In, Out, Mat]) extends
* This can be adjusted according to your needs by providing a custom [[Attributes.LogLevels]] attribute on the given Flow.
*
* Uses an internally created [[LoggingAdapter]] which uses `akka.stream.Log` as it's source (use this class to configure slf4j loggers).
*
* '''Emits when''' the mapping function returns an element
*
* '''Backpressures when''' downstream backpressures
*
* '''Completes when''' upstream completes
*
* '''Cancels when''' downstream cancels
*/
def log(name: String): javadsl.Flow[In, Out, Mat] =
this.log(name, ConstantFun.javaIdentityFunction[Out], null)