373 lines
16 KiB
Scala
373 lines
16 KiB
Scala
|
/**
|
||
|
|
* Copyright (C) 2015 Typesafe Inc. <http://www.typesafe.com>
|
||
|
|
*/
|
||
|
|
package akka.stream.impl
|
||
|
|
|
||
|
|
import akka.stream.scaladsl.{ Keep, OperationAttributes }
|
||
|
|
import akka.stream._
|
||
|
|
import org.reactivestreams.{ Subscription, Publisher, Subscriber }
|
||
|
|
import akka.event.Logging.simpleName
|
||
|
|
import scala.collection.mutable
|
||
|
|
|
||
|
|
/**
|
||
|
|
* INTERNAL API
|
||
|
|
*/
|
||
|
|
private[akka] object StreamLayout {
|
||
|
|
|
||
|
|
// compile-time constant
|
||
|
|
val debug = true
|
||
|
|
|
||
|
|
// TODO: Materialization order
|
||
|
|
// TODO: Special case linear composites
|
||
|
|
// TODO: Cycles
|
||
|
|
|
||
|
|
sealed trait MaterializedValueNode
|
||
|
|
case class Combine(f: (Any, Any) ⇒ Any, dep1: MaterializedValueNode, dep2: MaterializedValueNode) extends MaterializedValueNode
|
||
|
|
case class Atomic(module: Module) extends MaterializedValueNode
|
||
|
|
case class Transform(f: Any ⇒ Any, dep: MaterializedValueNode) extends MaterializedValueNode
|
||
|
|
case object Ignore extends MaterializedValueNode
|
||
|
|
|
||
|
|
trait Module {
|
||
|
|
def shape: Shape
|
||
|
|
/**
|
||
|
|
* Verify that the given Shape has the same ports and return a new module with that shape.
|
||
|
|
* Concrete implementations may throw UnsupportedOperationException where applicable.
|
||
|
|
*/
|
||
|
|
def replaceShape(s: Shape): Module
|
||
|
|
|
||
|
|
lazy val inPorts: Set[InPort] = shape.inlets.toSet
|
||
|
|
lazy val outPorts: Set[OutPort] = shape.outlets.toSet
|
||
|
|
|
||
|
|
def isRunnable: Boolean = inPorts.isEmpty && outPorts.isEmpty
|
||
|
|
def isSink: Boolean = (inPorts.size == 1) && outPorts.isEmpty
|
||
|
|
def isSource: Boolean = (outPorts.size == 1) && inPorts.isEmpty
|
||
|
|
def isFlow: Boolean = (inPorts.size == 1) && (outPorts.size == 1)
|
||
|
|
|
||
|
|
def growConnect(that: Module, from: OutPort, to: InPort): Module =
|
||
|
|
growConnect(that, from, to, Keep.left)
|
||
|
|
|
||
|
|
def growConnect[A, B, C](that: Module, from: OutPort, to: InPort, f: (A, B) ⇒ C): Module =
|
||
|
|
this.grow(that, f).connect(from, to)
|
||
|
|
|
||
|
|
def connect[A, B](from: OutPort, to: InPort): Module = {
|
||
|
|
if (debug) validate()
|
||
|
|
|
||
|
|
require(outPorts(from),
|
||
|
|
if (downstreams.contains(from)) s"The output port [$from] is already connected"
|
||
|
|
else s"The output port [$from] is not part of the underlying graph.")
|
||
|
|
require(inPorts(to),
|
||
|
|
if (upstreams.contains(to)) s"The input port [$to] is already connected"
|
||
|
|
else s"The input port [$to] is not part of the underlying graph.")
|
||
|
|
|
||
|
|
CompositeModule(
|
||
|
|
subModules,
|
||
|
|
AmorphousShape(shape.inlets.filterNot(_ == to), shape.outlets.filterNot(_ == from)),
|
||
|
|
(from, to) :: connections,
|
||
|
|
materializedValueComputation,
|
||
|
|
attributes)
|
||
|
|
}
|
||
|
|
|
||
|
|
def transformMaterializedValue(f: Any ⇒ Any): Module = {
|
||
|
|
if (debug) validate()
|
||
|
|
|
||
|
|
CompositeModule(
|
||
|
|
subModules = if (this.isAtomic) Set(this) else this.subModules,
|
||
|
|
shape,
|
||
|
|
connections,
|
||
|
|
Transform(f, this.materializedValueComputation),
|
||
|
|
attributes)
|
||
|
|
}
|
||
|
|
|
||
|
|
def grow(that: Module): Module = grow(that, Keep.left)
|
||
|
|
|
||
|
|
def grow[A, B, C](that: Module, f: (A, B) ⇒ C): Module = {
|
||
|
|
if (debug) validate()
|
||
|
|
|
||
|
|
require(that ne this, "A module cannot be added to itself. You should pass a separate instance to grow().")
|
||
|
|
require(!subModules(that), "An existing submodule cannot be added again. All contained modules must be unique.")
|
||
|
|
|
||
|
|
val modules1 = if (this.isAtomic) Set(this) else this.subModules
|
||
|
|
val modules2 = if (that.isAtomic) Set(that) else that.subModules
|
||
|
|
|
||
|
|
CompositeModule(
|
||
|
|
modules1 ++ modules2,
|
||
|
|
AmorphousShape(shape.inlets ++ that.shape.inlets, shape.outlets ++ that.shape.outlets),
|
||
|
|
connections reverse_::: that.connections,
|
||
|
|
if (f eq Keep.left) materializedValueComputation
|
||
|
|
else if (f eq Keep.right) that.materializedValueComputation
|
||
|
|
else Combine(f.asInstanceOf[(Any, Any) ⇒ Any], this.materializedValueComputation, that.materializedValueComputation),
|
||
|
|
attributes)
|
||
|
|
}
|
||
|
|
|
||
|
|
def wrap(): Module = {
|
||
|
|
if (debug) validate()
|
||
|
|
|
||
|
|
CompositeModule(
|
||
|
|
subModules = Set(this),
|
||
|
|
shape,
|
||
|
|
connections,
|
||
|
|
/*
|
||
|
|
* Wrapping like this shields the outer module from the details of the
|
||
|
|
* materialized value computation of its submodules, which is important
|
||
|
|
* to keep the re-binding of identities to computation nodes manageable
|
||
|
|
* in carbonCopy.
|
||
|
|
*/
|
||
|
|
Atomic(this),
|
||
|
|
OperationAttributes.none)
|
||
|
|
}
|
||
|
|
|
||
|
|
def subModules: Set[Module]
|
||
|
|
def isAtomic: Boolean = subModules.isEmpty
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A list of connections whose port-wise ordering is STABLE across carbonCopy.
|
||
|
|
*/
|
||
|
|
def connections: List[(OutPort, InPort)] = Nil
|
||
|
|
final lazy val downstreams: Map[OutPort, InPort] = connections.toMap
|
||
|
|
final lazy val upstreams: Map[InPort, OutPort] = connections.map(_.swap).toMap
|
||
|
|
|
||
|
|
def materializedValueComputation: MaterializedValueNode = Atomic(this)
|
||
|
|
def carbonCopy: Module
|
||
|
|
|
||
|
|
def attributes: OperationAttributes
|
||
|
|
def withAttributes(attributes: OperationAttributes): Module
|
||
|
|
|
||
|
|
final override def hashCode(): Int = super.hashCode()
|
||
|
|
final override def equals(obj: scala.Any): Boolean = super.equals(obj)
|
||
|
|
|
||
|
|
def validate(level: Int = 0, doPrint: Boolean = false, idMap: mutable.Map[AnyRef, Int] = mutable.Map.empty): Unit = {
|
||
|
|
val ids = Iterator from 1
|
||
|
|
def id(obj: AnyRef) = idMap get obj match {
|
||
|
|
case Some(x) ⇒ x
|
||
|
|
case None ⇒
|
||
|
|
val x = ids.next()
|
||
|
|
idMap(obj) = x
|
||
|
|
x
|
||
|
|
}
|
||
|
|
def in(i: InPort) = s"${i.toString}@${id(i)}"
|
||
|
|
def out(o: OutPort) = s"${o.toString}@${id(o)}"
|
||
|
|
def ins(i: Iterable[InPort]) = i.map(in).mkString("In[", ",", "]")
|
||
|
|
def outs(o: Iterable[OutPort]) = o.map(out).mkString("Out[", ",", "]")
|
||
|
|
def pair(p: (OutPort, InPort)) = s"${in(p._2)}->${out(p._1)}"
|
||
|
|
def pairs(p: Iterable[(OutPort, InPort)]) = p.map(pair).mkString("[", ",", "]")
|
||
|
|
|
||
|
|
val inset: Set[InPort] = shape.inlets.toSet
|
||
|
|
val outset: Set[OutPort] = shape.outlets.toSet
|
||
|
|
var problems: List[String] = Nil
|
||
|
|
|
||
|
|
if (inset.size != shape.inlets.size) problems ::= "shape has duplicate inlets: " + ins(shape.inlets)
|
||
|
|
if (inset != inPorts) problems ::= s"shape has extra ${ins(inset -- inPorts)}, module has extra ${ins(inPorts -- inset)}"
|
||
|
|
if (inset.intersect(upstreams.keySet).nonEmpty) problems ::= s"found connected inlets ${inset.intersect(upstreams.keySet)}"
|
||
|
|
if (outset.size != shape.outlets.size) problems ::= "shape has duplicate outlets: " + outs(shape.outlets)
|
||
|
|
if (outset != outPorts) problems ::= s"shape has extra ${outs(outset -- outPorts)}, module has extra ${outs(outPorts -- outset)}"
|
||
|
|
if (outset.intersect(downstreams.keySet).nonEmpty) problems ::= s"found connected outlets ${outset.intersect(downstreams.keySet)}"
|
||
|
|
val ups = upstreams.toSet
|
||
|
|
val ups2 = ups.map(_.swap)
|
||
|
|
val downs = downstreams.toSet
|
||
|
|
val inter = ups2.intersect(downs)
|
||
|
|
if (downs != ups2) problems ::= s"inconsistent maps: ups ${pairs(ups2 -- inter)} downs ${pairs(downs -- inter)}"
|
||
|
|
val (allIn, dupIn, allOut, dupOut) =
|
||
|
|
subModules.foldLeft((Set.empty[InPort], Set.empty[InPort], Set.empty[OutPort], Set.empty[OutPort])) {
|
||
|
|
case ((ai, di, ao, doo), m) ⇒ (ai ++ m.inPorts, di ++ ai.intersect(m.inPorts), ao ++ m.outPorts, doo ++ ao.intersect(m.outPorts))
|
||
|
|
}
|
||
|
|
if (dupIn.nonEmpty) problems ::= s"duplicate ports in submodules ${ins(dupIn)}"
|
||
|
|
if (dupOut.nonEmpty) problems ::= s"duplicate ports in submodules ${outs(dupOut)}"
|
||
|
|
if (!isAtomic && (inset -- allIn).nonEmpty) problems ::= s"foreign inlets ${ins(inset -- allIn)}"
|
||
|
|
if (!isAtomic && (outset -- allOut).nonEmpty) problems ::= s"foreign outlets ${outs(outset -- allOut)}"
|
||
|
|
val unIn = allIn -- inset -- upstreams.keySet
|
||
|
|
if (unIn.nonEmpty) problems ::= s"unconnected inlets ${ins(unIn)}"
|
||
|
|
val unOut = allOut -- outset -- downstreams.keySet
|
||
|
|
if (unOut.nonEmpty) problems ::= s"unconnected outlets ${outs(unOut)}"
|
||
|
|
def atomics(n: MaterializedValueNode): Set[Module] =
|
||
|
|
n match {
|
||
|
|
case Ignore ⇒ Set.empty
|
||
|
|
case Transform(f, dep) ⇒ atomics(dep)
|
||
|
|
case Atomic(m) ⇒ Set(m)
|
||
|
|
case Combine(f, left, right) ⇒ atomics(left) ++ atomics(right)
|
||
|
|
}
|
||
|
|
val atomic = atomics(materializedValueComputation)
|
||
|
|
if ((atomic -- subModules - this).nonEmpty) problems ::= s"computation refers to non-existent modules [${atomic -- subModules - this mkString ","}]"
|
||
|
|
|
||
|
|
val print = doPrint || problems.nonEmpty
|
||
|
|
|
||
|
|
if (print) {
|
||
|
|
val indent = " " * (level * 2)
|
||
|
|
println(s"$indent${simpleName(this)}($shape): ${ins(inPorts)} ${outs(outPorts)}")
|
||
|
|
downstreams foreach { case (o, i) ⇒ println(s"$indent ${out(o)} -> ${in(i)}") }
|
||
|
|
problems foreach (p ⇒ println(s"$indent -!- $p"))
|
||
|
|
}
|
||
|
|
|
||
|
|
subModules foreach (_.validate(level + 1, print, idMap))
|
||
|
|
|
||
|
|
if (problems.nonEmpty && !doPrint) throw new IllegalStateException(s"module inconsistent, found ${problems.size} problems")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
object EmptyModule extends Module {
|
||
|
|
override def shape = EmptyShape
|
||
|
|
override def replaceShape(s: Shape) =
|
||
|
|
if (s == EmptyShape) this
|
||
|
|
else throw new UnsupportedOperationException("cannot replace the shape of the EmptyModule")
|
||
|
|
|
||
|
|
override def grow(that: Module): Module = that
|
||
|
|
override def wrap(): Module = this
|
||
|
|
|
||
|
|
override def subModules: Set[Module] = Set.empty
|
||
|
|
|
||
|
|
override def withAttributes(attributes: OperationAttributes): Module =
|
||
|
|
throw new UnsupportedOperationException("EmptyModule cannot carry attributes")
|
||
|
|
override def attributes = OperationAttributes.none
|
||
|
|
|
||
|
|
override def carbonCopy: Module = this
|
||
|
|
|
||
|
|
override def isRunnable: Boolean = false
|
||
|
|
override def isAtomic: Boolean = false
|
||
|
|
override def materializedValueComputation: MaterializedValueNode = Ignore
|
||
|
|
}
|
||
|
|
|
||
|
|
final case class CompositeModule(
|
||
|
|
subModules: Set[Module],
|
||
|
|
shape: Shape,
|
||
|
|
override val connections: List[(OutPort, InPort)],
|
||
|
|
override val materializedValueComputation: MaterializedValueNode,
|
||
|
|
attributes: OperationAttributes) extends Module {
|
||
|
|
|
||
|
|
override def replaceShape(s: Shape): Module = {
|
||
|
|
shape.requireSamePortsAs(s)
|
||
|
|
copy(shape = s)
|
||
|
|
}
|
||
|
|
|
||
|
|
override def carbonCopy: Module = {
|
||
|
|
val out = mutable.Map[OutPort, OutPort]()
|
||
|
|
val in = mutable.Map[InPort, InPort]()
|
||
|
|
val subMap = mutable.Map[Module, Module]()
|
||
|
|
|
||
|
|
val subs = subModules map { s ⇒
|
||
|
|
val n = s.carbonCopy
|
||
|
|
out ++= s.shape.outlets.zip(n.shape.outlets)
|
||
|
|
in ++= s.shape.inlets.zip(n.shape.inlets)
|
||
|
|
s.connections.zip(n.connections) foreach {
|
||
|
|
case ((oldOut, oldIn), (newOut, newIn)) ⇒
|
||
|
|
out(oldOut) = newOut
|
||
|
|
in(oldIn) = newIn
|
||
|
|
}
|
||
|
|
subMap(s) = n
|
||
|
|
n
|
||
|
|
}
|
||
|
|
|
||
|
|
val newShape = shape.copyFromPorts(shape.inlets.map(in.asInstanceOf[Inlet[_] ⇒ Inlet[_]]),
|
||
|
|
shape.outlets.map(out.asInstanceOf[Outlet[_] ⇒ Outlet[_]]))
|
||
|
|
|
||
|
|
val conn = connections.map(p ⇒ (out(p._1), in(p._2)))
|
||
|
|
|
||
|
|
def mapComp(n: MaterializedValueNode): MaterializedValueNode =
|
||
|
|
n match {
|
||
|
|
case Ignore ⇒ Ignore
|
||
|
|
case Transform(f, dep) ⇒ Transform(f, mapComp(dep))
|
||
|
|
case Atomic(mod) ⇒ Atomic(subMap(mod))
|
||
|
|
case Combine(f, left, right) ⇒ Combine(f, mapComp(left), mapComp(right))
|
||
|
|
}
|
||
|
|
val comp =
|
||
|
|
try mapComp(materializedValueComputation)
|
||
|
|
catch {
|
||
|
|
case so: StackOverflowError ⇒
|
||
|
|
throw new UnsupportedOperationException("materialized value computation is too complex, please group into sub-graphs")
|
||
|
|
}
|
||
|
|
|
||
|
|
copy(subModules = subs, shape = newShape, connections = conn, materializedValueComputation = comp)
|
||
|
|
}
|
||
|
|
|
||
|
|
override def withAttributes(attributes: OperationAttributes): Module = copy(attributes = attributes)
|
||
|
|
|
||
|
|
override def toString =
|
||
|
|
s"""
|
||
|
|
| Modules: ${subModules.toSeq.map(m ⇒ " " + m.getClass.getName).mkString("\n")}
|
||
|
|
| Downstreams:
|
||
|
|
| ${downstreams.map { case (in, out) ⇒ s" $in -> $out" }.mkString("\n")}
|
||
|
|
| Upstreams:
|
||
|
|
| ${upstreams.map { case (out, in) ⇒ s" $out -> $in" }.mkString("\n")}
|
||
|
|
""".stripMargin
|
||
|
|
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* INTERNAL API
|
||
|
|
*/
|
||
|
|
private[stream] class VirtualSubscriber[T](val owner: VirtualPublisher[T]) extends Subscriber[T] {
|
||
|
|
override def onSubscribe(s: Subscription): Unit = throw new UnsupportedOperationException("This method should not be called")
|
||
|
|
override def onError(t: Throwable): Unit = throw new UnsupportedOperationException("This method should not be called")
|
||
|
|
override def onComplete(): Unit = throw new UnsupportedOperationException("This method should not be called")
|
||
|
|
override def onNext(t: T): Unit = throw new UnsupportedOperationException("This method should not be called")
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* INTERNAL API
|
||
|
|
*/
|
||
|
|
private[stream] class VirtualPublisher[T]() extends Publisher[T] {
|
||
|
|
@volatile var realPublisher: Publisher[T] = null
|
||
|
|
override def subscribe(s: Subscriber[_ >: T]): Unit = realPublisher.subscribe(s)
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* INTERNAL API
|
||
|
|
*/
|
||
|
|
private[stream] abstract class MaterializerSession(val topLevel: StreamLayout.Module) {
|
||
|
|
import StreamLayout._
|
||
|
|
|
||
|
|
private val subscribers = collection.mutable.HashMap[InPort, Subscriber[Any]]().withDefaultValue(null)
|
||
|
|
private val publishers = collection.mutable.HashMap[OutPort, Publisher[Any]]().withDefaultValue(null)
|
||
|
|
|
||
|
|
final def materialize(): Any = {
|
||
|
|
require(topLevel ne EmptyModule, "An empty module cannot be materialized (EmptyModule was given)")
|
||
|
|
require(
|
||
|
|
topLevel.isRunnable,
|
||
|
|
s"The top level module cannot be materialized because it has unconnected ports: ${(topLevel.inPorts ++ topLevel.outPorts).mkString(", ")}")
|
||
|
|
materializeModule(topLevel, topLevel.attributes)
|
||
|
|
}
|
||
|
|
|
||
|
|
protected def mergeAttributes(parent: OperationAttributes, current: OperationAttributes): OperationAttributes =
|
||
|
|
parent and current
|
||
|
|
|
||
|
|
protected def materializeModule(module: Module, effectiveAttributes: OperationAttributes): Any = {
|
||
|
|
val materializedValues = collection.mutable.HashMap.empty[Module, Any]
|
||
|
|
for (submodule ← module.subModules) {
|
||
|
|
val subEffectiveAttributes = mergeAttributes(effectiveAttributes, submodule.attributes)
|
||
|
|
if (submodule.isAtomic) materializedValues.put(submodule, materializeAtomic(submodule, subEffectiveAttributes))
|
||
|
|
else materializedValues.put(submodule, materializeComposite(submodule, subEffectiveAttributes))
|
||
|
|
}
|
||
|
|
resolveMaterialized(module.materializedValueComputation, materializedValues)
|
||
|
|
}
|
||
|
|
|
||
|
|
protected def materializeComposite(composite: Module, effectiveAttributes: OperationAttributes): Any = {
|
||
|
|
materializeModule(composite, effectiveAttributes)
|
||
|
|
}
|
||
|
|
|
||
|
|
protected def materializeAtomic(atomic: Module, effectiveAttributes: OperationAttributes): Any
|
||
|
|
|
||
|
|
private def resolveMaterialized(matNode: MaterializedValueNode, materializedValues: collection.Map[Module, Any]): Any = matNode match {
|
||
|
|
case Atomic(m) ⇒ materializedValues(m)
|
||
|
|
case Combine(f, d1, d2) ⇒ f(resolveMaterialized(d1, materializedValues), resolveMaterialized(d2, materializedValues))
|
||
|
|
case Transform(f, d) ⇒ f(resolveMaterialized(d, materializedValues))
|
||
|
|
case Ignore ⇒ ()
|
||
|
|
}
|
||
|
|
|
||
|
|
private def attach(p: Publisher[Any], s: Subscriber[Any]) = s match {
|
||
|
|
case v: VirtualSubscriber[Any] ⇒ v.owner.realPublisher = p
|
||
|
|
case _ ⇒ p.subscribe(s)
|
||
|
|
}
|
||
|
|
|
||
|
|
final protected def assignPort(in: InPort, subscriber: Subscriber[Any]): Unit = {
|
||
|
|
subscribers.put(in, subscriber)
|
||
|
|
val publisher = publishers(topLevel.upstreams(in))
|
||
|
|
if (publisher ne null) attach(publisher, subscriber)
|
||
|
|
}
|
||
|
|
|
||
|
|
final protected def assignPort(out: OutPort, publisher: Publisher[Any]): Unit = {
|
||
|
|
publishers.put(out, publisher)
|
||
|
|
val subscriber = subscribers(topLevel.downstreams(out))
|
||
|
|
if (subscriber ne null) attach(publisher, subscriber)
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|