!str #16902: Unify stream internal representation

also =str #16912: Fix StreamTcpSpec flakiness
This commit is contained in:
Endre Sándor Varga 2015-01-28 14:19:50 +01:00
parent cac9c9f2fb
commit 8d77fa8b29
230 changed files with 7814 additions and 9596 deletions

View file

@ -41,109 +41,105 @@ class FlowGraphCompileSpec extends AkkaSpec {
val out1 = Sink.publisher[String]
val out2 = Sink.head[String]
"FlowGraph" should {
"A Graph" should {
"build simple merge" in {
FlowGraph { b
val merge = Merge[String]
b.
addEdge(in1, f1, merge).
addEdge(in2, f2, merge).
addEdge(merge, f3, out1)
FlowGraph.closed() { b
val merge = b.add(Merge[String](2))
b.addEdge(b.add(in1), f1, merge.in(0))
b.addEdge(b.add(in2), f2, merge.in(1))
b.addEdge(merge.out, f3, b.add(out1))
}.run()
}
"build simple broadcast" in {
FlowGraph { b
val bcast = Broadcast[String]
b.
addEdge(in1, f1, bcast).
addEdge(bcast, f2, out1).
addEdge(bcast, f3, out2)
FlowGraph.closed() { b
val bcast = b.add(Broadcast[String](2))
b.addEdge(b.add(in1), f1, bcast.in)
b.addEdge(bcast.out(0), f2, b.add(out1))
b.addEdge(bcast.out(1), f3, b.add(out2))
}.run()
}
"build simple balance" in {
FlowGraph { b
val balance = Balance[String]
b.
addEdge(in1, f1, balance).
addEdge(balance, f2, out1).
addEdge(balance, f3, out2)
FlowGraph.closed() { b
val balance = b.add(Balance[String](2))
b.addEdge(b.add(in1), f1, balance.in)
b.addEdge(balance.out(0), f2, b.add(out1))
b.addEdge(balance.out(1), f3, b.add(out2))
}
}
"build simple merge - broadcast" in {
FlowGraph { b
val merge = Merge[String]
val bcast = Broadcast[String]
b.
addEdge(in1, f1, merge).
addEdge(in2, f2, merge).
addEdge(merge, f3, bcast).
addEdge(bcast, f4, out1).
addEdge(bcast, f5, out2)
FlowGraph.closed() { b
val merge = b.add(Merge[String](2))
val bcast = b.add(Broadcast[String](2))
b.addEdge(b.add(in1), f1, merge.in(0))
b.addEdge(b.add(in2), f2, merge.in(1))
b.addEdge(merge.out, f3, bcast.in)
b.addEdge(bcast.out(0), f4, b.add(out1))
b.addEdge(bcast.out(1), f5, b.add(out2))
}.run()
}
"build simple merge - broadcast with implicits" in {
FlowGraph { implicit b
import FlowGraphImplicits._
val merge = Merge[String]
val bcast = Broadcast[String]
in1 ~> f1 ~> merge ~> f2 ~> bcast ~> f3 ~> out1
in2 ~> f4 ~> merge
bcast ~> f5 ~> out2
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
val merge = b.add(Merge[String](2))
val bcast = b.add(Broadcast[String](2))
b.add(in1) ~> f1 ~> merge.in(0)
merge.out ~> f2 ~> bcast.in
bcast.out(0) ~> f3 ~> b.add(out1)
b.add(in2) ~> f4 ~> merge.in(1)
bcast.out(1) ~> f5 ~> b.add(out2)
}.run()
}
/**
* in ---> f1 -+-> f2 -+-> f3 ---> out1
* in ---> f1 -+-> f2 -+-> f3 ---> b.add(out1)
* ^ |
* | V
* f5 <-+- f4
* |
* V
* f6 ---> out2
* f6 ---> b.add(out2)
*/
"detect cycle in " in {
pending // FIXME needs cycle detection capability
intercept[IllegalArgumentException] {
FlowGraph { b
val merge = Merge[String]
val bcast1 = Broadcast[String]
val bcast2 = Broadcast[String]
FlowGraph.closed() { b
val merge = b.add(Merge[String](2))
val bcast1 = b.add(Broadcast[String](2))
val bcast2 = b.add(Broadcast[String](2))
val feedbackLoopBuffer = Flow[String].buffer(10, OverflowStrategy.dropBuffer)
b.
addEdge(in1, f1, merge).
addEdge(merge, f2, bcast1).
addEdge(bcast1, f3, out1).
addEdge(bcast1, feedbackLoopBuffer, bcast2).
addEdge(bcast2, f5, merge). // cycle
addEdge(bcast2, f6, out2)
b.addEdge(b.add(in1), f1, merge.in(0))
b.addEdge(merge.out, f2, bcast1.in)
b.addEdge(bcast1.out(0), f3, b.add(out1))
b.addEdge(bcast1.out(1), feedbackLoopBuffer, bcast2.in)
b.addEdge(bcast2.out(0), f5, merge.in(1)) // cycle
b.addEdge(bcast2.out(1), f6, b.add(out2))
}
}.getMessage.toLowerCase should include("cycle")
}
"express complex topologies in a readable way" in {
FlowGraph { implicit b
b.allowCycles()
val merge = Merge[String]
val bcast1 = Broadcast[String]
val bcast2 = Broadcast[String]
FlowGraph.closed() { implicit b
val merge = b.add(Merge[String](2))
val bcast1 = b.add(Broadcast[String](2))
val bcast2 = b.add(Broadcast[String](2))
val feedbackLoopBuffer = Flow[String].buffer(10, OverflowStrategy.dropBuffer)
import FlowGraphImplicits._
in1 ~> f1 ~> merge ~> f2 ~> bcast1 ~> f3 ~> out1
import FlowGraph.Implicits._
b.add(in1) ~> f1 ~> merge ~> f2 ~> bcast1 ~> f3 ~> b.add(out1)
bcast1 ~> feedbackLoopBuffer ~> bcast2 ~> f5 ~> merge
bcast2 ~> f6 ~> out2
bcast2 ~> f6 ~> b.add(out2)
}.run()
}
"build broadcast - merge" in {
FlowGraph { implicit b
val bcast = Broadcast[String]
val bcast2 = Broadcast[String]
val merge = Merge[String]
import FlowGraphImplicits._
FlowGraph.closed() { implicit b
val bcast = b.add(Broadcast[String](2))
val merge = b.add(Merge[String](2))
import FlowGraph.Implicits._
in1 ~> f1 ~> bcast ~> f2 ~> merge ~> f3 ~> out1
bcast ~> f4 ~> merge
}.run()
@ -151,14 +147,14 @@ class FlowGraphCompileSpec extends AkkaSpec {
"build wikipedia Topological_sorting" in {
// see https://en.wikipedia.org/wiki/Topological_sorting#mediaviewer/File:Directed_acyclic_graph.png
FlowGraph { implicit b
val b3 = Broadcast[String]
val b7 = Broadcast[String]
val b11 = Broadcast[String]
val m8 = Merge[String]
val m9 = Merge[String]
val m10 = Merge[String]
val m11 = Merge[String]
FlowGraph.closed() { implicit b
val b3 = b.add(Broadcast[String](2))
val b7 = b.add(Broadcast[String](2))
val b11 = b.add(Broadcast[String](3))
val m8 = b.add(Merge[String](2))
val m9 = b.add(Merge[String](2))
val m10 = b.add(Merge[String](2))
val m11 = b.add(Merge[String](2))
val in3 = Source(List("b"))
val in5 = Source(List("b"))
val in7 = Source(List("a"))
@ -166,7 +162,7 @@ class FlowGraphCompileSpec extends AkkaSpec {
val out9 = Sink.publisher[String]
val out10 = Sink.publisher[String]
def f(s: String) = Flow[String].section(name(s))(_.transform(op[String, String]))
import FlowGraphImplicits._
import FlowGraph.Implicits._
in7 ~> f("a") ~> b7 ~> f("b") ~> m11 ~> f("c") ~> b11 ~> f("d") ~> out2
b11 ~> f("e") ~> m9 ~> f("f") ~> out9
@ -178,109 +174,35 @@ class FlowGraphCompileSpec extends AkkaSpec {
}.run()
}
"attachSource and attachSink" in {
val mg = FlowGraph { b
val merge = Merge[String]
val undefinedSource1 = UndefinedSource[String]
val undefinedSource2 = UndefinedSource[String]
val undefinedSink1 = UndefinedSink[String]
b.
addEdge(undefinedSource1, f1, merge).
addEdge(undefinedSource2, f2, merge).
addEdge(merge, f3, undefinedSink1)
b.attachSource(undefinedSource1, in1)
b.attachSource(undefinedSource2, in2)
b.attachSink(undefinedSink1, out1)
}.run()
mg.get(out1) should not be (null)
}
"build partial flow graphs" in {
val undefinedSource1 = UndefinedSource[String]
val undefinedSource2 = UndefinedSource[String]
val undefinedSink1 = UndefinedSink[String]
val bcast = Broadcast[String]
val partial1 = PartialFlowGraph { implicit b
import FlowGraphImplicits._
val merge = Merge[String]
undefinedSource1 ~> f1 ~> merge ~> f2 ~> bcast ~> f3 ~> undefinedSink1
undefinedSource2 ~> f4 ~> merge
}
partial1.undefinedSources should be(Set(undefinedSource1, undefinedSource2))
partial1.undefinedSinks should be(Set(undefinedSink1))
val undefinedSink2 = UndefinedSink[String]
val partial2 = PartialFlowGraph(partial1) { implicit b
import FlowGraphImplicits._
b.attachSource(undefinedSource1, in1)
b.attachSource(undefinedSource2, in2)
bcast ~> f5 ~> undefinedSink2
}
partial2.undefinedSources should be(Set.empty)
partial2.undefinedSinks should be(Set(undefinedSink1, undefinedSink2))
FlowGraph(partial2) { b
b.attachSink(undefinedSink1, out1)
b.attachSink(undefinedSink2, out2)
}.run()
FlowGraph(partial2) { b
b.attachSink(undefinedSink1, f1.to(out1))
b.attachSink(undefinedSink2, f2.to(out2))
}.run()
FlowGraph(partial1) { implicit b
import FlowGraphImplicits._
b.attachSink(undefinedSink1, f1.to(out1))
b.attachSource(undefinedSource1, Source(List("a", "b", "c")).via(f1))
b.attachSource(undefinedSource2, Source(List("d", "e", "f")).via(f2))
bcast ~> f5 ~> out2
}.run()
}
"make it optional to specify flows" in {
FlowGraph { implicit b
val merge = Merge[String]
val bcast = Broadcast[String]
import FlowGraphImplicits._
FlowGraph.closed() { implicit b
val merge = b.add(Merge[String](2))
val bcast = b.add(Broadcast[String](2))
import FlowGraph.Implicits._
in1 ~> merge ~> bcast ~> out1
in2 ~> merge
bcast ~> out2
}.run()
}
"chain input and output ports" in {
FlowGraph { implicit b
val zip = Zip[Int, String]
val out = Sink.publisher[(Int, String)]
import FlowGraphImplicits._
Source(List(1, 2, 3)) ~> zip.left ~> out
Source(List("a", "b", "c")) ~> zip.right
}.run()
}
"build unzip - zip" in {
FlowGraph { implicit b
val zip = Zip[Int, String]
val unzip = Unzip[Int, String]
FlowGraph.closed() { implicit b
val zip = b.add(Zip[Int, String]())
val unzip = b.add(Unzip[Int, String]())
val out = Sink.publisher[(Int, String)]
import FlowGraphImplicits._
import FlowGraph.Implicits._
Source(List(1 -> "a", 2 -> "b", 3 -> "c")) ~> unzip.in
unzip.left ~> Flow[Int].map(_ * 2) ~> zip.left
unzip.right ~> zip.right
unzip.out0 ~> Flow[Int].map(_ * 2) ~> zip.in0
unzip.out1 ~> zip.in1
zip.out ~> out
}.run()
}
"distinguish between input and output ports" in {
intercept[IllegalArgumentException] {
FlowGraph { implicit b
val zip = Zip[Int, String]
val unzip = Unzip[Int, String]
FlowGraph.closed() { implicit b
val zip = b.add(Zip[Int, String]())
val unzip = b.add(Unzip[Int, String]())
val wrongOut = Sink.publisher[(Int, Int)]
val whatever = Sink.publisher[Any]
"Flow(List(1, 2, 3)) ~> zip.left ~> wrongOut" shouldNot compile
@ -290,240 +212,84 @@ class FlowGraphCompileSpec extends AkkaSpec {
"Flow(List(1, 2, 3)) ~> zip.left ~> wrongOut" shouldNot compile
"""Flow(List(1 -> "a", 2 -> "b", 3 -> "c")) ~> unzip.in ~> whatever""" shouldNot compile
}
}.getMessage should include("empty")
}
"check maximumInputCount" in {
intercept[IllegalArgumentException] {
FlowGraph { implicit b
val bcast = Broadcast[String]
import FlowGraphImplicits._
in1 ~> bcast ~> out1
in2 ~> bcast // wrong
}
}.getMessage should include("at most 1 incoming")
}
"check maximumOutputCount" in {
intercept[IllegalArgumentException] {
FlowGraph { implicit b
val merge = Merge[String]
import FlowGraphImplicits._
in1 ~> merge ~> out1
in2 ~> merge
merge ~> out2 // wrong
}
}.getMessage should include("at most 1 outgoing")
}.getMessage should include("unconnected")
}
"build with variance" in {
val out = Sink(SubscriberProbe[Fruit]())
FlowGraph { b
val merge = Merge[Fruit]
b.
addEdge(Source[Fruit](apples), Flow[Fruit], merge).
addEdge(Source[Apple](apples), Flow[Apple], merge).
addEdge(merge, Flow[Fruit].map(identity), out)
FlowGraph.closed() { b
val merge = b.add(Merge[Fruit](2))
b.addEdge(b add Source[Fruit](apples), Flow[Fruit], merge.in(0))
b.addEdge(b add Source[Apple](apples), Flow[Apple], merge.in(1))
b.addEdge(merge.out, Flow[Fruit].map(identity), b add out)
}
}
"build with implicits and variance" in {
PartialFlowGraph { implicit b
val inA = Source(PublisherProbe[Fruit]())
val inB = Source(PublisherProbe[Apple]())
val outA = Sink(SubscriberProbe[Fruit]())
val outB = Sink(SubscriberProbe[Fruit]())
val merge = Merge[Fruit]
val unzip = Unzip[Int, String]
val whatever = Sink.publisher[Any]
import FlowGraphImplicits._
Source[Fruit](apples) ~> merge
Source[Apple](apples) ~> merge
inA ~> merge
inB ~> merge
inA ~> Flow[Fruit].map(identity) ~> merge
inB ~> Flow[Apple].map(identity) ~> merge
UndefinedSource[Apple] ~> merge
UndefinedSource[Apple] ~> Flow[Fruit].map(identity) ~> merge
UndefinedSource[Apple] ~> Flow[Apple].map(identity) ~> merge
merge ~> Flow[Fruit].map(identity) ~> outA
FlowGraph.closed() { implicit b
def appleSource = b.add(Source(PublisherProbe[Apple]))
def fruitSource = b.add(Source(PublisherProbe[Fruit]))
val outA = b add Sink(SubscriberProbe[Fruit]())
val outB = b add Sink(SubscriberProbe[Fruit]())
val merge = b add Merge[Fruit](11)
val unzip = b add Unzip[Int, String]()
val whatever = b add Sink.publisher[Any]
import FlowGraph.Implicits._
b.add(Source[Fruit](apples)) ~> merge.in(0)
appleSource ~> merge.in(1)
appleSource ~> merge.in(2)
fruitSource ~> merge.in(3)
fruitSource ~> Flow[Fruit].map(identity) ~> merge.in(4)
appleSource ~> Flow[Apple].map(identity) ~> merge.in(5)
b.add(Source(apples)) ~> merge.in(6)
b.add(Source(apples)) ~> Flow[Fruit].map(identity) ~> merge.in(7)
b.add(Source(apples)) ~> Flow[Apple].map(identity) ~> merge.in(8)
merge.out ~> Flow[Fruit].map(identity) ~> outA
Source[Apple](apples) ~> Broadcast[Apple] ~> merge
Source[Apple](apples) ~> Broadcast[Apple] ~> outB
Source[Apple](apples) ~> Broadcast[Apple] ~> UndefinedSink[Fruit]
inB ~> Broadcast[Apple] ~> merge
b.add(Source(apples)) ~> Flow[Apple] ~> merge.in(9)
b.add(Source(apples)) ~> Flow[Apple] ~> outB
b.add(Source(apples)) ~> Flow[Apple] ~> b.add(Sink.publisher[Fruit])
appleSource ~> Flow[Apple] ~> merge.in(10)
Source(List(1 -> "a", 2 -> "b", 3 -> "c")) ~> unzip.in
unzip.right ~> whatever
unzip.left ~> UndefinedSink[Any]
unzip.out1 ~> whatever
unzip.out0 ~> b.add(Sink.publisher[Any])
"UndefinedSource[Fruit] ~> Flow[Apple].map(identity) ~> merge" shouldNot compile
"UndefinedSource[Fruit] ~> Broadcast[Apple]" shouldNot compile
"merge ~> Broadcast[Apple]" shouldNot compile
"merge ~> Flow[Fruit].map(identity) ~> Broadcast[Apple]" shouldNot compile
"inB ~> merge ~> Broadcast[Apple]" shouldNot compile
"inA ~> Broadcast[Apple]" shouldNot compile
"merge.out ~> b.add(Broadcast[Apple](2))" shouldNot compile
"merge.out ~> Flow[Fruit].map(identity) ~> b.add(Broadcast[Apple](2))" shouldNot compile
"fruitSource ~> merge ~> b.add(Broadcast[Apple](2))" shouldNot compile
}
}
"build with plain flow without junctions" in {
FlowGraph { b
b.addEdge(in1, f1, out1)
FlowGraph.closed() { b
b.addEdge(b.add(in1), f1, b.add(out1))
}.run()
FlowGraph { b
b.addEdge(in1, f1, f2.to(out1))
FlowGraph.closed() { b
b.addEdge(b.add(in1), f1, b.add(f2.to(out1)))
}.run()
FlowGraph { b
b.addEdge(in1.via(f1), f2, out1)
FlowGraph.closed() { b
b.addEdge(b.add(in1 via f1), f2, b.add(out1))
}.run()
FlowGraph { implicit b
import FlowGraphImplicits._
in1 ~> f1 ~> out1
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
b.add(in1) ~> f1 ~> b.add(out1)
}.run()
FlowGraph { implicit b
import FlowGraphImplicits._
in1 ~> out1
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
b.add(in1) ~> b.add(out1)
}.run()
FlowGraph { implicit b
import FlowGraphImplicits._
in1 ~> f1.to(out1)
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
b.add(in1) ~> b.add(f1 to out1)
}.run()
FlowGraph { implicit b
import FlowGraphImplicits._
in1.via(f1) ~> out1
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
b.add(in1 via f1) ~> b.add(out1)
}.run()
FlowGraph { implicit b
import FlowGraphImplicits._
in1.via(f1) ~> f2.to(out1)
}.run()
}
"build all combinations with implicits" when {
"Source is connected directly" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
Source.empty[Int] ~> Flow[Int]
Source.empty[Int] ~> Broadcast[Int]
Source.empty[Int] ~> Sink.ignore
Source.empty[Int] ~> UndefinedSink[Int]
}
}
"Source is connected through flow" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
Source.empty[Int] ~> Flow[Int] ~> Flow[Int]
Source.empty[Int] ~> Flow[Int] ~> Broadcast[Int]
Source.empty[Int] ~> Flow[Int] ~> Sink.ignore
Source.empty[Int] ~> Flow[Int] ~> UndefinedSink[Int]
}
}
"Junction is connected directly" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
Broadcast[Int] ~> Flow[Int]
Broadcast[Int] ~> Broadcast[Int]
Broadcast[Int] ~> Sink.ignore
Broadcast[Int] ~> UndefinedSink[Int]
}
}
"Junction is connected through flow" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
Broadcast[Int] ~> Flow[Int] ~> Flow[Int]
Broadcast[Int] ~> Flow[Int] ~> Broadcast[Int]
Broadcast[Int] ~> Flow[Int] ~> Sink.ignore
Broadcast[Int] ~> Flow[Int] ~> UndefinedSink[Int]
}
}
"Junction is connected through GraphBackedFlow" in {
val gflow = Flow[Int, String]() { implicit builder
import FlowGraphImplicits._
val in = UndefinedSource[Int]
val out = UndefinedSink[String]
in ~> Flow[Int].map(_.toString) ~> out
(in, out)
}
val sink = Sink.fold[Int, Int](0)(_ + _)
val graph = FlowGraph { implicit builder
import FlowGraphImplicits._
val merge = Merge[Int]
Source(List(1, 2, 3)) ~> merge
Source.empty[Int] ~> merge
merge ~> gflow.map(_.toInt) ~> sink
}
graph.run()
}
"UndefinedSource is connected directly" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
UndefinedSource[Int] ~> Flow[Int]
UndefinedSource[Int] ~> Broadcast[Int]
UndefinedSource[Int] ~> Sink.ignore
UndefinedSource[Int] ~> UndefinedSink[Int]
}
}
"UndefinedSource is connected through flow" in {
PartialFlowGraph { implicit b
import FlowGraphImplicits._
UndefinedSource[Int] ~> Flow[Int] ~> Flow[Int]
UndefinedSource[Int] ~> Flow[Int] ~> Broadcast[Int]
UndefinedSource[Int] ~> Flow[Int] ~> Sink.ignore
UndefinedSource[Int] ~> Flow[Int] ~> UndefinedSink[Int]
}
}
}
"build partial with only undefined sources and sinks" in {
PartialFlowGraph { b
b.addEdge(UndefinedSource[String], f1, UndefinedSink[String])
}
PartialFlowGraph { b
b.addEdge(UndefinedSource[String], f1, out1)
}
PartialFlowGraph { b
b.addEdge(in1, f1, UndefinedSink[String])
}
}
"support interconnect between two partial flow graphs" in {
val output1 = UndefinedSink[String]
val output2 = UndefinedSink[String]
val partial1 = PartialFlowGraph { implicit b
import FlowGraphImplicits._
val bcast = Broadcast[String]
in1 ~> bcast ~> output1
bcast ~> output2
}
val input1 = UndefinedSource[String]
val input2 = UndefinedSource[String]
val partial2 = PartialFlowGraph { implicit b
import FlowGraphImplicits._
val merge = Merge[String]
input1 ~> merge ~> out1
input2 ~> merge
}
FlowGraph { b
b.importPartialFlowGraph(partial1)
b.importPartialFlowGraph(partial2)
b.connect(output1, f1, input1)
b.connect(output2, f2, input2)
FlowGraph.closed() { implicit b
import FlowGraph.Implicits._
b.add(in1 via f1) ~> b.add(f2 to out1)
}.run()
}