support for parallel lanes, #21207

* for parallel serialziation/deserialization
* MergeHub for the outbound lanes
* BroadcastHub + filter for the inbound lanes, until we
  have a PartitionHub
* simplify materialization of test stage
* add RemoteSendConsistencyWithThreeLanesSpec

akka-remote-tests/src/multi-jvm/scala/akka/remote/artery/MaxThroughputSpec.scala

@@ -31,6 +31,7 @@ object MaxThroughputSpec extends MultiNodeConfig {
     ConfigFactory.parseString(s"""
        # for serious measurements you should increase the totalMessagesFactor (20)
        akka.test.MaxThroughputSpec.totalMessagesFactor = 1.0
+       akka.test.MaxThroughputSpec.real-message = off
        akka {
          loglevel = INFO
          log-dead-letters = 1000000
@@ -44,9 +45,11 @@ object MaxThroughputSpec extends MultiNodeConfig {
 
            serializers {
              test = "akka.remote.artery.MaxThroughputSpec$$TestSerializer"
+             test-message = "akka.remote.artery.TestMessageSerializer"
            }
            serialization-bindings {
              "akka.remote.artery.MaxThroughputSpec$$FlowControl" = test
+             "akka.remote.artery.TestMessage" = test-message
            }
          }
          remote.artery {
@@ -85,6 +88,9 @@ object MaxThroughputSpec extends MultiNodeConfig {
         if (msg.length != payloadSize) throw new IllegalArgumentException("Invalid message")
         reporter.onMessage(1, payloadSize)
         c += 1
+      case msg: TestMessage ⇒
+        reporter.onMessage(1, payloadSize)
+        c += 1
       case Start ⇒
         c = 0
         sender() ! Start
@@ -194,8 +200,19 @@ object MaxThroughputSpec extends MultiNodeConfig {
       val batchSize = math.min(remaining, burstSize)
       var i = 0
       while (i < batchSize) {
-        //        target ! payload
-        target.tell(payload, ActorRef.noSender)
+        val msg =
+          if (realMessage)
+            TestMessage(
+              id = totalMessages - remaining + i,
+              name = "abc",
+              status = i % 2 == 0,
+              description = "ABC",
+              payload = payload,
+              items = Vector(TestMessage.Item(1, "A"), TestMessage.Item(2, "B")))
+          else payload
+
+        //        target ! msg
+        target.tell(msg, ActorRef.noSender)
         i += 1
       }
       remaining -= batchSize
@@ -214,7 +231,8 @@ object MaxThroughputSpec extends MultiNodeConfig {
     totalMessages:       Long,
     burstSize:           Int,
     payloadSize:         Int,
-    senderReceiverPairs: Int) {
+    senderReceiverPairs: Int,
+    realMessage:         Boolean) {
     // data based on measurement
     def totalSize(system: ActorSystem) = payloadSize + (if (RARP(system).provider.remoteSettings.Artery.Advanced.Compression.Enabled) 38 else 110)
   }
@@ -267,6 +285,7 @@ abstract class MaxThroughputSpec
   import MaxThroughputSpec._
 
   val totalMessagesFactor = system.settings.config.getDouble("akka.test.MaxThroughputSpec.totalMessagesFactor")
+  val realMessage = system.settings.config.getBoolean("akka.test.MaxThroughputSpec.real-message")
 
   var plot = PlotResult()
 
@@ -302,31 +321,36 @@ abstract class MaxThroughputSpec
       totalMessages = adjustedTotalMessages(20000),
       burstSize = 1000,
       payloadSize = 100,
-      senderReceiverPairs = 1),
+      senderReceiverPairs = 1,
+      realMessage),
     TestSettings(
       testName = "1-to-1",
       totalMessages = adjustedTotalMessages(50000),
       burstSize = 1000,
       payloadSize = 100,
-      senderReceiverPairs = 1),
+      senderReceiverPairs = 1,
+      realMessage),
     TestSettings(
       testName = "1-to-1-size-1k",
       totalMessages = adjustedTotalMessages(20000),
       burstSize = 1000,
       payloadSize = 1000,
-      senderReceiverPairs = 1),
+      senderReceiverPairs = 1,
+      realMessage),
     TestSettings(
       testName = "1-to-1-size-10k",
       totalMessages = adjustedTotalMessages(10000),
       burstSize = 1000,
       payloadSize = 10000,
-      senderReceiverPairs = 1),
+      senderReceiverPairs = 1,
+      realMessage),
     TestSettings(
       testName = "5-to-5",
       totalMessages = adjustedTotalMessages(20000),
       burstSize = 200, // don't exceed the send queue capacity 200*5*3=3000
       payloadSize = 100,
-      senderReceiverPairs = 5))
+      senderReceiverPairs = 5,
+      realMessage))
 
   def test(testSettings: TestSettings): Unit = {
     import testSettings._