=rem #16623: Fix concurrent reliable delivery actors

(cherry picked from commit 4f7c90660226b05edc3581454d18fd53a4762802)

Conflicts:
	project/AkkaBuild.scala

akka-cluster/src/multi-jvm/scala/akka/cluster/AttemptSysMsgRedeliverySpec.scala

@@ -0,0 +1,90 @@
+/**
+ * Copyright (C) 2009-2014 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import scala.concurrent.duration._
+import akka.actor.Actor
+import akka.actor.ActorIdentity
+import akka.actor.ActorRef
+import akka.actor.Identify
+import akka.actor.Props
+import akka.remote.transport.ThrottlerTransportAdapter.Direction
+import akka.testkit._
+import akka.remote.testkit.{ STMultiNodeSpec, MultiNodeConfig, MultiNodeSpec }
+import akka.actor.PoisonPill
+
+object AttemptSysMsgRedeliveryMultiJvmSpec extends MultiNodeConfig {
+
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+
+  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+  testTransport(on = true)
+
+  class Echo extends Actor {
+    def receive = {
+      case m ⇒ sender ! m
+    }
+  }
+}
+
+class AttemptSysMsgRedeliveryMultiJvmNode1 extends AttemptSysMsgRedeliverySpec
+class AttemptSysMsgRedeliveryMultiJvmNode2 extends AttemptSysMsgRedeliverySpec
+class AttemptSysMsgRedeliveryMultiJvmNode3 extends AttemptSysMsgRedeliverySpec
+
+class AttemptSysMsgRedeliverySpec extends MultiNodeSpec(AttemptSysMsgRedeliveryMultiJvmSpec)
+  with MultiNodeClusterSpec with ImplicitSender with DefaultTimeout {
+  import AttemptSysMsgRedeliveryMultiJvmSpec._
+
+  "AttemptSysMsgRedelivery" must {
+    "reach initial convergence" taggedAs LongRunningTest in {
+      awaitClusterUp(first, second, third)
+
+      enterBarrier("after-1")
+    }
+
+    "redeliver system message after inactivity" taggedAs LongRunningTest in {
+      system.actorOf(Props[Echo], "echo")
+      enterBarrier("echo-started")
+
+      system.actorSelection(node(first) / "user" / "echo") ! Identify(None)
+      val firstRef: ActorRef = expectMsgType[ActorIdentity].ref.get
+      system.actorSelection(node(second) / "user" / "echo") ! Identify(None)
+      val secondRef: ActorRef = expectMsgType[ActorIdentity].ref.get
+      enterBarrier("refs-retrieved")
+
+      runOn(first) {
+        testConductor.blackhole(first, second, Direction.Both).await
+      }
+      enterBarrier("blackhole")
+
+      runOn(first, third) {
+        watch(secondRef)
+      }
+      runOn(second) {
+        watch(firstRef)
+      }
+      enterBarrier("watch-established")
+
+      runOn(first) {
+        testConductor.passThrough(first, second, Direction.Both).await
+      }
+      enterBarrier("pass-through")
+
+      system.actorSelection("/user/echo") ! PoisonPill
+
+      runOn(first, third) {
+        expectTerminated(secondRef, 10.seconds)
+      }
+      runOn(second) {
+        expectTerminated(firstRef, 10.seconds)
+      }
+
+      enterBarrier("done")
+    }
+  }
+
+}