Merge branch 'master' into wip-2214-heartbeats-patriknw

Conflicts:
	akka-cluster/src/main/scala/akka/cluster/AccrualFailureDetector.scala
	akka-cluster/src/main/scala/akka/cluster/Cluster.scala

akka-cluster/src/multi-jvm/scala/akka/cluster/ClientDowningNodeThatIsUnreachableSpec.scala

@@ -18,12 +18,17 @@ object ClientDowningNodeThatIsUnreachableMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class ClientDowningNodeThatIsUnreachableMultiJvmNode1 extends ClientDowningNodeThatIsUnreachableSpec
-class ClientDowningNodeThatIsUnreachableMultiJvmNode2 extends ClientDowningNodeThatIsUnreachableSpec
-class ClientDowningNodeThatIsUnreachableMultiJvmNode3 extends ClientDowningNodeThatIsUnreachableSpec
-class ClientDowningNodeThatIsUnreachableMultiJvmNode4 extends ClientDowningNodeThatIsUnreachableSpec
+class ClientDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode1 extends ClientDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode2 extends ClientDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode3 extends ClientDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode4 extends ClientDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
 
-class ClientDowningNodeThatIsUnreachableSpec
+class ClientDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode1 extends ClientDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode2 extends ClientDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode3 extends ClientDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode4 extends ClientDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+
+abstract class ClientDowningNodeThatIsUnreachableSpec
   extends MultiNodeSpec(ClientDowningNodeThatIsUnreachableMultiJvmSpec)
   with MultiNodeClusterSpec {
 
@@ -38,6 +43,7 @@ class ClientDowningNodeThatIsUnreachableSpec
       runOn(first) {
         // kill 'third' node
         testConductor.shutdown(third, 0)
+        markNodeAsUnavailable(thirdAddress)
 
         // mark 'third' node as DOWN
         cluster.down(thirdAddress)

akka-cluster/src/multi-jvm/scala/akka/cluster/ClientDowningNodeThatIsUpSpec.scala

@@ -18,12 +18,17 @@ object ClientDowningNodeThatIsUpMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class ClientDowningNodeThatIsUpMultiJvmNode1 extends ClientDowningNodeThatIsUpSpec
-class ClientDowningNodeThatIsUpMultiJvmNode2 extends ClientDowningNodeThatIsUpSpec
-class ClientDowningNodeThatIsUpMultiJvmNode3 extends ClientDowningNodeThatIsUpSpec
-class ClientDowningNodeThatIsUpMultiJvmNode4 extends ClientDowningNodeThatIsUpSpec
+class ClientDowningNodeThatIsUpWithFailureDetectorPuppetMultiJvmNode1 extends ClientDowningNodeThatIsUpSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUpWithFailureDetectorPuppetMultiJvmNode2 extends ClientDowningNodeThatIsUpSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUpWithFailureDetectorPuppetMultiJvmNode3 extends ClientDowningNodeThatIsUpSpec with FailureDetectorPuppetStrategy
+class ClientDowningNodeThatIsUpWithFailureDetectorPuppetMultiJvmNode4 extends ClientDowningNodeThatIsUpSpec with FailureDetectorPuppetStrategy
 
-class ClientDowningNodeThatIsUpSpec
+class ClientDowningNodeThatIsUpWithAccrualFailureDetectorMultiJvmNode1 extends ClientDowningNodeThatIsUpSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUpWithAccrualFailureDetectorMultiJvmNode2 extends ClientDowningNodeThatIsUpSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUpWithAccrualFailureDetectorMultiJvmNode3 extends ClientDowningNodeThatIsUpSpec with AccrualFailureDetectorStrategy
+class ClientDowningNodeThatIsUpWithAccrualFailureDetectorMultiJvmNode4 extends ClientDowningNodeThatIsUpSpec with AccrualFailureDetectorStrategy
+
+abstract class ClientDowningNodeThatIsUpSpec
   extends MultiNodeSpec(ClientDowningNodeThatIsUpMultiJvmSpec)
   with MultiNodeClusterSpec {
 
@@ -40,6 +45,8 @@ class ClientDowningNodeThatIsUpSpec
         cluster.down(thirdAddress)
         testConductor.enter("down-third-node")
 
+        markNodeAsUnavailable(thirdAddress)
+
         awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(thirdAddress))
         cluster.latestGossip.members.exists(_.address == thirdAddress) must be(false)
       }

akka-cluster/src/multi-jvm/scala/akka/cluster/ConvergenceSpec.scala

@@ -17,27 +17,29 @@ object ConvergenceMultiJvmSpec extends MultiNodeConfig {
   val fourth = role("fourth")
 
   commonConfig(debugConfig(on = false).
-    withFallback(ConfigFactory.parseString("""
-      akka.cluster {
-        failure-detector.threshold = 4
-      }
-    """)).
+    withFallback(ConfigFactory.parseString("akka.cluster.failure-detector.threshold = 4")).
     withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class ConvergenceMultiJvmNode1 extends ConvergenceSpec
-class ConvergenceMultiJvmNode2 extends ConvergenceSpec
-class ConvergenceMultiJvmNode3 extends ConvergenceSpec
-class ConvergenceMultiJvmNode4 extends ConvergenceSpec
+class ConvergenceWithFailureDetectorPuppetMultiJvmNode1 extends ConvergenceSpec with FailureDetectorPuppetStrategy
+class ConvergenceWithFailureDetectorPuppetMultiJvmNode2 extends ConvergenceSpec with FailureDetectorPuppetStrategy
+class ConvergenceWithFailureDetectorPuppetMultiJvmNode3 extends ConvergenceSpec with FailureDetectorPuppetStrategy
+class ConvergenceWithFailureDetectorPuppetMultiJvmNode4 extends ConvergenceSpec with FailureDetectorPuppetStrategy
+
+class ConvergenceWithAccrualFailureDetectorMultiJvmNode1 extends ConvergenceSpec with AccrualFailureDetectorStrategy
+class ConvergenceWithAccrualFailureDetectorMultiJvmNode2 extends ConvergenceSpec with AccrualFailureDetectorStrategy
+class ConvergenceWithAccrualFailureDetectorMultiJvmNode3 extends ConvergenceSpec with AccrualFailureDetectorStrategy
+class ConvergenceWithAccrualFailureDetectorMultiJvmNode4 extends ConvergenceSpec with AccrualFailureDetectorStrategy
 
 abstract class ConvergenceSpec
   extends MultiNodeSpec(ConvergenceMultiJvmSpec)
   with MultiNodeClusterSpec {
+
   import ConvergenceMultiJvmSpec._
 
   "A cluster of 3 members" must {
 
-    "reach initial convergence" taggedAs LongRunningTest in {
+    "reach initial convergence" taggedAs LongRunningTest ignore {
       awaitClusterUp(first, second, third)
 
       runOn(fourth) {
@@ -47,13 +49,14 @@ abstract class ConvergenceSpec
       testConductor.enter("after-1")
     }
 
-    "not reach convergence while any nodes are unreachable" taggedAs LongRunningTest in {
+    "not reach convergence while any nodes are unreachable" taggedAs LongRunningTest ignore {
       val thirdAddress = node(third).address
       testConductor.enter("before-shutdown")
 
       runOn(first) {
         // kill 'third' node
         testConductor.shutdown(third, 0)
+        markNodeAsUnavailable(thirdAddress)
       }
 
       runOn(first, second) {
@@ -78,7 +81,7 @@ abstract class ConvergenceSpec
       testConductor.enter("after-2")
     }
 
-    "not move a new joining node to Up while there is no convergence" taggedAs LongRunningTest in {
+    "not move a new joining node to Up while there is no convergence" taggedAs LongRunningTest ignore {
       runOn(fourth) {
         // try to join
         cluster.join(node(first).address)

akka-cluster/src/multi-jvm/scala/akka/cluster/FailureDetectorStrategy.scala

@@ -0,0 +1,61 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import akka.actor.Address
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+/**
+ * Base trait for all failure detector strategies.
+ */
+trait FailureDetectorStrategy {
+
+  /**
+   * Get or create the FailureDetector to be used in the cluster node.
+   * To be defined by subclass.
+   */
+  def failureDetector: FailureDetector
+
+  /**
+   * Marks a node as available in the failure detector.
+   * To be defined by subclass.
+   */
+  def markNodeAsAvailable(address: Address): Unit
+
+  /**
+   * Marks a node as unavailable in the failure detector.
+   * To be defined by subclass.
+   */
+  def markNodeAsUnavailable(address: Address): Unit
+}
+
+/**
+ * Defines a FailureDetectorPuppet-based FailureDetectorStrategy.
+ */
+trait FailureDetectorPuppetStrategy extends FailureDetectorStrategy { self: MultiNodeSpec ⇒
+
+  /**
+   * The puppet instance. Separated from 'failureDetector' field so we don't have to cast when using the puppet specific methods.
+   */
+  private val puppet = new FailureDetectorPuppet(system)
+
+  override def failureDetector: FailureDetector = puppet
+
+  override def markNodeAsAvailable(address: Address): Unit = puppet markNodeAsAvailable address
+
+  override def markNodeAsUnavailable(address: Address): Unit = puppet markNodeAsUnavailable address
+}
+
+/**
+ * Defines a AccrualFailureDetector-based FailureDetectorStrategy.
+ */
+trait AccrualFailureDetectorStrategy extends FailureDetectorStrategy { self: MultiNodeSpec ⇒
+
+  override val failureDetector: FailureDetector = new AccrualFailureDetector(system, new ClusterSettings(system.settings.config, system.name))
+
+  override def markNodeAsAvailable(address: Address): Unit = { /* no-op */ }
+
+  override def markNodeAsUnavailable(address: Address): Unit = { /* no-op */ }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/GossipingAccrualFailureDetectorSpec.scala

@@ -19,12 +19,14 @@ object GossipingAccrualFailureDetectorMultiJvmSpec extends MultiNodeConfig {
     withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class GossipingAccrualFailureDetectorMultiJvmNode1 extends GossipingAccrualFailureDetectorSpec
-class GossipingAccrualFailureDetectorMultiJvmNode2 extends GossipingAccrualFailureDetectorSpec
-class GossipingAccrualFailureDetectorMultiJvmNode3 extends GossipingAccrualFailureDetectorSpec
+class GossipingWithAccrualFailureDetectorMultiJvmNode1 extends GossipingAccrualFailureDetectorSpec with AccrualFailureDetectorStrategy
+class GossipingWithAccrualFailureDetectorMultiJvmNode2 extends GossipingAccrualFailureDetectorSpec with AccrualFailureDetectorStrategy
+class GossipingWithAccrualFailureDetectorMultiJvmNode3 extends GossipingAccrualFailureDetectorSpec with AccrualFailureDetectorStrategy
 
-abstract class GossipingAccrualFailureDetectorSpec extends MultiNodeSpec(GossipingAccrualFailureDetectorMultiJvmSpec)
+abstract class GossipingAccrualFailureDetectorSpec
+  extends MultiNodeSpec(GossipingAccrualFailureDetectorMultiJvmSpec)
   with MultiNodeClusterSpec {
+
   import GossipingAccrualFailureDetectorMultiJvmSpec._
 
   lazy val firstAddress = node(first).address

akka-cluster/src/multi-jvm/scala/akka/cluster/JoinTwoClustersSpec.scala

@@ -20,12 +20,12 @@ object JoinTwoClustersMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = true).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec
-class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec
-class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec
-class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec
-class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec
-class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
+class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
+class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
+class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
+class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
+class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec with FailureDetectorPuppetStrategy
 
 abstract class JoinTwoClustersSpec
   extends MultiNodeSpec(JoinTwoClustersMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderDowningNodeThatIsUnreachableSpec.scala

@@ -7,7 +7,7 @@ import com.typesafe.config.ConfigFactory
 import akka.remote.testkit.MultiNodeConfig
 import akka.remote.testkit.MultiNodeSpec
 import akka.testkit._
-import akka.actor.Address
+import akka.actor._
 import akka.util.duration._
 
 object LeaderDowningNodeThatIsUnreachableMultiJvmSpec extends MultiNodeConfig {
@@ -16,22 +16,22 @@ object LeaderDowningNodeThatIsUnreachableMultiJvmSpec extends MultiNodeConfig {
   val third = role("third")
   val fourth = role("fourth")
 
-  commonConfig(debugConfig(on = true).
-    withFallback(ConfigFactory.parseString("""
-      akka.cluster {
-        auto-down = on
-        failure-detector.threshold = 4
-      }
-    """)).
+  commonConfig(debugConfig(on = false).
+    withFallback(ConfigFactory.parseString("akka.cluster.auto-down = on")).
     withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class LeaderDowningNodeThatIsUnreachableMultiJvmNode1 extends LeaderDowningNodeThatIsUnreachableSpec
-class LeaderDowningNodeThatIsUnreachableMultiJvmNode2 extends LeaderDowningNodeThatIsUnreachableSpec
-class LeaderDowningNodeThatIsUnreachableMultiJvmNode3 extends LeaderDowningNodeThatIsUnreachableSpec
-class LeaderDowningNodeThatIsUnreachableMultiJvmNode4 extends LeaderDowningNodeThatIsUnreachableSpec
+class LeaderDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode1 extends LeaderDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class LeaderDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode2 extends LeaderDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class LeaderDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode3 extends LeaderDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
+class LeaderDowningNodeThatIsUnreachableWithFailureDetectorPuppetMultiJvmNode4 extends LeaderDowningNodeThatIsUnreachableSpec with FailureDetectorPuppetStrategy
 
-class LeaderDowningNodeThatIsUnreachableSpec
+class LeaderDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode1 extends LeaderDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class LeaderDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode2 extends LeaderDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class LeaderDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode3 extends LeaderDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+class LeaderDowningNodeThatIsUnreachableWithAccrualFailureDetectorMultiJvmNode4 extends LeaderDowningNodeThatIsUnreachableSpec with AccrualFailureDetectorStrategy
+
+abstract class LeaderDowningNodeThatIsUnreachableSpec
   extends MultiNodeSpec(LeaderDowningNodeThatIsUnreachableMultiJvmSpec)
   with MultiNodeClusterSpec {
 
@@ -40,14 +40,17 @@ class LeaderDowningNodeThatIsUnreachableSpec
   "The Leader in a 4 node cluster" must {
 
     "be able to DOWN a 'last' node that is UNREACHABLE" taggedAs LongRunningTest in {
-      val fourthAddress = node(fourth).address
       awaitClusterUp(first, second, third, fourth)
 
+      val fourthAddress = node(fourth).address
       runOn(first) {
         // kill 'fourth' node
         testConductor.shutdown(fourth, 0)
         testConductor.enter("down-fourth-node")
 
+        // mark the node as unreachable in the failure detector
+        markNodeAsUnavailable(fourthAddress)
+
         // --- HERE THE LEADER SHOULD DETECT FAILURE AND AUTO-DOWN THE UNREACHABLE NODE ---
 
         awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(fourthAddress), 30.seconds)
@@ -68,13 +71,16 @@ class LeaderDowningNodeThatIsUnreachableSpec
 
     "be able to DOWN a 'middle' node that is UNREACHABLE" taggedAs LongRunningTest in {
       val secondAddress = node(second).address
-      testConductor.enter("before-down-second-node")
 
+      testConductor.enter("before-down-second-node")
       runOn(first) {
         // kill 'second' node
         testConductor.shutdown(second, 0)
         testConductor.enter("down-second-node")
 
+        // mark the node as unreachable in the failure detector
+        markNodeAsUnavailable(secondAddress)
+
         // --- HERE THE LEADER SHOULD DETECT FAILURE AND AUTO-DOWN THE UNREACHABLE NODE ---
 
         awaitUpConvergence(numberOfMembers = 2, canNotBePartOfMemberRing = Seq(secondAddress), 30.seconds)

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala

@@ -19,11 +19,17 @@ object LeaderElectionMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class LeaderElectionMultiJvmNode1 extends LeaderElectionSpec
-class LeaderElectionMultiJvmNode2 extends LeaderElectionSpec
-class LeaderElectionMultiJvmNode3 extends LeaderElectionSpec
-class LeaderElectionMultiJvmNode4 extends LeaderElectionSpec
-class LeaderElectionMultiJvmNode5 extends LeaderElectionSpec
+class LeaderElectionWithFailureDetectorPuppetMultiJvmNode1 extends LeaderElectionSpec with FailureDetectorPuppetStrategy
+class LeaderElectionWithFailureDetectorPuppetMultiJvmNode2 extends LeaderElectionSpec with FailureDetectorPuppetStrategy
+class LeaderElectionWithFailureDetectorPuppetMultiJvmNode3 extends LeaderElectionSpec with FailureDetectorPuppetStrategy
+class LeaderElectionWithFailureDetectorPuppetMultiJvmNode4 extends LeaderElectionSpec with FailureDetectorPuppetStrategy
+class LeaderElectionWithFailureDetectorPuppetMultiJvmNode5 extends LeaderElectionSpec with FailureDetectorPuppetStrategy
+
+class LeaderElectionWithAccrualFailureDetectorMultiJvmNode1 extends LeaderElectionSpec with AccrualFailureDetectorStrategy
+class LeaderElectionWithAccrualFailureDetectorMultiJvmNode2 extends LeaderElectionSpec with AccrualFailureDetectorStrategy
+class LeaderElectionWithAccrualFailureDetectorMultiJvmNode3 extends LeaderElectionSpec with AccrualFailureDetectorStrategy
+class LeaderElectionWithAccrualFailureDetectorMultiJvmNode4 extends LeaderElectionSpec with AccrualFailureDetectorStrategy
+class LeaderElectionWithAccrualFailureDetectorMultiJvmNode5 extends LeaderElectionSpec with AccrualFailureDetectorStrategy
 
 abstract class LeaderElectionSpec
   extends MultiNodeSpec(LeaderElectionMultiJvmSpec)
@@ -57,9 +63,11 @@ abstract class LeaderElectionSpec
       myself match {
 
         case `controller` ⇒
+          val leaderAddress = node(leader).address
           testConductor.enter("before-shutdown")
           testConductor.shutdown(leader, 0)
           testConductor.enter("after-shutdown", "after-down", "completed")
+          markNodeAsUnavailable(leaderAddress)
 
         case `leader` ⇒
           testConductor.enter("before-shutdown", "after-shutdown")
@@ -71,6 +79,7 @@ abstract class LeaderElectionSpec
           // user marks the shutdown leader as DOWN
           cluster.down(leaderAddress)
           testConductor.enter("after-down", "completed")
+          markNodeAsUnavailable(leaderAddress)
 
         case _ if remainingRoles.contains(myself) ⇒
           // remaining cluster nodes, not shutdown

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerExitingSpec.scala

@@ -27,9 +27,9 @@ object MembershipChangeListenerExitingMultiJvmSpec extends MultiNodeConfig {
         .withFallback(MultiNodeClusterSpec.clusterConfig)))
 }
 
-class MembershipChangeListenerExitingMultiJvmNode1 extends MembershipChangeListenerExitingSpec
-class MembershipChangeListenerExitingMultiJvmNode2 extends MembershipChangeListenerExitingSpec
-class MembershipChangeListenerExitingMultiJvmNode3 extends MembershipChangeListenerExitingSpec
+class MembershipChangeListenerExitingMultiJvmNode1 extends MembershipChangeListenerExitingSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerExitingMultiJvmNode2 extends MembershipChangeListenerExitingSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerExitingMultiJvmNode3 extends MembershipChangeListenerExitingSpec with FailureDetectorPuppetStrategy
 
 abstract class MembershipChangeListenerExitingSpec
   extends MultiNodeSpec(MembershipChangeListenerExitingMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerJoinSpec.scala

@@ -17,16 +17,12 @@ object MembershipChangeListenerJoinMultiJvmSpec extends MultiNodeConfig {
 
   commonConfig(
     debugConfig(on = false)
-      .withFallback(ConfigFactory.parseString("""
-        akka.cluster {
-          leader-actions-interval = 5 s # increase the leader action task interval to allow time checking for JOIN before leader moves it to UP
-        }
-      """)
-        .withFallback(MultiNodeClusterSpec.clusterConfig)))
+      .withFallback(ConfigFactory.parseString("akka.cluster.leader-actions-interval = 5 s") // increase the leader action task interval to allow time checking for JOIN before leader moves it to UP
+      .withFallback(MultiNodeClusterSpec.clusterConfig)))
 }
 
-class MembershipChangeListenerJoinMultiJvmNode1 extends MembershipChangeListenerJoinSpec
-class MembershipChangeListenerJoinMultiJvmNode2 extends MembershipChangeListenerJoinSpec
+class MembershipChangeListenerJoinMultiJvmNode1 extends MembershipChangeListenerJoinSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerJoinMultiJvmNode2 extends MembershipChangeListenerJoinSpec with FailureDetectorPuppetStrategy
 
 abstract class MembershipChangeListenerJoinSpec
   extends MultiNodeSpec(MembershipChangeListenerJoinMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerLeavingSpec.scala

@@ -24,9 +24,9 @@ object MembershipChangeListenerLeavingMultiJvmSpec extends MultiNodeConfig {
       .withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class MembershipChangeListenerLeavingMultiJvmNode1 extends MembershipChangeListenerLeavingSpec
-class MembershipChangeListenerLeavingMultiJvmNode2 extends MembershipChangeListenerLeavingSpec
-class MembershipChangeListenerLeavingMultiJvmNode3 extends MembershipChangeListenerLeavingSpec
+class MembershipChangeListenerLeavingMultiJvmNode1 extends MembershipChangeListenerLeavingSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerLeavingMultiJvmNode2 extends MembershipChangeListenerLeavingSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerLeavingMultiJvmNode3 extends MembershipChangeListenerLeavingSpec with FailureDetectorPuppetStrategy
 
 abstract class MembershipChangeListenerLeavingSpec
   extends MultiNodeSpec(MembershipChangeListenerLeavingMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerSpec.scala

@@ -1,77 +0,0 @@
-/**
- *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
- */
-package akka.cluster
-
-import scala.collection.immutable.SortedSet
-import com.typesafe.config.ConfigFactory
-import akka.remote.testkit.MultiNodeConfig
-import akka.remote.testkit.MultiNodeSpec
-import akka.testkit._
-
-object MembershipChangeListenerMultiJvmSpec extends MultiNodeConfig {
-  val first = role("first")
-  val second = role("second")
-  val third = role("third")
-
-  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
-}
-
-class MembershipChangeListenerMultiJvmNode1 extends MembershipChangeListenerSpec
-class MembershipChangeListenerMultiJvmNode2 extends MembershipChangeListenerSpec
-class MembershipChangeListenerMultiJvmNode3 extends MembershipChangeListenerSpec
-
-abstract class MembershipChangeListenerSpec extends MultiNodeSpec(MembershipChangeListenerMultiJvmSpec)
-  with MultiNodeClusterSpec {
-  import MembershipChangeListenerMultiJvmSpec._
-
-  lazy val firstAddress = node(first).address
-  lazy val secondAddress = node(second).address
-
-  "A set of connected cluster systems" must {
-
-    "(when two nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
-
-      awaitClusterUp(first)
-
-      runOn(first, second) {
-        val latch = TestLatch()
-        cluster.registerListener(new MembershipChangeListener {
-          def notify(members: SortedSet[Member]) {
-            if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
-              latch.countDown()
-          }
-        })
-        testConductor.enter("listener-1-registered")
-        cluster.join(firstAddress)
-        latch.await
-      }
-
-      runOn(third) {
-        testConductor.enter("listener-1-registered")
-      }
-
-      testConductor.enter("after-1")
-    }
-
-    "(when three nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
-
-      val latch = TestLatch()
-      cluster.registerListener(new MembershipChangeListener {
-        def notify(members: SortedSet[Member]) {
-          if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
-            latch.countDown()
-        }
-      })
-      testConductor.enter("listener-2-registered")
-
-      runOn(third) {
-        cluster.join(firstAddress)
-      }
-
-      latch.await
-
-      testConductor.enter("after-2")
-    }
-  }
-}

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerUpSpec.scala

@@ -5,21 +5,21 @@ package akka.cluster
 
 import scala.collection.immutable.SortedSet
 import com.typesafe.config.ConfigFactory
-import org.scalatest.BeforeAndAfter
 import akka.remote.testkit.MultiNodeConfig
 import akka.remote.testkit.MultiNodeSpec
 import akka.testkit._
-import akka.util.duration._
 
 object MembershipChangeListenerUpMultiJvmSpec extends MultiNodeConfig {
   val first = role("first")
   val second = role("second")
+  val third = role("third")
 
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class MembershipChangeListenerUpMultiJvmNode1 extends MembershipChangeListenerUpSpec
-class MembershipChangeListenerUpMultiJvmNode2 extends MembershipChangeListenerUpSpec
+class MembershipChangeListenerUpMultiJvmNode1 extends MembershipChangeListenerUpSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerUpMultiJvmNode2 extends MembershipChangeListenerUpSpec with FailureDetectorPuppetStrategy
+class MembershipChangeListenerUpMultiJvmNode3 extends MembershipChangeListenerUpSpec with FailureDetectorPuppetStrategy
 
 abstract class MembershipChangeListenerUpSpec
   extends MultiNodeSpec(MembershipChangeListenerUpMultiJvmSpec)
@@ -30,29 +30,50 @@ abstract class MembershipChangeListenerUpSpec
   lazy val firstAddress = node(first).address
   lazy val secondAddress = node(second).address
 
-  "A registered MembershipChangeListener" must {
-    "be notified when new node is marked as UP by the leader" taggedAs LongRunningTest in {
+  "A set of connected cluster systems" must {
 
-      runOn(first) {
-        val upLatch = TestLatch()
+    "(when two nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
+
+      awaitClusterUp(first)
+
+      runOn(first, second) {
+        val latch = TestLatch()
         cluster.registerListener(new MembershipChangeListener {
           def notify(members: SortedSet[Member]) {
             if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
-              upLatch.countDown()
+              latch.countDown()
           }
         })
-        testConductor.enter("registered-listener")
-
-        upLatch.await
-        awaitUpConvergence(numberOfMembers = 2)
+        testConductor.enter("listener-1-registered")
+        cluster.join(firstAddress)
+        latch.await
       }
 
-      runOn(second) {
-        testConductor.enter("registered-listener")
+      runOn(third) {
+        testConductor.enter("listener-1-registered")
+      }
+
+      testConductor.enter("after-1")
+    }
+
+    "(when three nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
+
+      val latch = TestLatch()
+      cluster.registerListener(new MembershipChangeListener {
+        def notify(members: SortedSet[Member]) {
+          if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
+            latch.countDown()
+        }
+      })
+      testConductor.enter("listener-2-registered")
+
+      runOn(third) {
         cluster.join(firstAddress)
       }
 
-      testConductor.enter("after")
+      latch.await
+
+      testConductor.enter("after-2")
     }
   }
 }

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

@@ -5,7 +5,7 @@ package akka.cluster
 
 import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
-import akka.actor.Address
+import akka.actor.{Address, ExtendedActorSystem}
 import akka.remote.testconductor.RoleName
 import akka.remote.testkit.MultiNodeSpec
 import akka.testkit._
@@ -29,14 +29,19 @@ object MultiNodeClusterSpec {
     """)
 }
 
-trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
+trait MultiNodeClusterSpec extends FailureDetectorStrategy { self: MultiNodeSpec ⇒
 
   override def initialParticipants = roles.size
 
   /**
-   * Get or create a cluster node using 'Cluster(system)' extension.
+   * The cluster node instance. Needs to be lazily created.
    */
-  def cluster: Cluster = Cluster(system)
+  private lazy val clusterNode = new Cluster(system.asInstanceOf[ExtendedActorSystem], failureDetector)
+
+  /**
+   * Get the cluster node to use.
+   */
+  def cluster: Cluster = clusterNode
 
   /**
    * Use this method instead of 'cluster.self'
@@ -49,9 +54,7 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
    * nodes (roles). First node will be started first
    * and others will join the first.
    */
-  def startCluster(roles: RoleName*): Unit = {
-    awaitStartCluster(false, roles.toSeq)
-  }
+  def startCluster(roles: RoleName*): Unit = awaitStartCluster(false, roles.toSeq)
 
   /**
    * Initialize the cluster of the specified member

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeJoinSpec.scala

@@ -16,16 +16,12 @@ object NodeJoinMultiJvmSpec extends MultiNodeConfig {
 
   commonConfig(
     debugConfig(on = false)
-      .withFallback(ConfigFactory.parseString("""
-        akka.cluster {
-          leader-actions-interval = 5 s # increase the leader action task interval
-        }
-      """)
-        .withFallback(MultiNodeClusterSpec.clusterConfig)))
+      .withFallback(ConfigFactory.parseString("akka.cluster.leader-actions-interval = 5 s") // increase the leader action task interval
+      .withFallback(MultiNodeClusterSpec.clusterConfig)))
 }
 
-class NodeJoinMultiJvmNode1 extends NodeJoinSpec
-class NodeJoinMultiJvmNode2 extends NodeJoinSpec
+class NodeJoinMultiJvmNode1 extends NodeJoinSpec with FailureDetectorPuppetStrategy
+class NodeJoinMultiJvmNode2 extends NodeJoinSpec with FailureDetectorPuppetStrategy
 
 abstract class NodeJoinSpec
   extends MultiNodeSpec(NodeJoinMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingAndBeingRemovedSpec.scala

@@ -18,9 +18,9 @@ object NodeLeavingAndExitingAndBeingRemovedMultiJvmSpec extends MultiNodeConfig
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode1 extends NodeLeavingAndExitingAndBeingRemovedSpec
-class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode2 extends NodeLeavingAndExitingAndBeingRemovedSpec
-class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode3 extends NodeLeavingAndExitingAndBeingRemovedSpec
+class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode1 extends NodeLeavingAndExitingAndBeingRemovedSpec with AccrualFailureDetectorStrategy
+class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode2 extends NodeLeavingAndExitingAndBeingRemovedSpec with AccrualFailureDetectorStrategy
+class NodeLeavingAndExitingAndBeingRemovedMultiJvmNode3 extends NodeLeavingAndExitingAndBeingRemovedSpec with AccrualFailureDetectorStrategy
 
 abstract class NodeLeavingAndExitingAndBeingRemovedSpec
   extends MultiNodeSpec(NodeLeavingAndExitingAndBeingRemovedMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala

@@ -26,9 +26,9 @@ object NodeLeavingAndExitingMultiJvmSpec extends MultiNodeConfig {
         .withFallback(MultiNodeClusterSpec.clusterConfig)))
 }
 
-class NodeLeavingAndExitingMultiJvmNode1 extends NodeLeavingAndExitingSpec
-class NodeLeavingAndExitingMultiJvmNode2 extends NodeLeavingAndExitingSpec
-class NodeLeavingAndExitingMultiJvmNode3 extends NodeLeavingAndExitingSpec
+class NodeLeavingAndExitingMultiJvmNode1 extends NodeLeavingAndExitingSpec with FailureDetectorPuppetStrategy
+class NodeLeavingAndExitingMultiJvmNode2 extends NodeLeavingAndExitingSpec with FailureDetectorPuppetStrategy
+class NodeLeavingAndExitingMultiJvmNode3 extends NodeLeavingAndExitingSpec with FailureDetectorPuppetStrategy
 
 abstract class NodeLeavingAndExitingSpec
   extends MultiNodeSpec(NodeLeavingAndExitingMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingSpec.scala

@@ -16,18 +16,18 @@ object NodeLeavingMultiJvmSpec extends MultiNodeConfig {
 
   commonConfig(
     debugConfig(on = false)
-      .withFallback(ConfigFactory.parseString("""
-        akka.cluster.unreachable-nodes-reaper-frequency = 30 s
-      """))
+      .withFallback(ConfigFactory.parseString("akka.cluster.unreachable-nodes-reaper-frequency = 30 s"))
       .withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class NodeLeavingMultiJvmNode1 extends NodeLeavingSpec
-class NodeLeavingMultiJvmNode2 extends NodeLeavingSpec
-class NodeLeavingMultiJvmNode3 extends NodeLeavingSpec
+class NodeLeavingMultiJvmNode1 extends NodeLeavingSpec with FailureDetectorPuppetStrategy
+class NodeLeavingMultiJvmNode2 extends NodeLeavingSpec with FailureDetectorPuppetStrategy
+class NodeLeavingMultiJvmNode3 extends NodeLeavingSpec with FailureDetectorPuppetStrategy
 
-abstract class NodeLeavingSpec extends MultiNodeSpec(NodeLeavingMultiJvmSpec)
+abstract class NodeLeavingSpec
+  extends MultiNodeSpec(NodeLeavingMultiJvmSpec)
   with MultiNodeClusterSpec {
+
   import NodeLeavingMultiJvmSpec._
 
   lazy val firstAddress = node(first).address

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeMembershipSpec.scala

@@ -16,9 +16,9 @@ object NodeMembershipMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec
-class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec
-class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec
+class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec with FailureDetectorPuppetStrategy
+class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec with FailureDetectorPuppetStrategy
+class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec with FailureDetectorPuppetStrategy
 
 abstract class NodeMembershipSpec
   extends MultiNodeSpec(NodeMembershipMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeUpSpec.scala

@@ -19,8 +19,8 @@ object NodeUpMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
 }
 
-class NodeUpMultiJvmNode1 extends NodeUpSpec
-class NodeUpMultiJvmNode2 extends NodeUpSpec
+class NodeUpMultiJvmNode1 extends NodeUpSpec with FailureDetectorPuppetStrategy
+class NodeUpMultiJvmNode2 extends NodeUpSpec with FailureDetectorPuppetStrategy
 
 abstract class NodeUpSpec
   extends MultiNodeSpec(NodeUpMultiJvmSpec)

akka-cluster/src/multi-jvm/scala/akka/cluster/SingletonClusterSpec.scala

@@ -16,7 +16,7 @@ object SingletonClusterMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).
     withFallback(ConfigFactory.parseString("""
       akka.cluster {
-        auto-down = on
+        auto-down                  = on
         failure-detector.threshold = 4
       }
     """)).
@@ -24,10 +24,16 @@ object SingletonClusterMultiJvmSpec extends MultiNodeConfig {
 
 }
 
-class SingletonClusterMultiJvmNode1 extends SingletonClusterSpec
-class SingletonClusterMultiJvmNode2 extends SingletonClusterSpec
+class SingletonClusterWithFailureDetectorPuppetMultiJvmNode1 extends SingletonClusterSpec with FailureDetectorPuppetStrategy
+class SingletonClusterWithFailureDetectorPuppetMultiJvmNode2 extends SingletonClusterSpec with FailureDetectorPuppetStrategy
+
+class SingletonClusterWithAccrualFailureDetectorMultiJvmNode1 extends SingletonClusterSpec with AccrualFailureDetectorStrategy
+class SingletonClusterWithAccrualFailureDetectorMultiJvmNode2 extends SingletonClusterSpec with AccrualFailureDetectorStrategy
+
+abstract class SingletonClusterSpec
+  extends MultiNodeSpec(SingletonClusterMultiJvmSpec)
+  with MultiNodeClusterSpec {
 
-abstract class SingletonClusterSpec extends MultiNodeSpec(SingletonClusterMultiJvmSpec) with MultiNodeClusterSpec {
   import SingletonClusterMultiJvmSpec._
 
   "A cluster of 2 nodes" must {
@@ -44,6 +50,9 @@ abstract class SingletonClusterSpec extends MultiNodeSpec(SingletonClusterMultiJ
       runOn(first) {
         val secondAddress = node(second).address
         testConductor.shutdown(second, 0)
+
+        markNodeAsUnavailable(secondAddress)
+
         awaitUpConvergence(numberOfMembers = 1, canNotBePartOfMemberRing = Seq(secondAddress), 30.seconds)
         cluster.isSingletonCluster must be(true)
         assertLeader(first)

akka-cluster/src/multi-jvm/scala/akka/cluster/SunnyWeatherSpec.scala

@@ -21,18 +21,18 @@ object SunnyWeatherMultiJvmSpec extends MultiNodeConfig {
 
   commonConfig(ConfigFactory.parseString("""
     akka.cluster {
-      gossip-interval = 400 ms
+      gossip-interval    = 400 ms
       nr-of-deputy-nodes = 0
     }
     akka.loglevel = INFO
     """))
 }
 
-class SunnyWeatherMultiJvmNode1 extends SunnyWeatherSpec
-class SunnyWeatherMultiJvmNode2 extends SunnyWeatherSpec
-class SunnyWeatherMultiJvmNode3 extends SunnyWeatherSpec
-class SunnyWeatherMultiJvmNode4 extends SunnyWeatherSpec
-class SunnyWeatherMultiJvmNode5 extends SunnyWeatherSpec
+class SunnyWeatherMultiJvmNode1 extends SunnyWeatherSpec with FailureDetectorPuppetStrategy
+class SunnyWeatherMultiJvmNode2 extends SunnyWeatherSpec with FailureDetectorPuppetStrategy
+class SunnyWeatherMultiJvmNode3 extends SunnyWeatherSpec with FailureDetectorPuppetStrategy
+class SunnyWeatherMultiJvmNode4 extends SunnyWeatherSpec with FailureDetectorPuppetStrategy
+class SunnyWeatherMultiJvmNode5 extends SunnyWeatherSpec with FailureDetectorPuppetStrategy
 
 abstract class SunnyWeatherSpec
   extends MultiNodeSpec(SunnyWeatherMultiJvmSpec)