Merge branch 'master' into wip-2112-leader-downing-spec-jboner

Conflicts:
	akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

akka-cluster/src/multi-jvm/scala/akka/cluster/ClientDowningNodeThatIsUnreachableSpec.scala

@@ -0,0 +1,81 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.BeforeAndAfter
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+import akka.actor.Address
+
+object ClientDowningNodeThatIsUnreachableMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+
+  commonConfig(debugConfig(on = false).
+    withFallback(ConfigFactory.parseString("akka.cluster.auto-down = off")).
+    withFallback(MultiNodeClusterSpec.clusterConfig))
+}
+
+class ClientDowningNodeThatIsUnreachableMultiJvmNode1 extends ClientDowningNodeThatIsUnreachableSpec
+class ClientDowningNodeThatIsUnreachableMultiJvmNode2 extends ClientDowningNodeThatIsUnreachableSpec
+class ClientDowningNodeThatIsUnreachableMultiJvmNode3 extends ClientDowningNodeThatIsUnreachableSpec
+class ClientDowningNodeThatIsUnreachableMultiJvmNode4 extends ClientDowningNodeThatIsUnreachableSpec
+
+class ClientDowningNodeThatIsUnreachableSpec
+  extends MultiNodeSpec(ClientDowningNodeThatIsUnreachableMultiJvmSpec)
+  with MultiNodeClusterSpec
+  with ImplicitSender with BeforeAndAfter {
+  import ClientDowningNodeThatIsUnreachableMultiJvmSpec._
+
+  override def initialParticipants = 4
+
+  "Client of a 4 node cluster" must {
+
+    "be able to DOWN a node that is UNREACHABLE (killed)" taggedAs LongRunningTest in {
+      runOn(first) {
+        cluster.self
+        awaitUpConvergence(numberOfMembers = 4)
+
+        val thirdAddress = node(third).address
+        testConductor.enter("all-up")
+
+        // kill 'third' node
+        testConductor.shutdown(third, 0)
+        testConductor.removeNode(third)
+
+        // mark 'third' node as DOWN
+        cluster.down(thirdAddress)
+        testConductor.enter("down-third-node")
+
+        awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(thirdAddress))
+        cluster.latestGossip.members.exists(_.address == thirdAddress) must be(false)
+        testConductor.enter("await-completion")
+      }
+
+      runOn(third) {
+        cluster.join(node(first).address)
+
+        awaitUpConvergence(numberOfMembers = 4)
+        testConductor.enter("all-up")
+      }
+
+      runOn(second, fourth) {
+        cluster.join(node(first).address)
+        awaitUpConvergence(numberOfMembers = 4)
+
+        val thirdAddress = node(third).address
+        testConductor.enter("all-up")
+
+        testConductor.enter("down-third-node")
+
+        awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(thirdAddress))
+        testConductor.enter("await-completion")
+      }
+    }
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/ClientDowningNodeThatIsUpSpec.scala

@@ -0,0 +1,77 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.BeforeAndAfter
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+import akka.actor.Address
+
+object ClientDowningNodeThatIsUpMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+
+  commonConfig(debugConfig(on = false).
+    withFallback(ConfigFactory.parseString("akka.cluster.auto-down = off")).
+    withFallback(MultiNodeClusterSpec.clusterConfig))
+}
+
+class ClientDowningNodeThatIsUpMultiJvmNode1 extends ClientDowningNodeThatIsUpSpec
+class ClientDowningNodeThatIsUpMultiJvmNode2 extends ClientDowningNodeThatIsUpSpec
+class ClientDowningNodeThatIsUpMultiJvmNode3 extends ClientDowningNodeThatIsUpSpec
+class ClientDowningNodeThatIsUpMultiJvmNode4 extends ClientDowningNodeThatIsUpSpec
+
+class ClientDowningNodeThatIsUpSpec
+  extends MultiNodeSpec(ClientDowningNodeThatIsUpMultiJvmSpec)
+  with MultiNodeClusterSpec
+  with ImplicitSender with BeforeAndAfter {
+  import ClientDowningNodeThatIsUpMultiJvmSpec._
+
+  override def initialParticipants = 4
+
+  "Client of a 4 node cluster" must {
+
+    "be able to DOWN a node that is UP (healthy and available)" taggedAs LongRunningTest in {
+      runOn(first) {
+        cluster.self
+        awaitUpConvergence(numberOfMembers = 4)
+
+        val thirdAddress = node(third).address
+        testConductor.enter("all-up")
+
+        // mark 'third' node as DOWN
+        testConductor.removeNode(third)
+        cluster.down(thirdAddress)
+        testConductor.enter("down-third-node")
+
+        awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(thirdAddress))
+        cluster.latestGossip.members.exists(_.address == thirdAddress) must be(false)
+        testConductor.enter("await-completion")
+      }
+
+      runOn(third) {
+        cluster.join(node(first).address)
+        awaitUpConvergence(numberOfMembers = 4)
+        testConductor.enter("all-up")
+      }
+
+      runOn(second, fourth) {
+        cluster.join(node(first).address)
+        awaitUpConvergence(numberOfMembers = 4)
+
+        val thirdAddress = node(third).address
+        testConductor.enter("all-up")
+
+        testConductor.enter("down-third-node")
+
+        awaitUpConvergence(numberOfMembers = 3, canNotBePartOfMemberRing = Seq(thirdAddress))
+        testConductor.enter("await-completion")
+      }
+    }
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/JoinTwoClustersSpec.scala

@@ -1,90 +1,95 @@
-///**
-// *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
-// */
-//
-//package akka.cluster
-//
-//import org.scalatest.BeforeAndAfter
-//import com.typesafe.config.ConfigFactory
-//import akka.remote.testkit.MultiNodeConfig
-//import akka.remote.testkit.MultiNodeSpec
-//import akka.testkit._
-//
-//object JoinTwoClustersMultiJvmSpec extends MultiNodeConfig {
-//  val a1 = role("a1")
-//  val a2 = role("a2")
-//  val b1 = role("b1")
-//  val b2 = role("b2")
-//  val c1 = role("c1")
-//  val c2 = role("c2")
-//
-//  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
-//
-//}
-//
-//class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec
-//class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec
-//class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec
-//class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec
-//class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec
-//class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec
-//
-//abstract class JoinTwoClustersSpec extends MultiNodeSpec(JoinTwoClustersMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
-//  import JoinTwoClustersMultiJvmSpec._
-//
-//  override def initialParticipants = 6
-//
-//  after {
-//    testConductor.enter("after")
-//  }
-//
-//  val a1Address = node(a1).address
-//  val b1Address = node(b1).address
-//  val c1Address = node(c1).address
-//
-//  "Three different clusters (A, B and C)" must {
-//
-//    "be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
-//
-//      runOn(a1, a2) {
-//        cluster.join(a1Address)
-//      }
-//      runOn(b1, b2) {
-//        cluster.join(b1Address)
-//      }
-//      runOn(c1, c2) {
-//        cluster.join(c1Address)
-//      }
-//
-//      awaitUpConvergence(numberOfMembers = 2)
-//
-//      assertLeader(a1, a2)
-//      assertLeader(b1, b2)
-//      assertLeader(c1, c2)
-//
-//      runOn(b2) {
-//        cluster.join(a1Address)
-//      }
-//
-//      runOn(a1, a2, b1, b2) {
-//        awaitUpConvergence(numberOfMembers = 4)
-//      }
-//
-//      assertLeader(a1, a2, b1, b2)
-//      assertLeader(c1, c2)
-//
-//    }
-//
-//    "be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
-//
-//      runOn(b2) {
-//        cluster.join(c1Address)
-//      }
-//
-//      awaitUpConvergence(numberOfMembers = 6)
-//
-//      assertLeader(a1, a2, b1, b2, c1, c2)
-//    }
-//  }
-//
-//}
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+
+package akka.cluster
+
+import org.scalatest.BeforeAndAfter
+import com.typesafe.config.ConfigFactory
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object JoinTwoClustersMultiJvmSpec extends MultiNodeConfig {
+  val a1 = role("a1")
+  val a2 = role("a2")
+  val b1 = role("b1")
+  val b2 = role("b2")
+  val c1 = role("c1")
+  val c2 = role("c2")
+
+  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+}
+
+class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec
+class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec
+
+abstract class JoinTwoClustersSpec extends MultiNodeSpec(JoinTwoClustersMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
+  import JoinTwoClustersMultiJvmSpec._
+
+  override def initialParticipants = 6
+
+  after {
+    testConductor.enter("after")
+  }
+
+  lazy val a1Address = node(a1).address
+  lazy val b1Address = node(b1).address
+  lazy val c1Address = node(c1).address
+
+  "Three different clusters (A, B and C)" must {
+
+    "be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
+      // make sure that the node-to-join is started before other join
+      runOn(a1, b1, c1) {
+        cluster
+      }
+      testConductor.enter("first-started")
+
+      runOn(a1, a2) {
+        cluster.join(a1Address)
+      }
+      runOn(b1, b2) {
+        cluster.join(b1Address)
+      }
+      runOn(c1, c2) {
+        cluster.join(c1Address)
+      }
+
+      awaitUpConvergence(numberOfMembers = 2)
+
+      assertLeader(a1, a2)
+      assertLeader(b1, b2)
+      assertLeader(c1, c2)
+
+      runOn(b2) {
+        cluster.join(a1Address)
+      }
+
+      runOn(a1, a2, b1, b2) {
+        awaitUpConvergence(numberOfMembers = 4)
+      }
+
+      assertLeader(a1, a2, b1, b2)
+      assertLeader(c1, c2)
+
+    }
+
+    "be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
+
+      runOn(b2) {
+        cluster.join(c1Address)
+      }
+
+      awaitUpConvergence(numberOfMembers = 6)
+
+      assertLeader(a1, a2, b1, b2, c1, c2)
+    }
+  }
+
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala

@@ -0,0 +1,108 @@
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object LeaderElectionMultiJvmSpec extends MultiNodeConfig {
+  val controller = role("controller")
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+
+  commonConfig(debugConfig(on = false).
+    withFallback(ConfigFactory.parseString("""
+        akka.cluster.auto-down = off
+        """)).
+    withFallback(MultiNodeClusterSpec.clusterConfig))
+
+}
+
+class LeaderElectionMultiJvmNode1 extends LeaderElectionSpec
+class LeaderElectionMultiJvmNode2 extends LeaderElectionSpec
+class LeaderElectionMultiJvmNode3 extends LeaderElectionSpec
+class LeaderElectionMultiJvmNode4 extends LeaderElectionSpec
+class LeaderElectionMultiJvmNode5 extends LeaderElectionSpec
+
+abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSpec) with MultiNodeClusterSpec {
+  import LeaderElectionMultiJvmSpec._
+
+  override def initialParticipants = 5
+
+  lazy val firstAddress = node(first).address
+
+  // sorted in the order used by the cluster
+  lazy val roles = Seq(first, second, third, fourth).sorted
+
+  "A cluster of four nodes" must {
+
+    "be able to 'elect' a single leader" taggedAs LongRunningTest in {
+      // make sure that the node-to-join is started before other join
+      runOn(first) {
+        cluster
+      }
+      testConductor.enter("first-started")
+
+      if (mySelf != controller) {
+        cluster.join(firstAddress)
+        awaitUpConvergence(numberOfMembers = roles.size)
+        cluster.isLeader must be(mySelf == roles.head)
+      }
+      testConductor.enter("after")
+    }
+
+    def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = {
+      val currentRoles = roles.drop(alreadyShutdown)
+      currentRoles.size must be >= (2)
+      val leader = currentRoles.head
+      val aUser = currentRoles.last
+
+      mySelf match {
+
+        case `controller` ⇒
+          testConductor.enter("before-shutdown")
+          testConductor.shutdown(leader, 0)
+          testConductor.removeNode(leader)
+          testConductor.enter("after-shutdown", "after-down", "completed")
+
+        case `leader` ⇒
+          testConductor.enter("before-shutdown")
+        // this node will be shutdown by the controller and doesn't participate in more barriers
+
+        case `aUser` ⇒
+          val leaderAddress = node(leader).address
+          testConductor.enter("before-shutdown", "after-shutdown")
+          // user marks the shutdown leader as DOWN
+          cluster.down(leaderAddress)
+          testConductor.enter("after-down", "completed")
+
+        case _ if currentRoles.tail.contains(mySelf) ⇒
+          // remaining cluster nodes, not shutdown
+          testConductor.enter("before-shutdown", "after-shutdown", "after-down")
+
+          awaitUpConvergence(currentRoles.size - 1)
+          val nextExpectedLeader = currentRoles.tail.head
+          cluster.isLeader must be(mySelf == nextExpectedLeader)
+
+          testConductor.enter("completed")
+
+      }
+
+    }
+
+    "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in {
+      shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0)
+    }
+
+    "be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in {
+      shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1)
+    }
+  }
+
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MembershipChangeListenerSpec.scala

@@ -1,77 +1,83 @@
-///**
-// *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
-// */
-//package akka.cluster
-//
-//import scala.collection.immutable.SortedSet
-//import org.scalatest.BeforeAndAfter
-//import com.typesafe.config.ConfigFactory
-//import akka.remote.testkit.MultiNodeConfig
-//import akka.remote.testkit.MultiNodeSpec
-//import akka.testkit._
-//
-//object MembershipChangeListenerMultiJvmSpec extends MultiNodeConfig {
-//  val first = role("first")
-//  val second = role("second")
-//  val third = role("third")
-//
-//  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
-//
-//}
-//
-//class MembershipChangeListenerMultiJvmNode1 extends MembershipChangeListenerSpec
-//class MembershipChangeListenerMultiJvmNode2 extends MembershipChangeListenerSpec
-//class MembershipChangeListenerMultiJvmNode3 extends MembershipChangeListenerSpec
-//
-//abstract class MembershipChangeListenerSpec extends MultiNodeSpec(MembershipChangeListenerMultiJvmSpec)
-//  with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
-//  import MembershipChangeListenerMultiJvmSpec._
-//
-//  override def initialParticipants = 3
-//
-//  after {
-//    testConductor.enter("after")
-//  }
-//
-//  "A set of connected cluster systems" must {
-//
-//    val firstAddress = node(first).address
-//    val secondAddress = node(second).address
-//
-//    "(when two systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
-//
-//      runOn(first, second) {
-//        cluster.join(firstAddress)
-//        val latch = TestLatch()
-//        cluster.registerListener(new MembershipChangeListener {
-//          def notify(members: SortedSet[Member]) {
-//            if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
-//              latch.countDown()
-//          }
-//        })
-//        latch.await
-//        cluster.convergence.isDefined must be(true)
-//      }
-//
-//    }
-//
-//    "(when three systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
-//
-//      runOn(third) {
-//        cluster.join(firstAddress)
-//      }
-//
-//      val latch = TestLatch()
-//      cluster.registerListener(new MembershipChangeListener {
-//        def notify(members: SortedSet[Member]) {
-//          if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
-//            latch.countDown()
-//        }
-//      })
-//      latch.await
-//      cluster.convergence.isDefined must be(true)
-//
-//    }
-//  }
-//
-//}
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import scala.collection.immutable.SortedSet
+import org.scalatest.BeforeAndAfter
+import com.typesafe.config.ConfigFactory
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object MembershipChangeListenerMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+
+  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+}
+
+class MembershipChangeListenerMultiJvmNode1 extends MembershipChangeListenerSpec
+class MembershipChangeListenerMultiJvmNode2 extends MembershipChangeListenerSpec
+class MembershipChangeListenerMultiJvmNode3 extends MembershipChangeListenerSpec
+
+abstract class MembershipChangeListenerSpec extends MultiNodeSpec(MembershipChangeListenerMultiJvmSpec)
+  with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
+  import MembershipChangeListenerMultiJvmSpec._
+
+  override def initialParticipants = 3
+
+  after {
+    testConductor.enter("after")
+  }
+
+  lazy val firstAddress = node(first).address
+  lazy val secondAddress = node(second).address
+
+  "A set of connected cluster systems" must {
+
+    "(when two systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
+
+      // make sure that the node-to-join is started before other join
+      runOn(first) {
+        cluster
+      }
+      testConductor.enter("first-started")
+
+      runOn(first, second) {
+        cluster.join(firstAddress)
+        val latch = TestLatch()
+        cluster.registerListener(new MembershipChangeListener {
+          def notify(members: SortedSet[Member]) {
+            if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
+              latch.countDown()
+          }
+        })
+        latch.await
+        cluster.convergence.isDefined must be(true)
+      }
+
+    }
+
+    "(when three systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
+
+      runOn(third) {
+        cluster.join(firstAddress)
+      }
+
+      val latch = TestLatch()
+      cluster.registerListener(new MembershipChangeListener {
+        def notify(members: SortedSet[Member]) {
+          if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
+            latch.countDown()
+        }
+      })
+      latch.await
+      cluster.convergence.isDefined must be(true)
+
+    }
+  }
+
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

@@ -20,7 +20,6 @@ object MultiNodeClusterSpec {
       unreachable-nodes-reaper-frequency = 200 ms
       periodic-tasks-initial-delay       = 300 ms
     }
-
     akka.test {
       single-expect-default = 5 s
     }
@@ -49,8 +48,7 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
    */
   def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(mySelf)) {
     nodesInCluster.length must not be (0)
-    import Member.addressOrdering
-    val expectedLeader = nodesInCluster.map(role ⇒ (role, node(role).address)).sortBy(_._2).head._1
+    val expectedLeader = roleOfLeader(nodesInCluster)
     cluster.isLeader must be(ifNode(expectedLeader)(true)(false))
   }
 
@@ -67,7 +65,24 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
     awaitCond(cluster.convergence.isDefined, timeout)
     if (!canNotBePartOfMemberRing.isEmpty) // don't run this on an empty set
       awaitCond(
-        canNotBePartOfMemberRing forall (address => !(cluster.latestGossip.members exists (_.address == address))),
+        canNotBePartOfMemberRing forall (address ⇒ !(cluster.latestGossip.members exists (_.address == address))),
         timeout)
   }
+
+  def roleOfLeader(nodesInCluster: Seq[RoleName]): RoleName = {
+    nodesInCluster.length must not be (0)
+    nodesInCluster.sorted.head
+  }
+
+  /**
+   * Sort the roles in the order used by the cluster.
+   */
+  implicit val clusterOrdering: Ordering[RoleName] = new Ordering[RoleName] {
+    import Member.addressOrdering
+    def compare(x: RoleName, y: RoleName) = addressOrdering.compare(node(x).address, node(y).address)
+  }
+
+  def roleName(address: Address): Option[RoleName] = {
+    testConductor.getNodes.await.find(node(_).address == address)
+  }
 }

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeMembershipSpec.scala

@@ -1,70 +1,76 @@
-///**
-// *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
-// */
-//package akka.cluster
-//
-//import com.typesafe.config.ConfigFactory
-//import org.scalatest.BeforeAndAfter
-//import akka.remote.testkit.MultiNodeConfig
-//import akka.remote.testkit.MultiNodeSpec
-//import akka.testkit._
-//
-//object NodeMembershipMultiJvmSpec extends MultiNodeConfig {
-//  val first = role("first")
-//  val second = role("second")
-//  val third = role("third")
-//
-//  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
-//
-//}
-//
-//class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec
-//class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec
-//class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec
-//
-//abstract class NodeMembershipSpec extends MultiNodeSpec(NodeMembershipMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
-//  import NodeMembershipMultiJvmSpec._
-//
-//  override def initialParticipants = 3
-//
-//  after {
-//    testConductor.enter("after")
-//  }
-//
-//  val firstAddress = node(first).address
-//  val secondAddress = node(second).address
-//  val thirdAddress = node(third).address
-//
-//  "A set of connected cluster systems" must {
-//
-//    "(when two systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
-//
-//      runOn(first, second) {
-//        cluster.join(firstAddress)
-//        awaitCond(cluster.latestGossip.members.size == 2)
-//        assertMembers(cluster.latestGossip.members, firstAddress, secondAddress)
-//        awaitCond {
-//          cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
-//        }
-//        awaitCond(cluster.convergence.isDefined)
-//      }
-//
-//    }
-//
-//    "(when three systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
-//
-//      runOn(third) {
-//        cluster.join(firstAddress)
-//      }
-//
-//      awaitCond(cluster.latestGossip.members.size == 3)
-//      assertMembers(cluster.latestGossip.members, firstAddress, secondAddress, thirdAddress)
-//      awaitCond {
-//        cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
-//      }
-//      awaitCond(cluster.convergence.isDefined)
-//
-//    }
-//  }
-//
-//}
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.BeforeAndAfter
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object NodeMembershipMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+
+  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+}
+
+class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec
+class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec
+class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec
+
+abstract class NodeMembershipSpec extends MultiNodeSpec(NodeMembershipMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
+  import NodeMembershipMultiJvmSpec._
+
+  override def initialParticipants = 3
+
+  after {
+    testConductor.enter("after")
+  }
+
+  lazy val firstAddress = node(first).address
+  lazy val secondAddress = node(second).address
+  lazy val thirdAddress = node(third).address
+
+  "A set of connected cluster systems" must {
+
+    "(when two systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
+
+      // make sure that the node-to-join is started before other join
+      runOn(first) {
+        cluster
+      }
+      testConductor.enter("first-started")
+
+      runOn(first, second) {
+        cluster.join(firstAddress)
+        awaitCond(cluster.latestGossip.members.size == 2)
+        assertMembers(cluster.latestGossip.members, firstAddress, secondAddress)
+        awaitCond {
+          cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
+        }
+        awaitCond(cluster.convergence.isDefined)
+      }
+
+    }
+
+    "(when three systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
+
+      runOn(third) {
+        cluster.join(firstAddress)
+      }
+
+      awaitCond(cluster.latestGossip.members.size == 3)
+      assertMembers(cluster.latestGossip.members, firstAddress, secondAddress, thirdAddress)
+      awaitCond {
+        cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
+      }
+      awaitCond(cluster.convergence.isDefined)
+
+    }
+  }
+
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala

@@ -1,74 +1,74 @@
-///**
-// *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
-// */
-//package akka.cluster
-//
-//import com.typesafe.config.ConfigFactory
-//import org.scalatest.BeforeAndAfter
-//import akka.remote.testkit.MultiNodeConfig
-//import akka.remote.testkit.MultiNodeSpec
-//import akka.testkit._
-//
-//object NodeStartupMultiJvmSpec extends MultiNodeConfig {
-//  val first = role("first")
-//  val second = role("second")
-//
-//  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
-//
-//}
-//
-//class NodeStartupMultiJvmNode1 extends NodeStartupSpec
-//class NodeStartupMultiJvmNode2 extends NodeStartupSpec
-//
-//abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
-//  import NodeStartupMultiJvmSpec._
-//
-//  override def initialParticipants = 2
-//
-//  after {
-//    testConductor.enter("after")
-//  }
-//
-//  val firstAddress = node(first).address
-//  val secondAddress = node(second).address
-//
-//  "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
-//
-//    "be a singleton cluster when started up" taggedAs LongRunningTest in {
-//      runOn(first) {
-//        awaitCond(cluster.isSingletonCluster)
-//        // FIXME #2117 singletonCluster should reach convergence
-//        //awaitCond(cluster.convergence.isDefined)
-//      }
-//    }
-//
-//    "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
-//      runOn(first) {
-//        val members = cluster.latestGossip.members
-//        members.size must be(1)
-//
-//        val joiningMember = members find (_.address == firstAddress)
-//        joiningMember must not be (None)
-//        joiningMember.get.status must be(MemberStatus.Joining)
-//      }
-//    }
-//  }
-//
-//  "A second cluster node" must {
-//    "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
-//
-//      runOn(second) {
-//        cluster.join(firstAddress)
-//      }
-//
-//      awaitCond {
-//        cluster.latestGossip.members.exists { member ⇒
-//          member.address == secondAddress && member.status == MemberStatus.Up
-//        }
-//      }
-//      cluster.latestGossip.members.size must be(2)
-//      awaitCond(cluster.convergence.isDefined)
-//    }
-//  }
-//
-//}
+/**
+ *  Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
+ */
+package akka.cluster
+
+import com.typesafe.config.ConfigFactory
+import org.scalatest.BeforeAndAfter
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+object NodeStartupMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+
+  commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+}
+
+class NodeStartupMultiJvmNode1 extends NodeStartupSpec
+class NodeStartupMultiJvmNode2 extends NodeStartupSpec
+
+abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
+  import NodeStartupMultiJvmSpec._
+
+  override def initialParticipants = 2
+
+  after {
+    testConductor.enter("after")
+  }
+
+  lazy val firstAddress = node(first).address
+  lazy val secondAddress = node(second).address
+
+  "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
+
+    "be a singleton cluster when started up" taggedAs LongRunningTest in {
+      runOn(first) {
+        awaitCond(cluster.isSingletonCluster)
+        // FIXME #2117 singletonCluster should reach convergence
+        //awaitCond(cluster.convergence.isDefined)
+      }
+    }
+
+    "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
+      runOn(first) {
+        val members = cluster.latestGossip.members
+        members.size must be(1)
+
+        val joiningMember = members find (_.address == firstAddress)
+        joiningMember must not be (None)
+        joiningMember.get.status must be(MemberStatus.Joining)
+      }
+    }
+  }
+
+  "A second cluster node" must {
+    "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
+
+      runOn(second) {
+        cluster.join(firstAddress)
+      }
+
+      awaitCond {
+        cluster.latestGossip.members.exists { member ⇒
+          member.address == secondAddress && member.status == MemberStatus.Up
+        }
+      }
+      cluster.latestGossip.members.size must be(2)
+      awaitCond(cluster.convergence.isDefined)
+    }
+  }
+
+}