Merge branch 'master' into wip-2109-port-cluster-test-jboner

Conflicts:
	akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala
This commit is contained in:
Patrik Nordwall 2012-05-28 14:15:44 +02:00
commit c63a28f39a
31 changed files with 791 additions and 501 deletions

View file

@ -1,90 +1,95 @@
///**
// * Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
// */
//
//package akka.cluster
//
//import org.scalatest.BeforeAndAfter
//import com.typesafe.config.ConfigFactory
//import akka.remote.testkit.MultiNodeConfig
//import akka.remote.testkit.MultiNodeSpec
//import akka.testkit._
//
//object JoinTwoClustersMultiJvmSpec extends MultiNodeConfig {
// val a1 = role("a1")
// val a2 = role("a2")
// val b1 = role("b1")
// val b2 = role("b2")
// val c1 = role("c1")
// val c2 = role("c2")
//
// commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
//
//}
//
//class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec
//class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec
//class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec
//class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec
//class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec
//class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec
//
//abstract class JoinTwoClustersSpec extends MultiNodeSpec(JoinTwoClustersMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
// import JoinTwoClustersMultiJvmSpec._
//
// override def initialParticipants = 6
//
// after {
// testConductor.enter("after")
// }
//
// val a1Address = node(a1).address
// val b1Address = node(b1).address
// val c1Address = node(c1).address
//
// "Three different clusters (A, B and C)" must {
//
// "be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
//
// runOn(a1, a2) {
// cluster.join(a1Address)
// }
// runOn(b1, b2) {
// cluster.join(b1Address)
// }
// runOn(c1, c2) {
// cluster.join(c1Address)
// }
//
// awaitUpConvergence(numberOfMembers = 2)
//
// assertLeader(a1, a2)
// assertLeader(b1, b2)
// assertLeader(c1, c2)
//
// runOn(b2) {
// cluster.join(a1Address)
// }
//
// runOn(a1, a2, b1, b2) {
// awaitUpConvergence(numberOfMembers = 4)
// }
//
// assertLeader(a1, a2, b1, b2)
// assertLeader(c1, c2)
//
// }
//
// "be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
//
// runOn(b2) {
// cluster.join(c1Address)
// }
//
// awaitUpConvergence(numberOfMembers = 6)
//
// assertLeader(a1, a2, b1, b2, c1, c2)
// }
// }
//
//}
/**
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
*/
package akka.cluster
import org.scalatest.BeforeAndAfter
import com.typesafe.config.ConfigFactory
import akka.remote.testkit.MultiNodeConfig
import akka.remote.testkit.MultiNodeSpec
import akka.testkit._
object JoinTwoClustersMultiJvmSpec extends MultiNodeConfig {
val a1 = role("a1")
val a2 = role("a2")
val b1 = role("b1")
val b2 = role("b2")
val c1 = role("c1")
val c2 = role("c2")
commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
}
class JoinTwoClustersMultiJvmNode1 extends JoinTwoClustersSpec
class JoinTwoClustersMultiJvmNode2 extends JoinTwoClustersSpec
class JoinTwoClustersMultiJvmNode3 extends JoinTwoClustersSpec
class JoinTwoClustersMultiJvmNode4 extends JoinTwoClustersSpec
class JoinTwoClustersMultiJvmNode5 extends JoinTwoClustersSpec
class JoinTwoClustersMultiJvmNode6 extends JoinTwoClustersSpec
abstract class JoinTwoClustersSpec extends MultiNodeSpec(JoinTwoClustersMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
import JoinTwoClustersMultiJvmSpec._
override def initialParticipants = 6
after {
testConductor.enter("after")
}
lazy val a1Address = node(a1).address
lazy val b1Address = node(b1).address
lazy val c1Address = node(c1).address
"Three different clusters (A, B and C)" must {
"be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
// make sure that the node-to-join is started before other join
runOn(a1, b1, c1) {
cluster
}
testConductor.enter("first-started")
runOn(a1, a2) {
cluster.join(a1Address)
}
runOn(b1, b2) {
cluster.join(b1Address)
}
runOn(c1, c2) {
cluster.join(c1Address)
}
awaitUpConvergence(numberOfMembers = 2)
assertLeader(a1, a2)
assertLeader(b1, b2)
assertLeader(c1, c2)
runOn(b2) {
cluster.join(a1Address)
}
runOn(a1, a2, b1, b2) {
awaitUpConvergence(numberOfMembers = 4)
}
assertLeader(a1, a2, b1, b2)
assertLeader(c1, c2)
}
"be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
runOn(b2) {
cluster.join(c1Address)
}
awaitUpConvergence(numberOfMembers = 6)
assertLeader(a1, a2, b1, b2, c1, c2)
}
}
}

View file

@ -0,0 +1,108 @@
/**
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
*/
package akka.cluster
import com.typesafe.config.ConfigFactory
import akka.remote.testkit.MultiNodeConfig
import akka.remote.testkit.MultiNodeSpec
import akka.testkit._
object LeaderElectionMultiJvmSpec extends MultiNodeConfig {
val controller = role("controller")
val first = role("first")
val second = role("second")
val third = role("third")
val fourth = role("fourth")
commonConfig(debugConfig(on = false).
withFallback(ConfigFactory.parseString("""
akka.cluster.auto-down = off
""")).
withFallback(MultiNodeClusterSpec.clusterConfig))
}
class LeaderElectionMultiJvmNode1 extends LeaderElectionSpec
class LeaderElectionMultiJvmNode2 extends LeaderElectionSpec
class LeaderElectionMultiJvmNode3 extends LeaderElectionSpec
class LeaderElectionMultiJvmNode4 extends LeaderElectionSpec
class LeaderElectionMultiJvmNode5 extends LeaderElectionSpec
abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSpec) with MultiNodeClusterSpec {
import LeaderElectionMultiJvmSpec._
override def initialParticipants = 5
lazy val firstAddress = node(first).address
// sorted in the order used by the cluster
lazy val roles = Seq(first, second, third, fourth).sorted
"A cluster of four nodes" must {
"be able to 'elect' a single leader" taggedAs LongRunningTest in {
// make sure that the node-to-join is started before other join
runOn(first) {
cluster
}
testConductor.enter("first-started")
if (mySelf != controller) {
cluster.join(firstAddress)
awaitUpConvergence(numberOfMembers = roles.size)
cluster.isLeader must be(mySelf == roles.head)
}
testConductor.enter("after")
}
def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = {
val currentRoles = roles.drop(alreadyShutdown)
currentRoles.size must be >= (2)
val leader = currentRoles.head
val aUser = currentRoles.last
mySelf match {
case `controller`
testConductor.enter("before-shutdown")
testConductor.shutdown(leader, 0)
testConductor.removeNode(leader)
testConductor.enter("after-shutdown", "after-down", "completed")
case `leader`
testConductor.enter("before-shutdown")
// this node will be shutdown by the controller and doesn't participate in more barriers
case `aUser`
val leaderAddress = node(leader).address
testConductor.enter("before-shutdown", "after-shutdown")
// user marks the shutdown leader as DOWN
cluster.down(leaderAddress)
testConductor.enter("after-down", "completed")
case _ if currentRoles.tail.contains(mySelf)
// remaining cluster nodes, not shutdown
testConductor.enter("before-shutdown", "after-shutdown", "after-down")
awaitUpConvergence(currentRoles.size - 1)
val nextExpectedLeader = currentRoles.tail.head
cluster.isLeader must be(mySelf == nextExpectedLeader)
testConductor.enter("completed")
}
}
"be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in {
shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0)
}
"be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in {
shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1)
}
}
}

View file

@ -1,77 +1,83 @@
///**
// * Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
// */
//package akka.cluster
//
//import scala.collection.immutable.SortedSet
//import org.scalatest.BeforeAndAfter
//import com.typesafe.config.ConfigFactory
//import akka.remote.testkit.MultiNodeConfig
//import akka.remote.testkit.MultiNodeSpec
//import akka.testkit._
//
//object MembershipChangeListenerMultiJvmSpec extends MultiNodeConfig {
// val first = role("first")
// val second = role("second")
// val third = role("third")
//
// commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
//
//}
//
//class MembershipChangeListenerMultiJvmNode1 extends MembershipChangeListenerSpec
//class MembershipChangeListenerMultiJvmNode2 extends MembershipChangeListenerSpec
//class MembershipChangeListenerMultiJvmNode3 extends MembershipChangeListenerSpec
//
//abstract class MembershipChangeListenerSpec extends MultiNodeSpec(MembershipChangeListenerMultiJvmSpec)
// with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
// import MembershipChangeListenerMultiJvmSpec._
//
// override def initialParticipants = 3
//
// after {
// testConductor.enter("after")
// }
//
// "A set of connected cluster systems" must {
//
// val firstAddress = node(first).address
// val secondAddress = node(second).address
//
// "(when two systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
//
// runOn(first, second) {
// cluster.join(firstAddress)
// val latch = TestLatch()
// cluster.registerListener(new MembershipChangeListener {
// def notify(members: SortedSet[Member]) {
// if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
// latch.countDown()
// }
// })
// latch.await
// cluster.convergence.isDefined must be(true)
// }
//
// }
//
// "(when three systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
//
// runOn(third) {
// cluster.join(firstAddress)
// }
//
// val latch = TestLatch()
// cluster.registerListener(new MembershipChangeListener {
// def notify(members: SortedSet[Member]) {
// if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
// latch.countDown()
// }
// })
// latch.await
// cluster.convergence.isDefined must be(true)
//
// }
// }
//
//}
/**
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
*/
package akka.cluster
import scala.collection.immutable.SortedSet
import org.scalatest.BeforeAndAfter
import com.typesafe.config.ConfigFactory
import akka.remote.testkit.MultiNodeConfig
import akka.remote.testkit.MultiNodeSpec
import akka.testkit._
object MembershipChangeListenerMultiJvmSpec extends MultiNodeConfig {
val first = role("first")
val second = role("second")
val third = role("third")
commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
}
class MembershipChangeListenerMultiJvmNode1 extends MembershipChangeListenerSpec
class MembershipChangeListenerMultiJvmNode2 extends MembershipChangeListenerSpec
class MembershipChangeListenerMultiJvmNode3 extends MembershipChangeListenerSpec
abstract class MembershipChangeListenerSpec extends MultiNodeSpec(MembershipChangeListenerMultiJvmSpec)
with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
import MembershipChangeListenerMultiJvmSpec._
override def initialParticipants = 3
after {
testConductor.enter("after")
}
lazy val firstAddress = node(first).address
lazy val secondAddress = node(second).address
"A set of connected cluster systems" must {
"(when two systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
// make sure that the node-to-join is started before other join
runOn(first) {
cluster
}
testConductor.enter("first-started")
runOn(first, second) {
cluster.join(firstAddress)
val latch = TestLatch()
cluster.registerListener(new MembershipChangeListener {
def notify(members: SortedSet[Member]) {
if (members.size == 2 && members.forall(_.status == MemberStatus.Up))
latch.countDown()
}
})
latch.await
cluster.convergence.isDefined must be(true)
}
}
"(when three systems) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
runOn(third) {
cluster.join(firstAddress)
}
val latch = TestLatch()
cluster.registerListener(new MembershipChangeListener {
def notify(members: SortedSet[Member]) {
if (members.size == 3 && members.forall(_.status == MemberStatus.Up))
latch.countDown()
}
})
latch.await
cluster.convergence.isDefined must be(true)
}
}
}

View file

@ -45,8 +45,7 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
*/
def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(mySelf)) {
nodesInCluster.length must not be (0)
import Member.addressOrdering
val expectedLeader = nodesInCluster.map(role (role, node(role).address)).sortBy(_._2).head._1
val expectedLeader = roleOfLeader(nodesInCluster)
cluster.isLeader must be(ifNode(expectedLeader)(true)(false))
}
@ -66,6 +65,23 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
def awaitUpConvergence(nrOfMembers: Int, canNotBePartOfRing: Seq[Address] = Seq.empty[Address]): Unit = {
awaitCond(cluster.latestGossip.members.size == nrOfMembers)
awaitCond(cluster.latestGossip.members.forall(_.status == MemberStatus.Up))
awaitCond(canNotBePartOfRing forall (address => !(cluster.latestGossip.members exists (_.address.port == address.port))))
awaitCond(canNotBePartOfRing forall (address !(cluster.latestGossip.members exists (_.address == address))))
}
def roleOfLeader(nodesInCluster: Seq[RoleName]): RoleName = {
nodesInCluster.length must not be (0)
nodesInCluster.sorted.head
}
/**
* Sort the roles in the order used by the cluster.
*/
implicit val clusterOrdering: Ordering[RoleName] = new Ordering[RoleName] {
import Member.addressOrdering
def compare(x: RoleName, y: RoleName) = addressOrdering.compare(node(x).address, node(y).address)
}
def roleName(address: Address): Option[RoleName] = {
testConductor.getNodes.await.find(node(_).address == address)
}
}

View file

@ -1,70 +1,76 @@
///**
// * Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
// */
//package akka.cluster
//
//import com.typesafe.config.ConfigFactory
//import org.scalatest.BeforeAndAfter
//import akka.remote.testkit.MultiNodeConfig
//import akka.remote.testkit.MultiNodeSpec
//import akka.testkit._
//
//object NodeMembershipMultiJvmSpec extends MultiNodeConfig {
// val first = role("first")
// val second = role("second")
// val third = role("third")
//
// commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
//
//}
//
//class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec
//class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec
//class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec
//
//abstract class NodeMembershipSpec extends MultiNodeSpec(NodeMembershipMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
// import NodeMembershipMultiJvmSpec._
//
// override def initialParticipants = 3
//
// after {
// testConductor.enter("after")
// }
//
// val firstAddress = node(first).address
// val secondAddress = node(second).address
// val thirdAddress = node(third).address
//
// "A set of connected cluster systems" must {
//
// "(when two systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
//
// runOn(first, second) {
// cluster.join(firstAddress)
// awaitCond(cluster.latestGossip.members.size == 2)
// assertMembers(cluster.latestGossip.members, firstAddress, secondAddress)
// awaitCond {
// cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
// }
// awaitCond(cluster.convergence.isDefined)
// }
//
// }
//
// "(when three systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
//
// runOn(third) {
// cluster.join(firstAddress)
// }
//
// awaitCond(cluster.latestGossip.members.size == 3)
// assertMembers(cluster.latestGossip.members, firstAddress, secondAddress, thirdAddress)
// awaitCond {
// cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
// }
// awaitCond(cluster.convergence.isDefined)
//
// }
// }
//
//}
/**
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
*/
package akka.cluster
import com.typesafe.config.ConfigFactory
import org.scalatest.BeforeAndAfter
import akka.remote.testkit.MultiNodeConfig
import akka.remote.testkit.MultiNodeSpec
import akka.testkit._
object NodeMembershipMultiJvmSpec extends MultiNodeConfig {
val first = role("first")
val second = role("second")
val third = role("third")
commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
}
class NodeMembershipMultiJvmNode1 extends NodeMembershipSpec
class NodeMembershipMultiJvmNode2 extends NodeMembershipSpec
class NodeMembershipMultiJvmNode3 extends NodeMembershipSpec
abstract class NodeMembershipSpec extends MultiNodeSpec(NodeMembershipMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
import NodeMembershipMultiJvmSpec._
override def initialParticipants = 3
after {
testConductor.enter("after")
}
lazy val firstAddress = node(first).address
lazy val secondAddress = node(second).address
lazy val thirdAddress = node(third).address
"A set of connected cluster systems" must {
"(when two systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
// make sure that the node-to-join is started before other join
runOn(first) {
cluster
}
testConductor.enter("first-started")
runOn(first, second) {
cluster.join(firstAddress)
awaitCond(cluster.latestGossip.members.size == 2)
assertMembers(cluster.latestGossip.members, firstAddress, secondAddress)
awaitCond {
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
}
awaitCond(cluster.convergence.isDefined)
}
}
"(when three systems) start gossiping to each other so that both systems gets the same gossip info" taggedAs LongRunningTest in {
runOn(third) {
cluster.join(firstAddress)
}
awaitCond(cluster.latestGossip.members.size == 3)
assertMembers(cluster.latestGossip.members, firstAddress, secondAddress, thirdAddress)
awaitCond {
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
}
awaitCond(cluster.convergence.isDefined)
}
}
}

View file

@ -1,74 +1,74 @@
///**
// * Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
// */
//package akka.cluster
//
//import com.typesafe.config.ConfigFactory
//import org.scalatest.BeforeAndAfter
//import akka.remote.testkit.MultiNodeConfig
//import akka.remote.testkit.MultiNodeSpec
//import akka.testkit._
//
//object NodeStartupMultiJvmSpec extends MultiNodeConfig {
// val first = role("first")
// val second = role("second")
//
// commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
//
//}
//
//class NodeStartupMultiJvmNode1 extends NodeStartupSpec
//class NodeStartupMultiJvmNode2 extends NodeStartupSpec
//
//abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
// import NodeStartupMultiJvmSpec._
//
// override def initialParticipants = 2
//
// after {
// testConductor.enter("after")
// }
//
// val firstAddress = node(first).address
// val secondAddress = node(second).address
//
// "A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
//
// "be a singleton cluster when started up" taggedAs LongRunningTest in {
// runOn(first) {
// awaitCond(cluster.isSingletonCluster)
// // FIXME #2117 singletonCluster should reach convergence
// //awaitCond(cluster.convergence.isDefined)
// }
// }
//
// "be in 'Joining' phase when started up" taggedAs LongRunningTest in {
// runOn(first) {
// val members = cluster.latestGossip.members
// members.size must be(1)
//
// val joiningMember = members find (_.address == firstAddress)
// joiningMember must not be (None)
// joiningMember.get.status must be(MemberStatus.Joining)
// }
// }
// }
//
// "A second cluster node" must {
// "join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
//
// runOn(second) {
// cluster.join(firstAddress)
// }
//
// awaitCond {
// cluster.latestGossip.members.exists { member
// member.address == secondAddress && member.status == MemberStatus.Up
// }
// }
// cluster.latestGossip.members.size must be(2)
// awaitCond(cluster.convergence.isDefined)
// }
// }
//
//}
/**
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
*/
package akka.cluster
import com.typesafe.config.ConfigFactory
import org.scalatest.BeforeAndAfter
import akka.remote.testkit.MultiNodeConfig
import akka.remote.testkit.MultiNodeSpec
import akka.testkit._
object NodeStartupMultiJvmSpec extends MultiNodeConfig {
val first = role("first")
val second = role("second")
commonConfig(debugConfig(on = false).withFallback(MultiNodeClusterSpec.clusterConfig))
}
class NodeStartupMultiJvmNode1 extends NodeStartupSpec
class NodeStartupMultiJvmNode2 extends NodeStartupSpec
abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) with MultiNodeClusterSpec with ImplicitSender with BeforeAndAfter {
import NodeStartupMultiJvmSpec._
override def initialParticipants = 2
after {
testConductor.enter("after")
}
lazy val firstAddress = node(first).address
lazy val secondAddress = node(second).address
"A first cluster node with a 'node-to-join' config set to empty string (singleton cluster)" must {
"be a singleton cluster when started up" taggedAs LongRunningTest in {
runOn(first) {
awaitCond(cluster.isSingletonCluster)
// FIXME #2117 singletonCluster should reach convergence
//awaitCond(cluster.convergence.isDefined)
}
}
"be in 'Joining' phase when started up" taggedAs LongRunningTest in {
runOn(first) {
val members = cluster.latestGossip.members
members.size must be(1)
val joiningMember = members find (_.address == firstAddress)
joiningMember must not be (None)
joiningMember.get.status must be(MemberStatus.Joining)
}
}
}
"A second cluster node" must {
"join the other node cluster when sending a Join command" taggedAs LongRunningTest in {
runOn(second) {
cluster.join(firstAddress)
}
awaitCond {
cluster.latestGossip.members.exists { member
member.address == secondAddress && member.status == MemberStatus.Up
}
}
cluster.latestGossip.members.size must be(2)
awaitCond(cluster.convergence.isDefined)
}
}
}