From 57313cc9e0cbbc0bde9a2c7f33bc3c21b46a8eb7 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Fri, 25 May 2012 13:03:57 +0200 Subject: [PATCH 1/5] Move LeaderElectionSpec to multi-jvm. See #2113 --- .../scala/akka/cluster/LeaderElectionSpec.scala | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename akka-cluster/src/{test => multi-jvm}/scala/akka/cluster/LeaderElectionSpec.scala (100%) diff --git a/akka-cluster/src/test/scala/akka/cluster/LeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala similarity index 100% rename from akka-cluster/src/test/scala/akka/cluster/LeaderElectionSpec.scala rename to akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala From 597271052f9c650ec8c7df5cc8318ccfd0be4018 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Fri, 25 May 2012 14:48:00 +0200 Subject: [PATCH 2/5] Port LeaderElectionSpec to MultiNodeSpec. See #2113 --- .../akka/cluster/LeaderElectionSpec.scala | 212 ++++++++---------- .../akka/cluster/MultiNodeClusterSpec.scala | 20 +- 2 files changed, 110 insertions(+), 122 deletions(-) diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala index c262fad8c3..56cfbee75d 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala @@ -4,128 +4,100 @@ package akka.cluster +import com.typesafe.config.ConfigFactory +import akka.remote.testkit.MultiNodeConfig +import akka.remote.testkit.MultiNodeSpec import akka.testkit._ -import akka.dispatch._ -import akka.actor._ -import akka.remote._ -import akka.util.duration._ -import com.typesafe.config._ +object LeaderElectionMultiJvmSpec extends MultiNodeConfig { + val first = role("first") + val second = role("second") + val third = role("third") + val forth = role("forth") -import java.net.InetSocketAddress + commonConfig(debugConfig(on = false). + withFallback(ConfigFactory.parseString(""" + akka.cluster.auto-down = off + """)). + withFallback(MultiNodeClusterSpec.clusterConfig)) + +} + +class LeaderElectionMultiJvmNode1 extends LeaderElectionSpec +class LeaderElectionMultiJvmNode2 extends LeaderElectionSpec +class LeaderElectionMultiJvmNode3 extends LeaderElectionSpec +class LeaderElectionMultiJvmNode4 extends LeaderElectionSpec + +abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSpec) with MultiNodeClusterSpec { + import LeaderElectionMultiJvmSpec._ + + override def initialParticipants = 4 + + val firstAddress = node(first).address + val myAddress = node(mySelf).address + + // sorted in the order used by the cluster + val roles = Seq(first, second, third, forth).sorted + + "A cluster of three nodes" must { + + "be able to 'elect' a single leader" in { + // make sure that the first cluster is started before other join + runOn(first) { + cluster + } + testConductor.enter("first-started") + + cluster.join(firstAddress) + awaitUpConvergence(numberOfMembers = 4) + cluster.isLeader must be(mySelf == roles.head) + testConductor.enter("after") + } + + def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = { + val currentRoles = roles.drop(alreadyShutdown) + currentRoles.size must be >= (2) + + runOn(currentRoles.head) { + cluster.shutdown() + testConductor.enter("after-shutdown") + testConductor.enter("after-down") + } + + // runOn previously shutdown cluster nodes + if ((roles diff currentRoles).contains(mySelf)) { + testConductor.enter("after-shutdown") + testConductor.enter("after-down") + } + + // runOn remaining cluster nodes + if (currentRoles.tail.contains(mySelf)) { + + testConductor.enter("after-shutdown") + + runOn(currentRoles.last) { + // user marks the shutdown leader as DOWN + val leaderAddress = node(currentRoles.head).address + cluster.down(leaderAddress) + } + + testConductor.enter("after-down") + + awaitUpConvergence(currentRoles.size - 1) + val nextExpectedLeader = currentRoles.tail.head + cluster.isLeader must be(mySelf == nextExpectedLeader) + } + + testConductor.enter("after") + } + + "be able to 're-elect' a single leader after leader has left" in { + shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0) + } + + "be able to 're-elect' a single leader after leader has left (again)" in { + shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1) + } + } -class LeaderElectionSpec extends ClusterSpec with ImplicitSender { - val portPrefix = 5 - - var node1: Cluster = _ - var node2: Cluster = _ - var node3: Cluster = _ - - var system1: ActorSystemImpl = _ - var system2: ActorSystemImpl = _ - var system3: ActorSystemImpl = _ - - try { - "A cluster of three nodes" must { - - // ======= NODE 1 ======== - system1 = ActorSystem("system1", ConfigFactory - .parseString(""" - akka { - actor.provider = "akka.remote.RemoteActorRefProvider" - remote.netty.port = %d550 - }""".format(portPrefix)) - .withFallback(system.settings.config)) - .asInstanceOf[ActorSystemImpl] - node1 = Cluster(system1) - val address1 = node1.remoteAddress - - // ======= NODE 2 ======== - system2 = ActorSystem("system2", ConfigFactory - .parseString(""" - akka { - actor.provider = "akka.remote.RemoteActorRefProvider" - remote.netty.port = %d551 - cluster.node-to-join = "akka://system1@localhost:%d550" - }""".format(portPrefix, portPrefix)) - .withFallback(system.settings.config)) - .asInstanceOf[ActorSystemImpl] - node2 = Cluster(system2) - val address2 = node2.remoteAddress - - // ======= NODE 3 ======== - system3 = ActorSystem("system3", ConfigFactory - .parseString(""" - akka { - actor.provider = "akka.remote.RemoteActorRefProvider" - remote.netty.port = %d552 - cluster.node-to-join = "akka://system1@localhost:%d550" - }""".format(portPrefix, portPrefix)) - .withFallback(system.settings.config)) - .asInstanceOf[ActorSystemImpl] - node3 = Cluster(system3) - val address3 = node3.remoteAddress - - "be able to 'elect' a single leader" taggedAs LongRunningTest in { - - println("Give the system time to converge...") - awaitConvergence(node1 :: node2 :: node3 :: Nil) - - // check leader - node1.isLeader must be(true) - node2.isLeader must be(false) - node3.isLeader must be(false) - } - - "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in { - - // shut down system1 - the leader - node1.shutdown() - system1.shutdown() - - // user marks node1 as DOWN - node2.down(address1) - - println("Give the system time to converge...") - Thread.sleep(10.seconds.dilated.toMillis) - awaitConvergence(node2 :: node3 :: Nil) - - // check leader - node2.isLeader must be(true) - node3.isLeader must be(false) - } - - "be able to 're-elect' a single leader after leader has left (again, leaving a single node)" taggedAs LongRunningTest in { - - // shut down system1 - the leader - node2.shutdown() - system2.shutdown() - - // user marks node2 as DOWN - node3.down(address2) - - println("Give the system time to converge...") - Thread.sleep(10.seconds.dilated.toMillis) - awaitConvergence(node3 :: Nil) - - // check leader - node3.isLeader must be(true) - } - } - } catch { - case e: Exception ⇒ - e.printStackTrace - fail(e.toString) - } - - override def atTermination() { - if (node1 ne null) node1.shutdown() - if (system1 ne null) system1.shutdown() - - if (node2 ne null) node2.shutdown() - if (system2 ne null) system2.shutdown() - - if (node3 ne null) node3.shutdown() - if (system3 ne null) system3.shutdown() - } } diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala index 873d819dbb..48f1d0b520 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala @@ -45,8 +45,7 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒ */ def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(mySelf)) { nodesInCluster.length must not be (0) - import Member.addressOrdering - val expectedLeader = nodesInCluster.map(role ⇒ (role, node(role).address)).sortBy(_._2).head._1 + val expectedLeader = roleOfLeader(nodesInCluster) cluster.isLeader must be(ifNode(expectedLeader)(true)(false)) } @@ -60,4 +59,21 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒ awaitCond(cluster.convergence.isDefined, 10 seconds) } + def roleOfLeader(nodesInCluster: Seq[RoleName]): RoleName = { + nodesInCluster.length must not be (0) + nodesInCluster.sorted.head + } + + /** + * Sort the roles in the order used by the cluster. + */ + implicit val clusterOrdering: Ordering[RoleName] = new Ordering[RoleName] { + import Member.addressOrdering + def compare(x: RoleName, y: RoleName) = addressOrdering.compare(node(x).address, node(y).address) + } + + def roleName(address: Address): Option[RoleName] = { + testConductor.getNodes.await.find(node(_).address == address) + } + } \ No newline at end of file From a44bd10fc33e1ce4284c9d0cec79a7131466d71c Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Sun, 27 May 2012 19:15:31 +0200 Subject: [PATCH 3/5] Tag as LongRunningTest. See 2113 --- .../scala/akka/cluster/LeaderElectionSpec.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala index 56cfbee75d..007ab941dc 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala @@ -33,15 +33,14 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp override def initialParticipants = 4 - val firstAddress = node(first).address - val myAddress = node(mySelf).address + lazy val firstAddress = node(first).address // sorted in the order used by the cluster - val roles = Seq(first, second, third, forth).sorted + lazy val roles = Seq(first, second, third, forth).sorted - "A cluster of three nodes" must { + "A cluster of four nodes" must { - "be able to 'elect' a single leader" in { + "be able to 'elect' a single leader" taggedAs LongRunningTest in { // make sure that the first cluster is started before other join runOn(first) { cluster @@ -91,11 +90,11 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp testConductor.enter("after") } - "be able to 're-elect' a single leader after leader has left" in { + "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in { shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0) } - "be able to 're-elect' a single leader after leader has left (again)" in { + "be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in { shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1) } } From e3eec7e344c26cc912add339e611f5a8786029e9 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Mon, 28 May 2012 11:05:02 +0200 Subject: [PATCH 4/5] LeaderElectionSpec with hard exits. See #2113 and #2138 --- .../src/main/scala/akka/actor/Props.scala | 2 +- .../cluster/HardExitLeaderElectionSpec.scala | 108 ++++++++++++++++++ .../akka/cluster/LeaderElectionSpec.scala | 8 +- 3 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala diff --git a/akka-actor/src/main/scala/akka/actor/Props.scala b/akka-actor/src/main/scala/akka/actor/Props.scala index f6552179c3..dfd6200fd3 100644 --- a/akka-actor/src/main/scala/akka/actor/Props.scala +++ b/akka-actor/src/main/scala/akka/actor/Props.scala @@ -146,7 +146,7 @@ case class Props( /** * Returns a new Props with the specified creator set. - * + * * The creator must not return the same instance multiple times. * * Scala API. diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala new file mode 100644 index 0000000000..0360e4f1b8 --- /dev/null +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2009-2012 Typesafe Inc. + */ + +package akka.cluster + +import com.typesafe.config.ConfigFactory +import akka.remote.testkit.MultiNodeConfig +import akka.remote.testkit.MultiNodeSpec +import akka.testkit._ + +object HardExitLeaderElectionMultiJvmSpec extends MultiNodeConfig { + val controller = role("controller") + val first = role("first") + val second = role("second") + val third = role("third") + val fourth = role("fourth") + + commonConfig(debugConfig(on = false). + withFallback(ConfigFactory.parseString(""" + akka.cluster.auto-down = off + """)). + withFallback(MultiNodeClusterSpec.clusterConfig)) + +} + +class HardExitLeaderElectionMultiJvmNode1 extends HardExitLeaderElectionSpec +class HardExitLeaderElectionMultiJvmNode2 extends HardExitLeaderElectionSpec +class HardExitLeaderElectionMultiJvmNode3 extends HardExitLeaderElectionSpec +class HardExitLeaderElectionMultiJvmNode4 extends HardExitLeaderElectionSpec +class HardExitLeaderElectionMultiJvmNode5 extends HardExitLeaderElectionSpec + +abstract class HardExitLeaderElectionSpec extends MultiNodeSpec(HardExitLeaderElectionMultiJvmSpec) with MultiNodeClusterSpec { + import HardExitLeaderElectionMultiJvmSpec._ + + override def initialParticipants = 5 + + lazy val firstAddress = node(first).address + + // sorted in the order used by the cluster + lazy val roles = Seq(first, second, third, fourth).sorted + + "A cluster of four nodes" must { + + "be able to 'elect' a single leader" taggedAs LongRunningTest in { + // make sure that the node-to-join is started before other join + runOn(first) { + cluster + } + testConductor.enter("first-started") + + if (mySelf != controller) { + cluster.join(firstAddress) + awaitUpConvergence(numberOfMembers = roles.size) + cluster.isLeader must be(mySelf == roles.head) + } + testConductor.enter("after") + } + + def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = { + val currentRoles = roles.drop(alreadyShutdown) + currentRoles.size must be >= (2) + val leader = currentRoles.head + val aUser = currentRoles.last + + mySelf match { + + case `controller` ⇒ + testConductor.enter("before-shutdown") + testConductor.shutdown(leader, 0) + testConductor.removeNode(leader) + testConductor.enter("after-shutdown", "after-down", "completed") + + case `leader` ⇒ + testConductor.enter("before-shutdown") + // this node will be shutdown by the controller and doesn't participate in more barriers + + case `aUser` ⇒ + val leaderAddress = node(leader).address + testConductor.enter("before-shutdown", "after-shutdown") + // user marks the shutdown leader as DOWN + cluster.down(leaderAddress) + testConductor.enter("after-down", "completed") + + case _ if currentRoles.tail.contains(mySelf) ⇒ + // remaining cluster nodes, not shutdown + testConductor.enter("before-shutdown", "after-shutdown", "after-down") + + awaitUpConvergence(currentRoles.size - 1) + val nextExpectedLeader = currentRoles.tail.head + cluster.isLeader must be(mySelf == nextExpectedLeader) + + testConductor.enter("completed") + + } + + } + + "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in { + shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0) + } + + "be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in { + shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1) + } + } + +} diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala index 007ab941dc..886556de54 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala @@ -13,7 +13,7 @@ object LeaderElectionMultiJvmSpec extends MultiNodeConfig { val first = role("first") val second = role("second") val third = role("third") - val forth = role("forth") + val fourth = role("fourth") commonConfig(debugConfig(on = false). withFallback(ConfigFactory.parseString(""" @@ -36,19 +36,19 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp lazy val firstAddress = node(first).address // sorted in the order used by the cluster - lazy val roles = Seq(first, second, third, forth).sorted + lazy val roles = Seq(first, second, third, fourth).sorted "A cluster of four nodes" must { "be able to 'elect' a single leader" taggedAs LongRunningTest in { - // make sure that the first cluster is started before other join + // make sure that the node-to-join is started before other join runOn(first) { cluster } testConductor.enter("first-started") cluster.join(firstAddress) - awaitUpConvergence(numberOfMembers = 4) + awaitUpConvergence(numberOfMembers = roles.size) cluster.isLeader must be(mySelf == roles.head) testConductor.enter("after") } From 59dd754819764327f4cb1e5c4aad7d04cc3425f1 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Mon, 28 May 2012 13:55:22 +0200 Subject: [PATCH 5/5] Use only the hard exit LeaderElectionSpec, see #2113 --- .../cluster/HardExitLeaderElectionSpec.scala | 108 ------------------ .../akka/cluster/LeaderElectionSpec.scala | 58 +++++----- 2 files changed, 32 insertions(+), 134 deletions(-) delete mode 100644 akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala deleted file mode 100644 index 0360e4f1b8..0000000000 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/HardExitLeaderElectionSpec.scala +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Copyright (C) 2009-2012 Typesafe Inc. - */ - -package akka.cluster - -import com.typesafe.config.ConfigFactory -import akka.remote.testkit.MultiNodeConfig -import akka.remote.testkit.MultiNodeSpec -import akka.testkit._ - -object HardExitLeaderElectionMultiJvmSpec extends MultiNodeConfig { - val controller = role("controller") - val first = role("first") - val second = role("second") - val third = role("third") - val fourth = role("fourth") - - commonConfig(debugConfig(on = false). - withFallback(ConfigFactory.parseString(""" - akka.cluster.auto-down = off - """)). - withFallback(MultiNodeClusterSpec.clusterConfig)) - -} - -class HardExitLeaderElectionMultiJvmNode1 extends HardExitLeaderElectionSpec -class HardExitLeaderElectionMultiJvmNode2 extends HardExitLeaderElectionSpec -class HardExitLeaderElectionMultiJvmNode3 extends HardExitLeaderElectionSpec -class HardExitLeaderElectionMultiJvmNode4 extends HardExitLeaderElectionSpec -class HardExitLeaderElectionMultiJvmNode5 extends HardExitLeaderElectionSpec - -abstract class HardExitLeaderElectionSpec extends MultiNodeSpec(HardExitLeaderElectionMultiJvmSpec) with MultiNodeClusterSpec { - import HardExitLeaderElectionMultiJvmSpec._ - - override def initialParticipants = 5 - - lazy val firstAddress = node(first).address - - // sorted in the order used by the cluster - lazy val roles = Seq(first, second, third, fourth).sorted - - "A cluster of four nodes" must { - - "be able to 'elect' a single leader" taggedAs LongRunningTest in { - // make sure that the node-to-join is started before other join - runOn(first) { - cluster - } - testConductor.enter("first-started") - - if (mySelf != controller) { - cluster.join(firstAddress) - awaitUpConvergence(numberOfMembers = roles.size) - cluster.isLeader must be(mySelf == roles.head) - } - testConductor.enter("after") - } - - def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = { - val currentRoles = roles.drop(alreadyShutdown) - currentRoles.size must be >= (2) - val leader = currentRoles.head - val aUser = currentRoles.last - - mySelf match { - - case `controller` ⇒ - testConductor.enter("before-shutdown") - testConductor.shutdown(leader, 0) - testConductor.removeNode(leader) - testConductor.enter("after-shutdown", "after-down", "completed") - - case `leader` ⇒ - testConductor.enter("before-shutdown") - // this node will be shutdown by the controller and doesn't participate in more barriers - - case `aUser` ⇒ - val leaderAddress = node(leader).address - testConductor.enter("before-shutdown", "after-shutdown") - // user marks the shutdown leader as DOWN - cluster.down(leaderAddress) - testConductor.enter("after-down", "completed") - - case _ if currentRoles.tail.contains(mySelf) ⇒ - // remaining cluster nodes, not shutdown - testConductor.enter("before-shutdown", "after-shutdown", "after-down") - - awaitUpConvergence(currentRoles.size - 1) - val nextExpectedLeader = currentRoles.tail.head - cluster.isLeader must be(mySelf == nextExpectedLeader) - - testConductor.enter("completed") - - } - - } - - "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in { - shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0) - } - - "be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in { - shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1) - } - } - -} diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala index 886556de54..54f744a6c8 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala @@ -10,6 +10,7 @@ import akka.remote.testkit.MultiNodeSpec import akka.testkit._ object LeaderElectionMultiJvmSpec extends MultiNodeConfig { + val controller = role("controller") val first = role("first") val second = role("second") val third = role("third") @@ -27,11 +28,12 @@ class LeaderElectionMultiJvmNode1 extends LeaderElectionSpec class LeaderElectionMultiJvmNode2 extends LeaderElectionSpec class LeaderElectionMultiJvmNode3 extends LeaderElectionSpec class LeaderElectionMultiJvmNode4 extends LeaderElectionSpec +class LeaderElectionMultiJvmNode5 extends LeaderElectionSpec abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSpec) with MultiNodeClusterSpec { import LeaderElectionMultiJvmSpec._ - override def initialParticipants = 4 + override def initialParticipants = 5 lazy val firstAddress = node(first).address @@ -47,47 +49,51 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp } testConductor.enter("first-started") - cluster.join(firstAddress) - awaitUpConvergence(numberOfMembers = roles.size) - cluster.isLeader must be(mySelf == roles.head) + if (mySelf != controller) { + cluster.join(firstAddress) + awaitUpConvergence(numberOfMembers = roles.size) + cluster.isLeader must be(mySelf == roles.head) + } testConductor.enter("after") } def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = { val currentRoles = roles.drop(alreadyShutdown) currentRoles.size must be >= (2) + val leader = currentRoles.head + val aUser = currentRoles.last - runOn(currentRoles.head) { - cluster.shutdown() - testConductor.enter("after-shutdown") - testConductor.enter("after-down") - } + mySelf match { - // runOn previously shutdown cluster nodes - if ((roles diff currentRoles).contains(mySelf)) { - testConductor.enter("after-shutdown") - testConductor.enter("after-down") - } + case `controller` ⇒ + testConductor.enter("before-shutdown") + testConductor.shutdown(leader, 0) + testConductor.removeNode(leader) + testConductor.enter("after-shutdown", "after-down", "completed") - // runOn remaining cluster nodes - if (currentRoles.tail.contains(mySelf)) { + case `leader` ⇒ + testConductor.enter("before-shutdown") + // this node will be shutdown by the controller and doesn't participate in more barriers - testConductor.enter("after-shutdown") - - runOn(currentRoles.last) { + case `aUser` ⇒ + val leaderAddress = node(leader).address + testConductor.enter("before-shutdown", "after-shutdown") // user marks the shutdown leader as DOWN - val leaderAddress = node(currentRoles.head).address cluster.down(leaderAddress) - } + testConductor.enter("after-down", "completed") - testConductor.enter("after-down") + case _ if currentRoles.tail.contains(mySelf) ⇒ + // remaining cluster nodes, not shutdown + testConductor.enter("before-shutdown", "after-shutdown", "after-down") + + awaitUpConvergence(currentRoles.size - 1) + val nextExpectedLeader = currentRoles.tail.head + cluster.isLeader must be(mySelf == nextExpectedLeader) + + testConductor.enter("completed") - awaitUpConvergence(currentRoles.size - 1) - val nextExpectedLeader = currentRoles.tail.head - cluster.isLeader must be(mySelf == nextExpectedLeader) } - testConductor.enter("after") } "be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in {