2012-02-20 17:22:07 +01:00
|
|
|
/**
|
|
|
|
|
* Copyright (C) 2009-2012 Typesafe Inc. <http://www.typesafe.com>
|
|
|
|
|
*/
|
2012-02-28 17:04:48 +01:00
|
|
|
|
2012-02-20 17:22:07 +01:00
|
|
|
package akka.cluster
|
|
|
|
|
|
2012-05-25 14:48:00 +02:00
|
|
|
import com.typesafe.config.ConfigFactory
|
|
|
|
|
import akka.remote.testkit.MultiNodeConfig
|
|
|
|
|
import akka.remote.testkit.MultiNodeSpec
|
2012-02-20 17:22:07 +01:00
|
|
|
import akka.testkit._
|
|
|
|
|
|
2012-09-06 21:48:40 +02:00
|
|
|
case class LeaderElectionMultiNodeConfig(failureDetectorPuppet: Boolean) extends MultiNodeConfig {
|
2012-05-28 13:55:22 +02:00
|
|
|
val controller = role("controller")
|
2012-05-25 14:48:00 +02:00
|
|
|
val first = role("first")
|
|
|
|
|
val second = role("second")
|
|
|
|
|
val third = role("third")
|
2012-05-28 11:05:02 +02:00
|
|
|
val fourth = role("fourth")
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-10-10 16:45:58 +02:00
|
|
|
commonConfig(debugConfig(on = true).withFallback(MultiNodeClusterSpec.clusterConfig(failureDetectorPuppet)))
|
2012-05-25 14:48:00 +02:00
|
|
|
}
|
2012-03-09 12:56:56 +01:00
|
|
|
|
2012-09-06 21:48:40 +02:00
|
|
|
class LeaderElectionWithFailureDetectorPuppetMultiJvmNode1 extends LeaderElectionSpec(failureDetectorPuppet = true)
|
|
|
|
|
class LeaderElectionWithFailureDetectorPuppetMultiJvmNode2 extends LeaderElectionSpec(failureDetectorPuppet = true)
|
|
|
|
|
class LeaderElectionWithFailureDetectorPuppetMultiJvmNode3 extends LeaderElectionSpec(failureDetectorPuppet = true)
|
|
|
|
|
class LeaderElectionWithFailureDetectorPuppetMultiJvmNode4 extends LeaderElectionSpec(failureDetectorPuppet = true)
|
|
|
|
|
class LeaderElectionWithFailureDetectorPuppetMultiJvmNode5 extends LeaderElectionSpec(failureDetectorPuppet = true)
|
2012-06-11 16:48:19 +02:00
|
|
|
|
2012-09-06 21:48:40 +02:00
|
|
|
class LeaderElectionWithAccrualFailureDetectorMultiJvmNode1 extends LeaderElectionSpec(failureDetectorPuppet = false)
|
|
|
|
|
class LeaderElectionWithAccrualFailureDetectorMultiJvmNode2 extends LeaderElectionSpec(failureDetectorPuppet = false)
|
|
|
|
|
class LeaderElectionWithAccrualFailureDetectorMultiJvmNode3 extends LeaderElectionSpec(failureDetectorPuppet = false)
|
|
|
|
|
class LeaderElectionWithAccrualFailureDetectorMultiJvmNode4 extends LeaderElectionSpec(failureDetectorPuppet = false)
|
|
|
|
|
class LeaderElectionWithAccrualFailureDetectorMultiJvmNode5 extends LeaderElectionSpec(failureDetectorPuppet = false)
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-09-06 21:48:40 +02:00
|
|
|
abstract class LeaderElectionSpec(multiNodeConfig: LeaderElectionMultiNodeConfig)
|
|
|
|
|
extends MultiNodeSpec(multiNodeConfig)
|
2012-06-04 23:21:28 +02:00
|
|
|
with MultiNodeClusterSpec {
|
|
|
|
|
|
2012-09-06 21:48:40 +02:00
|
|
|
def this(failureDetectorPuppet: Boolean) = this(LeaderElectionMultiNodeConfig(failureDetectorPuppet))
|
|
|
|
|
|
|
|
|
|
import multiNodeConfig._
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-05-25 14:48:00 +02:00
|
|
|
// sorted in the order used by the cluster
|
2012-06-08 09:23:36 +02:00
|
|
|
lazy val sortedRoles = Seq(first, second, third, fourth).sorted
|
2012-03-09 12:56:56 +01:00
|
|
|
|
2012-05-27 19:15:31 +02:00
|
|
|
"A cluster of four nodes" must {
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-05-27 19:15:31 +02:00
|
|
|
"be able to 'elect' a single leader" taggedAs LongRunningTest in {
|
2012-06-05 14:13:44 +02:00
|
|
|
awaitClusterUp(first, second, third, fourth)
|
2012-05-25 14:48:00 +02:00
|
|
|
|
2012-05-28 15:29:00 +02:00
|
|
|
if (myself != controller) {
|
2012-08-16 18:28:01 +02:00
|
|
|
clusterView.isLeader must be(myself == sortedRoles.head)
|
2012-06-08 09:23:36 +02:00
|
|
|
assertLeaderIn(sortedRoles)
|
2012-05-28 13:55:22 +02:00
|
|
|
}
|
2012-06-05 14:13:44 +02:00
|
|
|
|
2012-06-20 13:30:30 +02:00
|
|
|
enterBarrier("after-1")
|
2012-02-20 17:22:07 +01:00
|
|
|
}
|
|
|
|
|
|
2012-05-25 14:48:00 +02:00
|
|
|
def shutdownLeaderAndVerifyNewLeader(alreadyShutdown: Int): Unit = {
|
2012-06-08 09:23:36 +02:00
|
|
|
val currentRoles = sortedRoles.drop(alreadyShutdown)
|
2012-05-25 14:48:00 +02:00
|
|
|
currentRoles.size must be >= (2)
|
2012-05-28 13:55:22 +02:00
|
|
|
val leader = currentRoles.head
|
|
|
|
|
val aUser = currentRoles.last
|
2012-06-01 11:37:44 +02:00
|
|
|
val remainingRoles = currentRoles.tail
|
2012-10-10 16:45:58 +02:00
|
|
|
val n = "-" + (alreadyShutdown + 1)
|
2012-05-25 14:48:00 +02:00
|
|
|
|
2012-05-28 15:29:00 +02:00
|
|
|
myself match {
|
2012-05-25 14:48:00 +02:00
|
|
|
|
2012-05-28 13:55:22 +02:00
|
|
|
case `controller` ⇒
|
2012-06-18 11:54:44 +02:00
|
|
|
val leaderAddress = address(leader)
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("before-shutdown" + n)
|
2012-05-28 13:55:22 +02:00
|
|
|
testConductor.shutdown(leader, 0)
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("after-shutdown" + n, "after-down" + n, "completed" + n)
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-05-28 13:55:22 +02:00
|
|
|
case `leader` ⇒
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("before-shutdown" + n, "after-shutdown" + n)
|
2012-05-28 13:55:22 +02:00
|
|
|
// this node will be shutdown by the controller and doesn't participate in more barriers
|
2012-02-20 17:22:07 +01:00
|
|
|
|
2012-05-28 13:55:22 +02:00
|
|
|
case `aUser` ⇒
|
2012-06-18 11:54:44 +02:00
|
|
|
val leaderAddress = address(leader)
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("before-shutdown" + n, "after-shutdown" + n)
|
2012-05-25 14:48:00 +02:00
|
|
|
// user marks the shutdown leader as DOWN
|
|
|
|
|
cluster.down(leaderAddress)
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("after-down" + n, "completed" + n)
|
2012-06-11 16:48:19 +02:00
|
|
|
markNodeAsUnavailable(leaderAddress)
|
2012-05-25 14:48:00 +02:00
|
|
|
|
2012-06-01 11:37:44 +02:00
|
|
|
case _ if remainingRoles.contains(myself) ⇒
|
2012-05-28 13:55:22 +02:00
|
|
|
// remaining cluster nodes, not shutdown
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("before-shutdown" + n, "after-shutdown" + n, "after-down" + n)
|
2012-05-28 13:55:22 +02:00
|
|
|
|
|
|
|
|
awaitUpConvergence(currentRoles.size - 1)
|
2012-06-01 11:37:44 +02:00
|
|
|
val nextExpectedLeader = remainingRoles.head
|
2012-08-16 18:28:01 +02:00
|
|
|
clusterView.isLeader must be(myself == nextExpectedLeader)
|
2012-06-01 11:37:44 +02:00
|
|
|
assertLeaderIn(remainingRoles)
|
2012-05-28 13:55:22 +02:00
|
|
|
|
2012-10-10 16:45:58 +02:00
|
|
|
enterBarrier("completed" + n)
|
2012-05-25 14:48:00 +02:00
|
|
|
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2012-05-27 19:15:31 +02:00
|
|
|
"be able to 're-elect' a single leader after leader has left" taggedAs LongRunningTest in {
|
2012-05-25 14:48:00 +02:00
|
|
|
shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 0)
|
2012-06-20 13:30:30 +02:00
|
|
|
enterBarrier("after-2")
|
2012-05-25 14:48:00 +02:00
|
|
|
}
|
|
|
|
|
|
2012-05-27 19:15:31 +02:00
|
|
|
"be able to 're-elect' a single leader after leader has left (again)" taggedAs LongRunningTest in {
|
2012-05-25 14:48:00 +02:00
|
|
|
shutdownLeaderAndVerifyNewLeader(alreadyShutdown = 1)
|
2012-06-20 13:30:30 +02:00
|
|
|
enterBarrier("after-3")
|
2012-05-25 14:48:00 +02:00
|
|
|
}
|
2012-02-20 17:22:07 +01:00
|
|
|
}
|
|
|
|
|
}
|