Remove address vals in tests, fix race in TransitionSpec, see #2201
This commit is contained in:
parent
fb04786072
commit
12e90a98dc
19 changed files with 53 additions and 94 deletions
|
|
@ -37,7 +37,7 @@ abstract class ClientDowningNodeThatIsUnreachableSpec
|
||||||
"Client of a 4 node cluster" must {
|
"Client of a 4 node cluster" must {
|
||||||
|
|
||||||
"be able to DOWN a node that is UNREACHABLE (killed)" taggedAs LongRunningTest in {
|
"be able to DOWN a node that is UNREACHABLE (killed)" taggedAs LongRunningTest in {
|
||||||
val thirdAddress = node(third).address
|
val thirdAddress = address(third)
|
||||||
awaitClusterUp(first, second, third, fourth)
|
awaitClusterUp(first, second, third, fourth)
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ abstract class ClientDowningNodeThatIsUpSpec
|
||||||
"Client of a 4 node cluster" must {
|
"Client of a 4 node cluster" must {
|
||||||
|
|
||||||
"be able to DOWN a node that is UP (healthy and available)" taggedAs LongRunningTest in {
|
"be able to DOWN a node that is UP (healthy and available)" taggedAs LongRunningTest in {
|
||||||
val thirdAddress = node(third).address
|
val thirdAddress = address(third)
|
||||||
awaitClusterUp(first, second, third, fourth)
|
awaitClusterUp(first, second, third, fourth)
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,7 @@ abstract class ConvergenceSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
"not reach convergence while any nodes are unreachable" taggedAs LongRunningTest in {
|
"not reach convergence while any nodes are unreachable" taggedAs LongRunningTest in {
|
||||||
val thirdAddress = node(third).address
|
val thirdAddress = address(third)
|
||||||
testConductor.enter("before-shutdown")
|
testConductor.enter("before-shutdown")
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
|
|
@ -60,15 +60,13 @@ abstract class ConvergenceSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
runOn(first, second) {
|
runOn(first, second) {
|
||||||
val firstAddress = node(first).address
|
|
||||||
val secondAddress = node(second).address
|
|
||||||
|
|
||||||
within(28 seconds) {
|
within(28 seconds) {
|
||||||
// third becomes unreachable
|
// third becomes unreachable
|
||||||
awaitCond(cluster.latestGossip.overview.unreachable.size == 1)
|
awaitCond(cluster.latestGossip.overview.unreachable.size == 1)
|
||||||
awaitCond(cluster.latestGossip.members.size == 2)
|
awaitCond(cluster.latestGossip.members.size == 2)
|
||||||
awaitCond(cluster.latestGossip.members.forall(_.status == MemberStatus.Up))
|
awaitCond(cluster.latestGossip.members.forall(_.status == MemberStatus.Up))
|
||||||
awaitSeenSameState(Seq(firstAddress, secondAddress))
|
awaitSeenSameState(first, second)
|
||||||
// still one unreachable
|
// still one unreachable
|
||||||
cluster.latestGossip.overview.unreachable.size must be(1)
|
cluster.latestGossip.overview.unreachable.size must be(1)
|
||||||
cluster.latestGossip.overview.unreachable.head.address must be(thirdAddress)
|
cluster.latestGossip.overview.unreachable.head.address must be(thirdAddress)
|
||||||
|
|
@ -84,24 +82,20 @@ abstract class ConvergenceSpec
|
||||||
"not move a new joining node to Up while there is no convergence" taggedAs LongRunningTest in {
|
"not move a new joining node to Up while there is no convergence" taggedAs LongRunningTest in {
|
||||||
runOn(fourth) {
|
runOn(fourth) {
|
||||||
// try to join
|
// try to join
|
||||||
cluster.join(node(first).address)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
|
|
||||||
val firstAddress = node(first).address
|
|
||||||
val secondAddress = node(second).address
|
|
||||||
val fourthAddress = node(fourth).address
|
|
||||||
|
|
||||||
def memberStatus(address: Address): Option[MemberStatus] =
|
def memberStatus(address: Address): Option[MemberStatus] =
|
||||||
cluster.latestGossip.members.collectFirst { case m if m.address == address ⇒ m.status }
|
cluster.latestGossip.members.collectFirst { case m if m.address == address ⇒ m.status }
|
||||||
|
|
||||||
def assertNotMovedUp: Unit = {
|
def assertNotMovedUp: Unit = {
|
||||||
within(20 seconds) {
|
within(20 seconds) {
|
||||||
awaitCond(cluster.latestGossip.members.size == 3)
|
awaitCond(cluster.latestGossip.members.size == 3)
|
||||||
awaitSeenSameState(Seq(firstAddress, secondAddress, fourthAddress))
|
awaitSeenSameState(first, second, fourth)
|
||||||
memberStatus(firstAddress) must be(Some(MemberStatus.Up))
|
memberStatus(first) must be(Some(MemberStatus.Up))
|
||||||
memberStatus(secondAddress) must be(Some(MemberStatus.Up))
|
memberStatus(second) must be(Some(MemberStatus.Up))
|
||||||
// leader is not allowed to move the new node to Up
|
// leader is not allowed to move the new node to Up
|
||||||
memberStatus(fourthAddress) must be(Some(MemberStatus.Joining))
|
memberStatus(fourth) must be(Some(MemberStatus.Joining))
|
||||||
// still no convergence
|
// still no convergence
|
||||||
cluster.convergence.isDefined must be(false)
|
cluster.convergence.isDefined must be(false)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,10 +33,6 @@ abstract class JoinTwoClustersSpec
|
||||||
|
|
||||||
import JoinTwoClustersMultiJvmSpec._
|
import JoinTwoClustersMultiJvmSpec._
|
||||||
|
|
||||||
lazy val a1Address = node(a1).address
|
|
||||||
lazy val b1Address = node(b1).address
|
|
||||||
lazy val c1Address = node(c1).address
|
|
||||||
|
|
||||||
"Three different clusters (A, B and C)" must {
|
"Three different clusters (A, B and C)" must {
|
||||||
|
|
||||||
"be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
|
"be able to 'elect' a single leader after joining (A -> B)" taggedAs LongRunningTest in {
|
||||||
|
|
@ -47,13 +43,13 @@ abstract class JoinTwoClustersSpec
|
||||||
testConductor.enter("first-started")
|
testConductor.enter("first-started")
|
||||||
|
|
||||||
runOn(a1, a2) {
|
runOn(a1, a2) {
|
||||||
cluster.join(a1Address)
|
cluster.join(a1)
|
||||||
}
|
}
|
||||||
runOn(b1, b2) {
|
runOn(b1, b2) {
|
||||||
cluster.join(b1Address)
|
cluster.join(b1)
|
||||||
}
|
}
|
||||||
runOn(c1, c2) {
|
runOn(c1, c2) {
|
||||||
cluster.join(c1Address)
|
cluster.join(c1)
|
||||||
}
|
}
|
||||||
|
|
||||||
awaitUpConvergence(numberOfMembers = 2)
|
awaitUpConvergence(numberOfMembers = 2)
|
||||||
|
|
@ -65,7 +61,7 @@ abstract class JoinTwoClustersSpec
|
||||||
testConductor.enter("two-members")
|
testConductor.enter("two-members")
|
||||||
|
|
||||||
runOn(b2) {
|
runOn(b2) {
|
||||||
cluster.join(a1Address)
|
cluster.join(a1)
|
||||||
}
|
}
|
||||||
|
|
||||||
runOn(a1, a2, b1, b2) {
|
runOn(a1, a2, b1, b2) {
|
||||||
|
|
@ -81,7 +77,7 @@ abstract class JoinTwoClustersSpec
|
||||||
"be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
|
"be able to 'elect' a single leader after joining (C -> A + B)" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
runOn(b2) {
|
runOn(b2) {
|
||||||
cluster.join(c1Address)
|
cluster.join(c1)
|
||||||
}
|
}
|
||||||
|
|
||||||
awaitUpConvergence(numberOfMembers = 6)
|
awaitUpConvergence(numberOfMembers = 6)
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ abstract class LeaderDowningNodeThatIsUnreachableSpec
|
||||||
"be able to DOWN a 'last' node that is UNREACHABLE" taggedAs LongRunningTest in {
|
"be able to DOWN a 'last' node that is UNREACHABLE" taggedAs LongRunningTest in {
|
||||||
awaitClusterUp(first, second, third, fourth)
|
awaitClusterUp(first, second, third, fourth)
|
||||||
|
|
||||||
val fourthAddress = node(fourth).address
|
val fourthAddress = address(fourth)
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
// kill 'fourth' node
|
// kill 'fourth' node
|
||||||
testConductor.shutdown(fourth, 0)
|
testConductor.shutdown(fourth, 0)
|
||||||
|
|
@ -70,7 +70,7 @@ abstract class LeaderDowningNodeThatIsUnreachableSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
"be able to DOWN a 'middle' node that is UNREACHABLE" taggedAs LongRunningTest in {
|
"be able to DOWN a 'middle' node that is UNREACHABLE" taggedAs LongRunningTest in {
|
||||||
val secondAddress = node(second).address
|
val secondAddress = address(second)
|
||||||
|
|
||||||
testConductor.enter("before-down-second-node")
|
testConductor.enter("before-down-second-node")
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@ abstract class LeaderElectionSpec
|
||||||
myself match {
|
myself match {
|
||||||
|
|
||||||
case `controller` ⇒
|
case `controller` ⇒
|
||||||
val leaderAddress = node(leader).address
|
val leaderAddress = address(leader)
|
||||||
testConductor.enter("before-shutdown")
|
testConductor.enter("before-shutdown")
|
||||||
testConductor.shutdown(leader, 0)
|
testConductor.shutdown(leader, 0)
|
||||||
testConductor.enter("after-shutdown", "after-down", "completed")
|
testConductor.enter("after-shutdown", "after-down", "completed")
|
||||||
|
|
@ -74,7 +74,7 @@ abstract class LeaderElectionSpec
|
||||||
// this node will be shutdown by the controller and doesn't participate in more barriers
|
// this node will be shutdown by the controller and doesn't participate in more barriers
|
||||||
|
|
||||||
case `aUser` ⇒
|
case `aUser` ⇒
|
||||||
val leaderAddress = node(leader).address
|
val leaderAddress = address(leader)
|
||||||
testConductor.enter("before-shutdown", "after-shutdown")
|
testConductor.enter("before-shutdown", "after-shutdown")
|
||||||
// user marks the shutdown leader as DOWN
|
// user marks the shutdown leader as DOWN
|
||||||
cluster.down(leaderAddress)
|
cluster.down(leaderAddress)
|
||||||
|
|
|
||||||
|
|
@ -37,10 +37,6 @@ abstract class MembershipChangeListenerExitingSpec
|
||||||
|
|
||||||
import MembershipChangeListenerExitingMultiJvmSpec._
|
import MembershipChangeListenerExitingMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A registered MembershipChangeListener" must {
|
"A registered MembershipChangeListener" must {
|
||||||
"be notified when new node is EXITING" taggedAs LongRunningTest in {
|
"be notified when new node is EXITING" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
|
|
@ -48,7 +44,7 @@ abstract class MembershipChangeListenerExitingSpec
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
testConductor.enter("registered-listener")
|
testConductor.enter("registered-listener")
|
||||||
cluster.leave(secondAddress)
|
cluster.leave(second)
|
||||||
}
|
}
|
||||||
|
|
||||||
runOn(second) {
|
runOn(second) {
|
||||||
|
|
@ -59,7 +55,7 @@ abstract class MembershipChangeListenerExitingSpec
|
||||||
val exitingLatch = TestLatch()
|
val exitingLatch = TestLatch()
|
||||||
cluster.registerListener(new MembershipChangeListener {
|
cluster.registerListener(new MembershipChangeListener {
|
||||||
def notify(members: SortedSet[Member]) {
|
def notify(members: SortedSet[Member]) {
|
||||||
if (members.size == 3 && members.exists(m ⇒ m.address == secondAddress && m.status == MemberStatus.Exiting))
|
if (members.size == 3 && members.exists(m ⇒ m.address == address(second) && m.status == MemberStatus.Exiting))
|
||||||
exitingLatch.countDown()
|
exitingLatch.countDown()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -30,15 +30,12 @@ abstract class MembershipChangeListenerJoinSpec
|
||||||
|
|
||||||
import MembershipChangeListenerJoinMultiJvmSpec._
|
import MembershipChangeListenerJoinMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
|
|
||||||
"A registered MembershipChangeListener" must {
|
"A registered MembershipChangeListener" must {
|
||||||
"be notified when new node is JOINING" taggedAs LongRunningTest in {
|
"be notified when new node is JOINING" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
val joinLatch = TestLatch()
|
val joinLatch = TestLatch()
|
||||||
val expectedAddresses = Set(firstAddress, secondAddress)
|
val expectedAddresses = Set(first, second) map address
|
||||||
cluster.registerListener(new MembershipChangeListener {
|
cluster.registerListener(new MembershipChangeListener {
|
||||||
def notify(members: SortedSet[Member]) {
|
def notify(members: SortedSet[Member]) {
|
||||||
if (members.map(_.address) == expectedAddresses && members.exists(_.status == MemberStatus.Joining))
|
if (members.map(_.address) == expectedAddresses && members.exists(_.status == MemberStatus.Joining))
|
||||||
|
|
@ -52,7 +49,7 @@ abstract class MembershipChangeListenerJoinSpec
|
||||||
|
|
||||||
runOn(second) {
|
runOn(second) {
|
||||||
testConductor.enter("registered-listener")
|
testConductor.enter("registered-listener")
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
|
|
||||||
awaitUpConvergence(2)
|
awaitUpConvergence(2)
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import com.typesafe.config.ConfigFactory
|
||||||
import akka.remote.testkit.MultiNodeConfig
|
import akka.remote.testkit.MultiNodeConfig
|
||||||
import akka.remote.testkit.MultiNodeSpec
|
import akka.remote.testkit.MultiNodeSpec
|
||||||
import akka.testkit._
|
import akka.testkit._
|
||||||
|
import akka.actor.Address
|
||||||
|
|
||||||
object MembershipChangeListenerLeavingMultiJvmSpec extends MultiNodeConfig {
|
object MembershipChangeListenerLeavingMultiJvmSpec extends MultiNodeConfig {
|
||||||
val first = role("first")
|
val first = role("first")
|
||||||
|
|
@ -34,10 +35,6 @@ abstract class MembershipChangeListenerLeavingSpec
|
||||||
|
|
||||||
import MembershipChangeListenerLeavingMultiJvmSpec._
|
import MembershipChangeListenerLeavingMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A registered MembershipChangeListener" must {
|
"A registered MembershipChangeListener" must {
|
||||||
"be notified when new node is LEAVING" taggedAs LongRunningTest in {
|
"be notified when new node is LEAVING" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
|
|
@ -45,7 +42,7 @@ abstract class MembershipChangeListenerLeavingSpec
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
testConductor.enter("registered-listener")
|
testConductor.enter("registered-listener")
|
||||||
cluster.leave(secondAddress)
|
cluster.leave(second)
|
||||||
}
|
}
|
||||||
|
|
||||||
runOn(second) {
|
runOn(second) {
|
||||||
|
|
@ -54,11 +51,11 @@ abstract class MembershipChangeListenerLeavingSpec
|
||||||
|
|
||||||
runOn(third) {
|
runOn(third) {
|
||||||
val latch = TestLatch()
|
val latch = TestLatch()
|
||||||
val expectedAddresses = Set(firstAddress, secondAddress, thirdAddress)
|
val expectedAddresses = Set(first, second, third) map address
|
||||||
cluster.registerListener(new MembershipChangeListener {
|
cluster.registerListener(new MembershipChangeListener {
|
||||||
def notify(members: SortedSet[Member]) {
|
def notify(members: SortedSet[Member]) {
|
||||||
if (members.map(_.address) == expectedAddresses &&
|
if (members.map(_.address) == expectedAddresses &&
|
||||||
members.exists(m ⇒ m.address == secondAddress && m.status == MemberStatus.Leaving))
|
members.exists(m ⇒ m.address == address(second) && m.status == MemberStatus.Leaving))
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -27,10 +27,6 @@ abstract class MembershipChangeListenerUpSpec
|
||||||
|
|
||||||
import MembershipChangeListenerUpMultiJvmSpec._
|
import MembershipChangeListenerUpMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A set of connected cluster systems" must {
|
"A set of connected cluster systems" must {
|
||||||
|
|
||||||
"(when two nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
|
"(when two nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
|
||||||
|
|
@ -39,7 +35,7 @@ abstract class MembershipChangeListenerUpSpec
|
||||||
|
|
||||||
runOn(first, second) {
|
runOn(first, second) {
|
||||||
val latch = TestLatch()
|
val latch = TestLatch()
|
||||||
val expectedAddresses = Set(firstAddress, secondAddress)
|
val expectedAddresses = Set(first, second) map address
|
||||||
cluster.registerListener(new MembershipChangeListener {
|
cluster.registerListener(new MembershipChangeListener {
|
||||||
def notify(members: SortedSet[Member]) {
|
def notify(members: SortedSet[Member]) {
|
||||||
if (members.map(_.address) == expectedAddresses && members.forall(_.status == MemberStatus.Up))
|
if (members.map(_.address) == expectedAddresses && members.forall(_.status == MemberStatus.Up))
|
||||||
|
|
@ -47,7 +43,7 @@ abstract class MembershipChangeListenerUpSpec
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
testConductor.enter("listener-1-registered")
|
testConductor.enter("listener-1-registered")
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
latch.await
|
latch.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -61,7 +57,7 @@ abstract class MembershipChangeListenerUpSpec
|
||||||
"(when three nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
|
"(when three nodes) after cluster convergence updates the membership table then all MembershipChangeListeners should be triggered" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
val latch = TestLatch()
|
val latch = TestLatch()
|
||||||
val expectedAddresses = Set(firstAddress, secondAddress, thirdAddress)
|
val expectedAddresses = Set(first, second, third) map address
|
||||||
cluster.registerListener(new MembershipChangeListener {
|
cluster.registerListener(new MembershipChangeListener {
|
||||||
def notify(members: SortedSet[Member]) {
|
def notify(members: SortedSet[Member]) {
|
||||||
if (members.map(_.address) == expectedAddresses && members.forall(_.status == MemberStatus.Up))
|
if (members.map(_.address) == expectedAddresses && members.forall(_.status == MemberStatus.Up))
|
||||||
|
|
@ -71,7 +67,7 @@ abstract class MembershipChangeListenerUpSpec
|
||||||
testConductor.enter("listener-2-registered")
|
testConductor.enter("listener-2-registered")
|
||||||
|
|
||||||
runOn(third) {
|
runOn(third) {
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
|
|
||||||
latch.await
|
latch.await
|
||||||
|
|
|
||||||
|
|
@ -173,7 +173,7 @@ trait MultiNodeClusterSpec extends FailureDetectorStrategy with Suite { self: Mu
|
||||||
/**
|
/**
|
||||||
* Wait until the specified nodes have seen the same gossip overview.
|
* Wait until the specified nodes have seen the same gossip overview.
|
||||||
*/
|
*/
|
||||||
def awaitSeenSameState(addresses: Seq[Address]): Unit = {
|
def awaitSeenSameState(addresses: Address*): Unit = {
|
||||||
awaitCond {
|
awaitCond {
|
||||||
val seen = cluster.latestGossip.overview.seen
|
val seen = cluster.latestGossip.overview.seen
|
||||||
val seenVectorClocks = addresses.flatMap(seen.get(_))
|
val seenVectorClocks = addresses.flatMap(seen.get(_))
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ object NodeJoinMultiJvmSpec extends MultiNodeConfig {
|
||||||
commonConfig(
|
commonConfig(
|
||||||
debugConfig(on = false)
|
debugConfig(on = false)
|
||||||
.withFallback(ConfigFactory.parseString("akka.cluster.leader-actions-interval = 5 s") // increase the leader action task interval
|
.withFallback(ConfigFactory.parseString("akka.cluster.leader-actions-interval = 5 s") // increase the leader action task interval
|
||||||
.withFallback(MultiNodeClusterSpec.clusterConfig)))
|
.withFallback(MultiNodeClusterSpec.clusterConfig)))
|
||||||
}
|
}
|
||||||
|
|
||||||
class NodeJoinMultiJvmNode1 extends NodeJoinSpec with FailureDetectorPuppetStrategy
|
class NodeJoinMultiJvmNode1 extends NodeJoinSpec with FailureDetectorPuppetStrategy
|
||||||
|
|
@ -29,9 +29,6 @@ abstract class NodeJoinSpec
|
||||||
|
|
||||||
import NodeJoinMultiJvmSpec._
|
import NodeJoinMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
|
|
||||||
"A cluster node" must {
|
"A cluster node" must {
|
||||||
"join another cluster and get status JOINING - when sending a 'Join' command" taggedAs LongRunningTest in {
|
"join another cluster and get status JOINING - when sending a 'Join' command" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
|
|
@ -40,10 +37,10 @@ abstract class NodeJoinSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
runOn(second) {
|
runOn(second) {
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
|
|
||||||
awaitCond(cluster.latestGossip.members.exists { member ⇒ member.address == secondAddress && member.status == MemberStatus.Joining })
|
awaitCond(cluster.latestGossip.members.exists { member ⇒ member.address == address(second) && member.status == MemberStatus.Joining })
|
||||||
|
|
||||||
testConductor.enter("after")
|
testConductor.enter("after")
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,10 +28,6 @@ abstract class NodeLeavingAndExitingAndBeingRemovedSpec
|
||||||
|
|
||||||
import NodeLeavingAndExitingAndBeingRemovedMultiJvmSpec._
|
import NodeLeavingAndExitingAndBeingRemovedMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
val reaperWaitingTime = 30.seconds.dilated
|
val reaperWaitingTime = 30.seconds.dilated
|
||||||
|
|
||||||
"A node that is LEAVING a non-singleton cluster" must {
|
"A node that is LEAVING a non-singleton cluster" must {
|
||||||
|
|
@ -42,13 +38,13 @@ abstract class NodeLeavingAndExitingAndBeingRemovedSpec
|
||||||
awaitClusterUp(first, second, third)
|
awaitClusterUp(first, second, third)
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
cluster.leave(secondAddress)
|
cluster.leave(second)
|
||||||
}
|
}
|
||||||
testConductor.enter("second-left")
|
testConductor.enter("second-left")
|
||||||
|
|
||||||
runOn(first, third) {
|
runOn(first, third) {
|
||||||
// verify that the 'second' node is no longer part of the 'members' set
|
// verify that the 'second' node is no longer part of the 'members' set
|
||||||
awaitCond(cluster.latestGossip.members.forall(_.address != secondAddress), reaperWaitingTime)
|
awaitCond(cluster.latestGossip.members.forall(_.address != address(second)), reaperWaitingTime)
|
||||||
|
|
||||||
// verify that the 'second' node is part of the 'unreachable' set
|
// verify that the 'second' node is part of the 'unreachable' set
|
||||||
awaitCond(cluster.latestGossip.overview.unreachable.exists(_.status == MemberStatus.Removed), reaperWaitingTime)
|
awaitCond(cluster.latestGossip.overview.unreachable.exists(_.status == MemberStatus.Removed), reaperWaitingTime)
|
||||||
|
|
@ -56,7 +52,7 @@ abstract class NodeLeavingAndExitingAndBeingRemovedSpec
|
||||||
// verify node that got removed is 'second' node
|
// verify node that got removed is 'second' node
|
||||||
val isRemoved = cluster.latestGossip.overview.unreachable.find(_.status == MemberStatus.Removed)
|
val isRemoved = cluster.latestGossip.overview.unreachable.find(_.status == MemberStatus.Removed)
|
||||||
isRemoved must be('defined)
|
isRemoved must be('defined)
|
||||||
isRemoved.get.address must be(secondAddress)
|
isRemoved.get.address must be(address(second))
|
||||||
}
|
}
|
||||||
|
|
||||||
testConductor.enter("finished")
|
testConductor.enter("finished")
|
||||||
|
|
|
||||||
|
|
@ -36,10 +36,6 @@ abstract class NodeLeavingAndExitingSpec
|
||||||
|
|
||||||
import NodeLeavingAndExitingMultiJvmSpec._
|
import NodeLeavingAndExitingMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A node that is LEAVING a non-singleton cluster" must {
|
"A node that is LEAVING a non-singleton cluster" must {
|
||||||
|
|
||||||
// FIXME make it work and remove ignore
|
// FIXME make it work and remove ignore
|
||||||
|
|
@ -48,7 +44,7 @@ abstract class NodeLeavingAndExitingSpec
|
||||||
awaitClusterUp(first, second, third)
|
awaitClusterUp(first, second, third)
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
cluster.leave(secondAddress)
|
cluster.leave(second)
|
||||||
}
|
}
|
||||||
testConductor.enter("second-left")
|
testConductor.enter("second-left")
|
||||||
|
|
||||||
|
|
@ -60,13 +56,13 @@ abstract class NodeLeavingAndExitingSpec
|
||||||
awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Leaving)) // wait on LEAVING
|
awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Leaving)) // wait on LEAVING
|
||||||
val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving) // verify node that left
|
val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving) // verify node that left
|
||||||
hasLeft must be('defined)
|
hasLeft must be('defined)
|
||||||
hasLeft.get.address must be(secondAddress)
|
hasLeft.get.address must be(address(second))
|
||||||
|
|
||||||
// 2. Verify that 'second' node is set to EXITING
|
// 2. Verify that 'second' node is set to EXITING
|
||||||
awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Exiting)) // wait on EXITING
|
awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Exiting)) // wait on EXITING
|
||||||
val hasExited = cluster.latestGossip.members.find(_.status == MemberStatus.Exiting) // verify node that exited
|
val hasExited = cluster.latestGossip.members.find(_.status == MemberStatus.Exiting) // verify node that exited
|
||||||
hasExited must be('defined)
|
hasExited must be('defined)
|
||||||
hasExited.get.address must be(secondAddress)
|
hasExited.get.address must be(address(second))
|
||||||
}
|
}
|
||||||
|
|
||||||
testConductor.enter("finished")
|
testConductor.enter("finished")
|
||||||
|
|
|
||||||
|
|
@ -30,10 +30,6 @@ abstract class NodeLeavingSpec
|
||||||
|
|
||||||
import NodeLeavingMultiJvmSpec._
|
import NodeLeavingMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A node that is LEAVING a non-singleton cluster" must {
|
"A node that is LEAVING a non-singleton cluster" must {
|
||||||
|
|
||||||
// FIXME make it work and remove ignore
|
// FIXME make it work and remove ignore
|
||||||
|
|
@ -42,7 +38,7 @@ abstract class NodeLeavingSpec
|
||||||
awaitClusterUp(first, second, third)
|
awaitClusterUp(first, second, third)
|
||||||
|
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
cluster.leave(secondAddress)
|
cluster.leave(second)
|
||||||
}
|
}
|
||||||
testConductor.enter("second-left")
|
testConductor.enter("second-left")
|
||||||
|
|
||||||
|
|
@ -51,7 +47,7 @@ abstract class NodeLeavingSpec
|
||||||
|
|
||||||
val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving)
|
val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving)
|
||||||
hasLeft must be('defined)
|
hasLeft must be('defined)
|
||||||
hasLeft.get.address must be(secondAddress)
|
hasLeft.get.address must be(address(second))
|
||||||
}
|
}
|
||||||
|
|
||||||
testConductor.enter("finished")
|
testConductor.enter("finished")
|
||||||
|
|
|
||||||
|
|
@ -26,10 +26,6 @@ abstract class NodeMembershipSpec
|
||||||
|
|
||||||
import NodeMembershipMultiJvmSpec._
|
import NodeMembershipMultiJvmSpec._
|
||||||
|
|
||||||
lazy val firstAddress = node(first).address
|
|
||||||
lazy val secondAddress = node(second).address
|
|
||||||
lazy val thirdAddress = node(third).address
|
|
||||||
|
|
||||||
"A set of connected cluster systems" must {
|
"A set of connected cluster systems" must {
|
||||||
|
|
||||||
"(when two nodes) start gossiping to each other so that both nodes gets the same gossip info" taggedAs LongRunningTest in {
|
"(when two nodes) start gossiping to each other so that both nodes gets the same gossip info" taggedAs LongRunningTest in {
|
||||||
|
|
@ -41,9 +37,9 @@ abstract class NodeMembershipSpec
|
||||||
testConductor.enter("first-started")
|
testConductor.enter("first-started")
|
||||||
|
|
||||||
runOn(first, second) {
|
runOn(first, second) {
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
awaitCond(cluster.latestGossip.members.size == 2)
|
awaitCond(cluster.latestGossip.members.size == 2)
|
||||||
assertMembers(cluster.latestGossip.members, firstAddress, secondAddress)
|
assertMembers(cluster.latestGossip.members, first, second)
|
||||||
awaitCond {
|
awaitCond {
|
||||||
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
|
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
|
||||||
}
|
}
|
||||||
|
|
@ -56,11 +52,11 @@ abstract class NodeMembershipSpec
|
||||||
"(when three nodes) start gossiping to each other so that all nodes gets the same gossip info" taggedAs LongRunningTest in {
|
"(when three nodes) start gossiping to each other so that all nodes gets the same gossip info" taggedAs LongRunningTest in {
|
||||||
|
|
||||||
runOn(third) {
|
runOn(third) {
|
||||||
cluster.join(firstAddress)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
|
|
||||||
awaitCond(cluster.latestGossip.members.size == 3)
|
awaitCond(cluster.latestGossip.members.size == 3)
|
||||||
assertMembers(cluster.latestGossip.members, firstAddress, secondAddress, thirdAddress)
|
assertMembers(cluster.latestGossip.members, first, second, third)
|
||||||
awaitCond {
|
awaitCond {
|
||||||
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
|
cluster.latestGossip.members.forall(_.status == MemberStatus.Up)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ abstract class NodeUpSpec
|
||||||
testConductor.enter("listener-registered")
|
testConductor.enter("listener-registered")
|
||||||
|
|
||||||
runOn(second) {
|
runOn(second) {
|
||||||
cluster.join(node(first).address)
|
cluster.join(first)
|
||||||
}
|
}
|
||||||
testConductor.enter("joined-again")
|
testConductor.enter("joined-again")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ abstract class SingletonClusterSpec
|
||||||
|
|
||||||
"become singleton cluster when one node is shutdown" taggedAs LongRunningTest in {
|
"become singleton cluster when one node is shutdown" taggedAs LongRunningTest in {
|
||||||
runOn(first) {
|
runOn(first) {
|
||||||
val secondAddress = node(second).address
|
val secondAddress = address(second)
|
||||||
testConductor.shutdown(second, 0)
|
testConductor.shutdown(second, 0)
|
||||||
|
|
||||||
markNodeAsUnavailable(secondAddress)
|
markNodeAsUnavailable(secondAddress)
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ abstract class TransitionSpec
|
||||||
}
|
}
|
||||||
|
|
||||||
def awaitSeen(addresses: Address*): Unit = awaitCond {
|
def awaitSeen(addresses: Address*): Unit = awaitCond {
|
||||||
seenLatestGossip.map(node(_).address) == addresses.toSet
|
(seenLatestGossip map address) == addresses.toSet
|
||||||
}
|
}
|
||||||
|
|
||||||
def awaitMembers(addresses: Address*): Unit = awaitCond {
|
def awaitMembers(addresses: Address*): Unit = awaitCond {
|
||||||
|
|
@ -86,7 +86,7 @@ abstract class TransitionSpec
|
||||||
}
|
}
|
||||||
runOn(fromRole) {
|
runOn(fromRole) {
|
||||||
testConductor.enter("before-gossip-" + gossipBarrierCounter)
|
testConductor.enter("before-gossip-" + gossipBarrierCounter)
|
||||||
cluster.gossipTo(node(toRole).address) // send gossip
|
cluster.gossipTo(toRole) // send gossip
|
||||||
testConductor.enter("after-gossip-" + gossipBarrierCounter)
|
testConductor.enter("after-gossip-" + gossipBarrierCounter)
|
||||||
}
|
}
|
||||||
runOn(roles.filterNot(r ⇒ r == fromRole || r == toRole): _*) {
|
runOn(roles.filterNot(r ⇒ r == fromRole || r == toRole): _*) {
|
||||||
|
|
@ -252,7 +252,7 @@ abstract class TransitionSpec
|
||||||
// first non-leader gossipTo the other non-leader
|
// first non-leader gossipTo the other non-leader
|
||||||
nonLeader(first, second, third).head gossipTo nonLeader(first, second, third).tail.head
|
nonLeader(first, second, third).head gossipTo nonLeader(first, second, third).tail.head
|
||||||
runOn(nonLeader(first, second, third).head) {
|
runOn(nonLeader(first, second, third).head) {
|
||||||
cluster.gossipTo(node(nonLeader(first, second, third).tail.head).address)
|
cluster.gossipTo(nonLeader(first, second, third).tail.head)
|
||||||
}
|
}
|
||||||
runOn(nonLeader(first, second, third).tail.head) {
|
runOn(nonLeader(first, second, third).tail.head) {
|
||||||
memberStatus(third) must be(Up)
|
memberStatus(third) must be(Up)
|
||||||
|
|
@ -412,6 +412,8 @@ abstract class TransitionSpec
|
||||||
awaitMemberStatus(second, Down)
|
awaitMemberStatus(second, Down)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
testConductor.enter("after-third-down")
|
||||||
|
|
||||||
// spread the word
|
// spread the word
|
||||||
val gossipRound2 = List(third, fourth, fifth, first, third, fourth, fifth)
|
val gossipRound2 = List(third, fourth, fifth, first, third, fourth, fifth)
|
||||||
for (x :: y :: Nil ← gossipRound2.sliding(2)) {
|
for (x :: y :: Nil ← gossipRound2.sliding(2)) {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue