From f970c87ea449d764dd3540300860b0730cebd1f9 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Fri, 29 Jun 2012 09:38:06 +0200 Subject: [PATCH 1/2] Fix time sensitivity in LeaderLeavingSpec, see #2287 --- .../akka/cluster/LeaderLeavingSpec.scala | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderLeavingSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderLeavingSpec.scala index 54154b6973..e232802eeb 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderLeavingSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderLeavingSpec.scala @@ -18,10 +18,8 @@ object LeaderLeavingMultiJvmSpec extends MultiNodeConfig { commonConfig( debugConfig(on = false) .withFallback(ConfigFactory.parseString(""" - akka.cluster { - leader-actions-interval = 5 s # increase the leader action task frequency to make sure we get a chance to test the LEAVING state - unreachable-nodes-reaper-interval = 30 s - }""") + # turn off unreachable reaper + akka.cluster.unreachable-nodes-reaper-interval = 300 s""") .withFallback(MultiNodeClusterSpec.clusterConfig))) } @@ -35,7 +33,7 @@ abstract class LeaderLeavingSpec import LeaderLeavingMultiJvmSpec._ - val leaderHandoffWaitingTime = 30.seconds.dilated + val leaderHandoffWaitingTime = 30.seconds "A LEADER that is LEAVING" must { @@ -45,41 +43,60 @@ abstract class LeaderLeavingSpec val oldLeaderAddress = cluster.leader - if (cluster.isLeader) { + within(leaderHandoffWaitingTime) { - cluster.leave(oldLeaderAddress) - enterBarrier("leader-left") + if (cluster.isLeader) { - // verify that a NEW LEADER have taken over - awaitCond(!cluster.isLeader) + enterBarrier("registered-listener") - // verify that the LEADER is shut down - awaitCond(!cluster.isRunning, 30.seconds.dilated) + cluster.leave(oldLeaderAddress) + enterBarrier("leader-left") - // verify that the LEADER is REMOVED - awaitCond(cluster.status == MemberStatus.Removed) + // verify that a NEW LEADER have taken over + awaitCond(!cluster.isLeader) - } else { + // verify that the LEADER is shut down + awaitCond(!cluster.isRunning) - enterBarrier("leader-left") + // verify that the LEADER is REMOVED + awaitCond(cluster.status == MemberStatus.Removed) - // verify that the LEADER is LEAVING - awaitCond(cluster.latestGossip.members.exists(m ⇒ m.status == MemberStatus.Leaving && m.address == oldLeaderAddress), leaderHandoffWaitingTime) // wait on LEAVING + } else { - // verify that the LEADER is EXITING - awaitCond(cluster.latestGossip.members.exists(m ⇒ m.status == MemberStatus.Exiting && m.address == oldLeaderAddress), leaderHandoffWaitingTime) // wait on EXITING + val leavingLatch = TestLatch() + val exitingLatch = TestLatch() + val expectedAddresses = roles.toSet map address + cluster.registerListener(new MembershipChangeListener { + def notify(members: SortedSet[Member]) { + def check(status: MemberStatus): Boolean = + (members.map(_.address) == expectedAddresses && + members.exists(m ⇒ m.address == oldLeaderAddress && m.status == status)) + if (check(MemberStatus.Leaving)) leavingLatch.countDown() + if (check(MemberStatus.Exiting)) exitingLatch.countDown() + } + }) + enterBarrier("registered-listener") - // verify that the LEADER is no longer part of the 'members' set - awaitCond(cluster.latestGossip.members.forall(_.address != oldLeaderAddress), leaderHandoffWaitingTime) + enterBarrier("leader-left") - // verify that the LEADER is not part of the 'unreachable' set - awaitCond(cluster.latestGossip.overview.unreachable.forall(_.address != oldLeaderAddress), leaderHandoffWaitingTime) + // verify that the LEADER is LEAVING + leavingLatch.await - // verify that we have a new LEADER - awaitCond(cluster.leader != oldLeaderAddress, leaderHandoffWaitingTime) + // verify that the LEADER is EXITING + exitingLatch.await + + // verify that the LEADER is no longer part of the 'members' set + awaitCond(cluster.latestGossip.members.forall(_.address != oldLeaderAddress)) + + // verify that the LEADER is not part of the 'unreachable' set + awaitCond(cluster.latestGossip.overview.unreachable.forall(_.address != oldLeaderAddress)) + + // verify that we have a new LEADER + awaitCond(cluster.leader != oldLeaderAddress) + } + + enterBarrier("finished") } - - enterBarrier("finished") } } } From aa88818f715e0784da4af9952b35973ec3d8110b Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Tue, 3 Jul 2012 11:09:34 +0200 Subject: [PATCH 2/2] Fix time sensitivity in NodeLeavingAndExitingSpec, see #2289 --- .../cluster/NodeLeavingAndExitingSpec.scala | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala index 5f9efb0b47..565d78e9d8 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala @@ -18,11 +18,8 @@ object NodeLeavingAndExitingMultiJvmSpec extends MultiNodeConfig { commonConfig( debugConfig(on = false) .withFallback(ConfigFactory.parseString(""" - akka.cluster { - leader-actions-interval = 5 s # increase the leader action task frequency to make sure we get a chance to test the LEAVING state - unreachable-nodes-reaper-interval = 300 s # turn "off" - } - """) + # turn off unreachable reaper + akka.cluster.unreachable-nodes-reaper-interval = 300 s""") .withFallback(MultiNodeClusterSpec.clusterConfig))) } @@ -42,26 +39,39 @@ abstract class NodeLeavingAndExitingSpec awaitClusterUp(first, second, third) - runOn(first) { - cluster.leave(second) - } - enterBarrier("second-left") - runOn(first, third) { + val secondAddess = address(second) + val leavingLatch = TestLatch() + val exitingLatch = TestLatch() + val expectedAddresses = roles.toSet map address + cluster.registerListener(new MembershipChangeListener { + def notify(members: SortedSet[Member]) { + def check(status: MemberStatus): Boolean = + (members.map(_.address) == expectedAddresses && + members.exists(m ⇒ m.address == secondAddess && m.status == status)) + if (check(MemberStatus.Leaving)) leavingLatch.countDown() + if (check(MemberStatus.Exiting)) exitingLatch.countDown() + } + }) + enterBarrier("registered-listener") - // 1. Verify that 'second' node is set to LEAVING - // We have set the 'leader-actions-interval' to 5 seconds to make sure that we get a - // chance to test the LEAVING state before the leader moves the node to EXITING - awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Leaving)) // wait on LEAVING - val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving) // verify node that left - hasLeft must be('defined) - hasLeft.get.address must be(address(second)) + runOn(third) { + cluster.leave(second) + } + enterBarrier("second-left") - // 2. Verify that 'second' node is set to EXITING - awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Exiting)) // wait on EXITING - val hasExited = cluster.latestGossip.members.find(_.status == MemberStatus.Exiting) // verify node that exited - hasExited must be('defined) - hasExited.get.address must be(address(second)) + // Verify that 'second' node is set to LEAVING + leavingLatch.await + + // Verify that 'second' node is set to EXITING + exitingLatch.await + + } + + // node that is leaving + runOn(second) { + enterBarrier("registered-listener") + enterBarrier("second-left") } enterBarrier("finished")