raboof/pekko
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions 11

harden MultiDcSplitBrainSpec, #24955

Browse source 
* separate the testing of publication of DataCenterReachabilityEvent
  to separate test case, since doing it in the other cases is difficult
  due to timing and race conditions (flipping back-n-forth)
This commit is contained in:
Patrik Nordwall 2019-08-27 12:49:29 +02:00
parent c074b227ab
commit 46804546c6
1 changed files with 39 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

45
akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala
View file

@ -73,11 +73,6 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
def splitDataCenters(doNotVerify: Set[RoleName]): Unit = { def splitDataCenters(doNotVerify: Set[RoleName]): Unit = {
splits += 1 splits += 1
val memberNodes = (dc1 ++ dc2).filterNot(doNotVerify) val memberNodes = (dc1 ++ dc2).filterNot(doNotVerify)
val probe = TestProbe()
runOn(memberNodes: _*) {
cluster.subscribe(probe.ref, classOf[UnreachableDataCenter])
probe.expectMsgType[CurrentClusterState]
}
enterBarrier(s"split-$splits") enterBarrier(s"split-$splits")
runOn(first) { runOn(first) {
@ -89,8 +84,7 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
enterBarrier(s"after-split-$splits") enterBarrier(s"after-split-$splits")
runOn(memberNodes: _*) { runOn(memberNodes: _*) {
probe.expectMsgType[UnreachableDataCenter](15.seconds) within(15.seconds) {
cluster.unsubscribe(probe.ref)
runOn(dc1: _*) { runOn(dc1: _*) {
awaitAssert { awaitAssert {
cluster.state.unreachableDataCenters should ===(Set("dc2")) cluster.state.unreachableDataCenters should ===(Set("dc2"))
@ -101,6 +95,8 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
cluster.state.unreachableDataCenters should ===(Set("dc1")) cluster.state.unreachableDataCenters should ===(Set("dc1"))
} }
} }
}
system.log.debug("Cluster state after split: {}", cluster.state)
cluster.state.unreachable should ===(Set.empty) cluster.state.unreachable should ===(Set.empty)
} }
enterBarrier(s"after-split-verified-$splits") enterBarrier(s"after-split-verified-$splits")
@ -109,11 +105,6 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
def unsplitDataCenters(notMembers: Set[RoleName]): Unit = { def unsplitDataCenters(notMembers: Set[RoleName]): Unit = {
unsplits += 1 unsplits += 1
val memberNodes = (dc1 ++ dc2).filterNot(notMembers) val memberNodes = (dc1 ++ dc2).filterNot(notMembers)
val probe = TestProbe()
runOn(memberNodes: _*) {
cluster.subscribe(probe.ref, classOf[ReachableDataCenter])
probe.expectMsgType[CurrentClusterState]
}
enterBarrier(s"unsplit-$unsplits") enterBarrier(s"unsplit-$unsplits")
runOn(first) { runOn(first) {
@ -125,12 +116,11 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
enterBarrier(s"after-unsplit-$unsplits") enterBarrier(s"after-unsplit-$unsplits")
runOn(memberNodes: _*) { runOn(memberNodes: _*) {
probe.expectMsgType[ReachableDataCenter](25.seconds) within(15.seconds) {
system.log.debug("Reachable data center received")
cluster.unsubscribe(probe.ref)
awaitAssert { awaitAssert {
cluster.state.unreachableDataCenters should ===(Set.empty) cluster.state.unreachableDataCenters should ===(Set.empty)
system.log.debug("Cluster state: {}", cluster.state) }
system.log.debug("Cluster state after unsplit: {}", cluster.state)
} }
} }
enterBarrier(s"after-unsplit-verified-$unsplits") enterBarrier(s"after-unsplit-verified-$unsplits")
@ -141,6 +131,28 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
awaitClusterUp(first, second, third) awaitClusterUp(first, second, third)
} }
"publish DataCenterReachabilityEvent" in {
val dcReachabilityProbe = TestProbe()
Cluster(system).subscribe(
dcReachabilityProbe.ref,
ClusterEvent.InitialStateAsSnapshot,
classOf[DataCenterReachabilityEvent])
dcReachabilityProbe.expectMsgType[CurrentClusterState].unreachableDataCenters should ===(Set.empty)
splitDataCenters(doNotVerify = Set(fourth, fifth))
runOn(first, second, third) {
dcReachabilityProbe.expectMsgType[UnreachableDataCenter]
}
enterBarrier("unreachable-verified")
unsplitDataCenters(notMembers = Set(fourth, fifth))
runOn(first, second, third) {
dcReachabilityProbe.expectMsgType[ReachableDataCenter]
}
enterBarrier("publish-event-verified")
}
"be able to have a data center member join while there is inter data center split" in within(20.seconds) { "be able to have a data center member join while there is inter data center split" in within(20.seconds) {
// introduce a split between data centers // introduce a split between data centers
splitDataCenters(doNotVerify = Set(fourth, fifth)) splitDataCenters(doNotVerify = Set(fourth, fifth))
@ -189,6 +201,7 @@ abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMult
runOn(first, second) { runOn(first, second) {
awaitAssert(clusterView.members.filter(_.address == address(fourth)) should ===(Set.empty)) awaitAssert(clusterView.members.filter(_.address == address(fourth)) should ===(Set.empty))
} }
enterBarrier("inter-data-center-split-2-done") enterBarrier("inter-data-center-split-2-done")
} }