From ab3efff3bd5268e5a730008222405b895dd115e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Andr=C3=A9n?= Date: Wed, 5 Jul 2017 11:08:55 +0200 Subject: [PATCH] MultiDcSplitBrainSpec fixed #23288 --- .../akka/cluster/MultiDcSplitBrainSpec.scala | 46 ++++++++++--------- .../akka/cluster/MultiNodeClusterSpec.scala | 4 +- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala index 86b75dbe09..4b0700beb8 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala @@ -3,6 +3,7 @@ */ package akka.cluster +import akka.remote.testconductor.RoleName import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec } import akka.remote.transport.ThrottlerTransportAdapter.Direction import com.typesafe.config.ConfigFactory @@ -15,28 +16,29 @@ object MultiDcSplitBrainMultiJvmSpec extends MultiNodeConfig { val third = role("third") val fourth = role("fourth") - commonConfig(MultiNodeClusterSpec.clusterConfig) + commonConfig(ConfigFactory.parseString( + """ + akka.loglevel = INFO + akka.cluster.run-coordinated-shutdown-when-down = off + """).withFallback(MultiNodeClusterSpec.clusterConfig)) nodeConfig(first, second)(ConfigFactory.parseString( """ akka.cluster.data-center = "dc1" - akka.loglevel = INFO """)) nodeConfig(third, fourth)(ConfigFactory.parseString( """ akka.cluster.data-center = "dc2" - akka.loglevel = INFO """)) testTransport(on = true) } -class MultiDcSplitBrainMultiJvmNode1 extends MultiDcSpec -class MultiDcSplitBrainMultiJvmNode2 extends MultiDcSpec -class MultiDcSplitBrainMultiJvmNode3 extends MultiDcSpec -class MultiDcSplitBrainMultiJvmNode4 extends MultiDcSpec -class MultiDcSplitBrainMultiJvmNode5 extends MultiDcSpec +class MultiDcSplitBrainMultiJvmNode1 extends MultiDcSplitBrainSpec +class MultiDcSplitBrainMultiJvmNode2 extends MultiDcSplitBrainSpec +class MultiDcSplitBrainMultiJvmNode3 extends MultiDcSplitBrainSpec +class MultiDcSplitBrainMultiJvmNode4 extends MultiDcSplitBrainSpec abstract class MultiDcSplitBrainSpec extends MultiNodeSpec(MultiDcSplitBrainMultiJvmSpec) @@ -47,7 +49,7 @@ abstract class MultiDcSplitBrainSpec val dc1 = List(first, second) val dc2 = List(third, fourth) - def splitDataCenters(): Unit = { + def splitDataCenters(dc1: Seq[RoleName], dc2: Seq[RoleName]): Unit = { runOn(first) { for { dc1Node ← dc1 @@ -58,15 +60,15 @@ abstract class MultiDcSplitBrainSpec } runOn(dc1: _*) { - awaitAssert(clusterView.unreachableMembers.map(_.address) should ===(dc2.map(address))) + awaitAssert(clusterView.unreachableMembers.map(_.address) should contain allElementsOf (dc2.map(address))) } runOn(dc2: _*) { - awaitAssert(clusterView.unreachableMembers.map(_.address) should ===(dc1.map(address))) + awaitAssert(clusterView.unreachableMembers.map(_.address) should contain allElementsOf (dc1.map(address))) } } - def unsplitDataCenters(): Unit = { + def unsplitDataCenters(dc1: Seq[RoleName], dc2: Seq[RoleName]): Unit = { runOn(first) { for { dc1Node ← dc1 @@ -76,7 +78,9 @@ abstract class MultiDcSplitBrainSpec } } - awaitAllReachable() + runOn(dc1 ++ dc2: _*) { + awaitAssert(clusterView.unreachableMembers.map(_.address) should be(empty)) + } } "A cluster with multiple data centers" must { @@ -86,7 +90,7 @@ abstract class MultiDcSplitBrainSpec "be able to have a data center member join while there is inter data center split" in within(20.seconds) { // introduce a split between data centers - splitDataCenters() + splitDataCenters(dc1 = List(first, second), dc2 = List(third)) enterBarrier("data-center-split-1") runOn(fourth) { @@ -99,36 +103,36 @@ abstract class MultiDcSplitBrainSpec runOn(third, fourth) { awaitAssert(clusterView.members.collect { case m if m.dataCenter == "dc2" && m.status == MemberStatus.Up ⇒ m.address - }) should ===(Set(address(third), address(fourth))) + } should ===(Set(address(third), address(fourth)))) } enterBarrier("dc2-join-completed") - unsplitDataCenters() + unsplitDataCenters(dc1 = List(first, second), dc2 = List(third)) enterBarrier("data-center-unsplit-1") runOn(dc1: _*) { awaitAssert(clusterView.members.collect { case m if m.dataCenter == "dc2" && m.status == MemberStatus.Up ⇒ m.address - }) should ===(Set(address(third), address(fourth))) + } should ===(Set(address(third), address(fourth)))) } enterBarrier("inter-data-center-split-1-done") } "be able to have data center member leave while there is inter data center split" in within(20.seconds) { - splitDataCenters() + splitDataCenters(dc1, dc2) enterBarrier("data-center-split-2") runOn(fourth) { - cluster.leave(third) + cluster.leave(fourth) } - runOn(third, fourth) { + runOn(third) { awaitAssert(clusterView.members.filter(_.address == address(fourth)) should ===(Set.empty)) } enterBarrier("node-4-left") - unsplitDataCenters() + unsplitDataCenters(dc1, List(third)) enterBarrier("data-center-unsplit-2") runOn(first, second) { diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala index b249676e18..cb7f0e666b 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala @@ -305,7 +305,9 @@ trait MultiNodeClusterSpec extends Suite with STMultiNodeSpec with WatchedByCoro awaitAssert(clusterView.members.size should ===(numberOfMembers)) awaitAssert(clusterView.members.map(_.status) should ===(Set(MemberStatus.Up))) // clusterView.leader is updated by LeaderChanged, await that to be updated also - val expectedLeader = clusterView.members.headOption.map(_.address) + val expectedLeader = clusterView.members.collectFirst { + case m if m.dataCenter == cluster.settings.DataCenter ⇒ m.address + } awaitAssert(clusterView.leader should ===(expectedLeader)) } }