686729c75b
* Harden multi-dc joining, #29280 * failing test MultiDcJoinSpec * require that all have seen the gossip seen for the first member in other DC * the test also revealed that gossip wasn't propagated between DCs when the VectorClock was the same and only seen is different * add a SHA-1 disgest of the seen in the GossipStatus to detect that they are different and that full gossip should be exchanged * comments * another test * mima version
143 lines
4 KiB
Scala
143 lines
4 KiB
Scala
/*
|
|
* Copyright (C) 2009-2020 Lightbend Inc. <https://www.lightbend.com>
|
|
*/
|
|
|
|
package akka.cluster
|
|
|
|
import scala.concurrent.duration._
|
|
|
|
import com.typesafe.config.ConfigFactory
|
|
|
|
import akka.actor.Address
|
|
import akka.cluster.ClusterEvent.InitialStateAsEvents
|
|
import akka.cluster.ClusterEvent.MemberUp
|
|
import akka.remote.testkit.MultiNodeConfig
|
|
import akka.remote.testkit.MultiNodeSpec
|
|
import akka.testkit._
|
|
|
|
// Similar to MultiDcJoinSpec, but slightly different scenario
|
|
object MultiDcJoin2MultiJvmSpec extends MultiNodeConfig {
|
|
val first = role("first")
|
|
val second = role("second")
|
|
val third = role("third")
|
|
val fourth = role("fourth")
|
|
val fifth = role("fifth")
|
|
|
|
nodeConfig(first, second, third)(ConfigFactory.parseString("""
|
|
akka {
|
|
cluster.multi-data-center.self-data-center = alpha
|
|
}
|
|
"""))
|
|
|
|
nodeConfig(fourth, fifth)(ConfigFactory.parseString("""
|
|
akka {
|
|
cluster.multi-data-center.self-data-center = beta
|
|
}
|
|
"""))
|
|
|
|
commonConfig(ConfigFactory.parseString("""
|
|
akka {
|
|
actor.provider = cluster
|
|
|
|
loggers = ["akka.testkit.TestEventListener"]
|
|
loglevel = INFO
|
|
|
|
remote.log-remote-lifecycle-events = off
|
|
|
|
cluster {
|
|
debug.verbose-heartbeat-logging = off
|
|
debug.verbose-gossip-logging = off
|
|
|
|
multi-data-center {
|
|
cross-data-center-connections = 1
|
|
}
|
|
}
|
|
}
|
|
"""))
|
|
|
|
}
|
|
|
|
class MultiDcJoin2MultiJvmNode1 extends MultiDcJoin2Spec
|
|
class MultiDcJoin2MultiJvmNode2 extends MultiDcJoin2Spec
|
|
class MultiDcJoin2MultiJvmNode3 extends MultiDcJoin2Spec
|
|
class MultiDcJoin2MultiJvmNode4 extends MultiDcJoin2Spec
|
|
class MultiDcJoin2MultiJvmNode5 extends MultiDcJoin2Spec
|
|
|
|
abstract class MultiDcJoin2Spec extends MultiNodeSpec(MultiDcJoin2MultiJvmSpec) with MultiNodeClusterSpec {
|
|
import MultiDcJoin2MultiJvmSpec._
|
|
|
|
"Joining a multi-dc cluster, scenario 2" must {
|
|
"make sure oldest is selected correctly" taggedAs LongRunningTest in {
|
|
|
|
val observer = TestProbe("beta-observer")
|
|
runOn(fourth, fifth) {
|
|
Cluster(system).subscribe(observer.ref, InitialStateAsEvents, classOf[MemberUp])
|
|
}
|
|
val memberUpFromFifth = TestProbe("fromFifth")
|
|
runOn(fourth) {
|
|
system.actorOf(TestActors.forwardActorProps(memberUpFromFifth.ref), "fwFromFifth")
|
|
}
|
|
|
|
// all alpha nodes
|
|
awaitClusterUp(first, second, third)
|
|
|
|
runOn(fourth) {
|
|
Cluster(system).join(first)
|
|
within(20.seconds) {
|
|
awaitAssert {
|
|
Cluster(system).state.members
|
|
.exists(m => m.address == address(fourth) && m.status == MemberStatus.Up) should ===(true)
|
|
}
|
|
}
|
|
}
|
|
|
|
// at the same time join fifth, which is the difference compared to MultiDcJoinSpec
|
|
runOn(fifth) {
|
|
Cluster(system).join(second)
|
|
within(10.seconds) {
|
|
awaitAssert {
|
|
Cluster(system).state.members
|
|
.exists(m => m.address == address(fifth) && m.status == MemberStatus.Up) should ===(true)
|
|
}
|
|
}
|
|
}
|
|
enterBarrier("beta-joined")
|
|
|
|
runOn(fifth) {
|
|
val events = observer
|
|
.receiveN(5)
|
|
.map(_.asInstanceOf[MemberUp])
|
|
.filter(_.member.dataCenter == "beta")
|
|
.sortBy(_.member.upNumber)
|
|
|
|
events.head.member.upNumber should ===(1)
|
|
events(1).member.upNumber should ===(2)
|
|
|
|
// only sending Address since it's serializable
|
|
events.foreach(evt => system.actorSelection(node(fourth) / "user" / "fwFromFifth") ! evt.member.address)
|
|
}
|
|
enterBarrier("fw-from-fifth")
|
|
|
|
runOn(fourth) {
|
|
val events4 = observer
|
|
.receiveN(5)
|
|
.map(_.asInstanceOf[MemberUp])
|
|
.filter(_.member.dataCenter == "beta")
|
|
.sortBy(_.member.upNumber)
|
|
|
|
val upFromFifth = memberUpFromFifth.receiveN(2).map(_.asInstanceOf[Address])
|
|
|
|
events4.head.member.address should ===(upFromFifth.head)
|
|
|
|
events4.head.member.upNumber should ===(1)
|
|
events4(1).member.upNumber should ===(2)
|
|
|
|
observer.expectNoMessage(2.seconds)
|
|
}
|
|
|
|
enterBarrier("done")
|
|
}
|
|
|
|
}
|
|
|
|
}
|