Harden multi-dc joining, #29280 (#29346)

* Harden multi-dc joining, #29280

* failing test MultiDcJoinSpec
* require that all have seen the gossip seen for the first member in other DC
* the test also revealed that gossip wasn't propagated between DCs when
  the VectorClock was the same and only seen is different
* add a SHA-1 disgest of the seen in the GossipStatus to detect that they
  are different and that full gossip should be exchanged

* comments

* another test

* mima version

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcJoin2Spec.scala

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2009-2020 Lightbend Inc. <https://www.lightbend.com>
+ */
+
+package akka.cluster
+
+import scala.concurrent.duration._
+
+import com.typesafe.config.ConfigFactory
+
+import akka.actor.Address
+import akka.cluster.ClusterEvent.InitialStateAsEvents
+import akka.cluster.ClusterEvent.MemberUp
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.testkit._
+
+// Similar to MultiDcJoinSpec, but slightly different scenario
+object MultiDcJoin2MultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+  val fifth = role("fifth")
+
+  nodeConfig(first, second, third)(ConfigFactory.parseString("""
+    akka {
+      cluster.multi-data-center.self-data-center = alpha
+    }
+    """))
+
+  nodeConfig(fourth, fifth)(ConfigFactory.parseString("""
+    akka {
+      cluster.multi-data-center.self-data-center = beta
+    }
+    """))
+
+  commonConfig(ConfigFactory.parseString("""
+    akka {
+      actor.provider = cluster
+
+      loggers = ["akka.testkit.TestEventListener"]
+      loglevel = INFO
+
+      remote.log-remote-lifecycle-events = off
+
+      cluster {
+        debug.verbose-heartbeat-logging = off
+        debug.verbose-gossip-logging = off
+
+        multi-data-center {
+          cross-data-center-connections = 1
+        }
+      }
+    }
+    """))
+
+}
+
+class MultiDcJoin2MultiJvmNode1 extends MultiDcJoin2Spec
+class MultiDcJoin2MultiJvmNode2 extends MultiDcJoin2Spec
+class MultiDcJoin2MultiJvmNode3 extends MultiDcJoin2Spec
+class MultiDcJoin2MultiJvmNode4 extends MultiDcJoin2Spec
+class MultiDcJoin2MultiJvmNode5 extends MultiDcJoin2Spec
+
+abstract class MultiDcJoin2Spec extends MultiNodeSpec(MultiDcJoin2MultiJvmSpec) with MultiNodeClusterSpec {
+  import MultiDcJoin2MultiJvmSpec._
+
+  "Joining a multi-dc cluster, scenario 2" must {
+    "make sure oldest is selected correctly" taggedAs LongRunningTest in {
+
+      val observer = TestProbe("beta-observer")
+      runOn(fourth, fifth) {
+        Cluster(system).subscribe(observer.ref, InitialStateAsEvents, classOf[MemberUp])
+      }
+      val memberUpFromFifth = TestProbe("fromFifth")
+      runOn(fourth) {
+        system.actorOf(TestActors.forwardActorProps(memberUpFromFifth.ref), "fwFromFifth")
+      }
+
+      // all alpha nodes
+      awaitClusterUp(first, second, third)
+
+      runOn(fourth) {
+        Cluster(system).join(first)
+        within(20.seconds) {
+          awaitAssert {
+            Cluster(system).state.members
+              .exists(m => m.address == address(fourth) && m.status == MemberStatus.Up) should ===(true)
+          }
+        }
+      }
+
+      // at the same time join fifth, which is the difference compared to MultiDcJoinSpec
+      runOn(fifth) {
+        Cluster(system).join(second)
+        within(10.seconds) {
+          awaitAssert {
+            Cluster(system).state.members
+              .exists(m => m.address == address(fifth) && m.status == MemberStatus.Up) should ===(true)
+          }
+        }
+      }
+      enterBarrier("beta-joined")
+
+      runOn(fifth) {
+        val events = observer
+          .receiveN(5)
+          .map(_.asInstanceOf[MemberUp])
+          .filter(_.member.dataCenter == "beta")
+          .sortBy(_.member.upNumber)
+
+        events.head.member.upNumber should ===(1)
+        events(1).member.upNumber should ===(2)
+
+        // only sending Address since it's serializable
+        events.foreach(evt => system.actorSelection(node(fourth) / "user" / "fwFromFifth") ! evt.member.address)
+      }
+      enterBarrier("fw-from-fifth")
+
+      runOn(fourth) {
+        val events4 = observer
+          .receiveN(5)
+          .map(_.asInstanceOf[MemberUp])
+          .filter(_.member.dataCenter == "beta")
+          .sortBy(_.member.upNumber)
+
+        val upFromFifth = memberUpFromFifth.receiveN(2).map(_.asInstanceOf[Address])
+
+        events4.head.member.address should ===(upFromFifth.head)
+
+        events4.head.member.upNumber should ===(1)
+        events4(1).member.upNumber should ===(2)
+
+        observer.expectNoMessage(2.seconds)
+      }
+
+      enterBarrier("done")
+    }
+
+  }
+
+}