diff --git a/akka-cluster/src/main/scala/akka/cluster/MembershipState.scala b/akka-cluster/src/main/scala/akka/cluster/MembershipState.scala index 6c4a326d42..d2443066fb 100644 --- a/akka-cluster/src/main/scala/akka/cluster/MembershipState.scala +++ b/akka-cluster/src/main/scala/akka/cluster/MembershipState.scala @@ -60,14 +60,18 @@ import akka.util.ccompat._ !members.exists(member => member.dataCenter == selfDc && convergenceMemberStatus(member.status)) // If another member in the data center that is UP or LEAVING and has not seen this gossip or is exiting - // convergence cannot be reached. For the first member in a secondary DC all members must have seen - // the gossip state. - def memberHinderingConvergenceExists = + // convergence cannot be reached. For the first member in a secondary DC all Joining, WeaklyUp, Up or Leaving + // members must have seen the gossip state. The reason for the stronger requirement for a first member in a + // secondary DC is that first member should only be moved to Up once to ensure that the first upNumber is + // only assigned once. + def memberHinderingConvergenceExists = { + val memberStatus = if (firstMemberInDc) convergenceMemberStatus + Joining + WeaklyUp else convergenceMemberStatus members.exists( member => (firstMemberInDc || member.dataCenter == selfDc) && - convergenceMemberStatus(member.status) && + memberStatus(member.status) && !(latestGossip.seenByNode(member.uniqueAddress) || exitingConfirmed(member.uniqueAddress))) + } // Find cluster members in the data center that are unreachable from other members of the data center // excluding observations from members outside of the data center, that have status DOWN or is passed in as confirmed exiting. diff --git a/akka-cluster/src/test/scala/akka/cluster/GossipSpec.scala b/akka-cluster/src/test/scala/akka/cluster/GossipSpec.scala index c57ec39849..01213ce250 100644 --- a/akka-cluster/src/test/scala/akka/cluster/GossipSpec.scala +++ b/akka-cluster/src/test/scala/akka/cluster/GossipSpec.scala @@ -28,6 +28,7 @@ class GossipSpec extends AnyWordSpec with Matchers { val e1 = TestMember(Address("akka", "sys", "e", 2552), Joining) val e2 = TestMember(e1.address, Up) val e3 = TestMember(e1.address, Down) + val f1 = TestMember(Address("akka", "sys", "f", 2552), Joining) val dc1a1 = TestMember(Address("akka", "sys", "a", 2552), Up, Set.empty, dataCenter = "dc1") val dc1b1 = TestMember(Address("akka", "sys", "b", 2552), Up, Set.empty, dataCenter = "dc1") @@ -272,6 +273,24 @@ class GossipSpec extends AnyWordSpec with Matchers { state(g2, dc2e1).convergence(Set.empty) should ===(true) } + "not reach convergence for first member of other data center until all have seen the gossip 2" in { + // reproducer test for issue #29486 + val dc2e1 = TestMember(e1.address, status = Joining, roles = Set.empty, dataCenter = "dc2") + val dc2f1 = TestMember(f1.address, status = Joining, roles = Set.empty, dataCenter = "dc2") + val g = + Gossip(members = SortedSet(dc1a1, dc1b1, dc2e1, dc2f1)) + .seen(dc1a1.uniqueAddress) + .seen(dc1b1.uniqueAddress) + .seen(dc2f1.uniqueAddress) + + // dc2 hasn't reached convergence because dc2e1 has not seen it (and that matters even though it is only Joining) + state(g, dc2f1).convergence(Set.empty) should ===(false) + + // until all have seen it + val g2 = g.seen(dc2e1.uniqueAddress) + state(g2, dc2f1).convergence(Set.empty) should ===(true) + } + "reach convergence per data center even if another data center contains unreachable" in { val r1 = Reachability.empty.unreachable(dc2c1.uniqueAddress, dc2d1.uniqueAddress)