Handle lost typed receptionist removals #24887

Keep track of removed actors and re-remove them when ORMultiMap conflict has reintroduced them

akka-distributed-data/src/main/scala/akka/cluster/ddata/ORMultiMap.scala

@@ -58,6 +58,8 @@ object ORMultiMap {
  * [[ORMap]] with an [[ORSet]] for the map's value.
  *
  * This class is immutable, i.e. "modifying" methods return a new instance.
+ *
+ * Note that on concurrent adds and removals for the same key (on the same set), removals can be lost.
  */
 @SerialVersionUID(1L)
 final class ORMultiMap[A, B] private[akka] (

akka-distributed-data/src/test/scala/akka/cluster/ddata/ORMultiMapSpec.scala

@@ -11,8 +11,8 @@ import org.scalatest.{ Matchers, WordSpec }
 
 class ORMultiMapSpec extends WordSpec with Matchers {
 
-  val node1 = UniqueAddress(Address("akka.tcp", "Sys", "localhost", 2551), 1)
-  val node2 = UniqueAddress(node1.address.copy(port = Some(2552)), 2)
+  val node1 = UniqueAddress(Address("akka.tcp", "Sys", "localhost", 2551), 1L)
+  val node2 = UniqueAddress(node1.address.copy(port = Some(2552)), 2L)
 
   "A ORMultiMap" must {
 
@@ -34,6 +34,26 @@ class ORMultiMapSpec extends WordSpec with Matchers {
       m.entries should be(Map("a" → Set("B")))
     }
 
+    "not handle concurrent updates to the same set" in {
+      val m = ORMultiMap().addBinding(node1, "a", "A")
+
+      val m1 = m.removeBinding(node1, "a", "A")
+
+      val m2 = m.addBinding(node2, "a", "B")
+
+      val merged1 = m1.merge(m2)
+      val merged2 = m2.merge(m1)
+
+      // more to document that the concurrent removal from the set may be lost
+      // than asserting anything
+      merged1.entries should be(Map(
+        "a" -> Set("A", "B")
+      ))
+      merged2.entries should be(Map(
+        "a" -> Set("A", "B")
+      ))
+    }
+
     "be able to have its entries correctly merged with another ORMultiMap with other entries" in {
       val m1 = ORMultiMap().addBinding(node1, "a", "A").addBinding(node1, "b", "B")
       val m2 = ORMultiMap().addBinding(node2, "c", "C")