causal delivery of deltas, #22188

* keep track of delta interval versions and skip deltas
  that are not consequtive, i.e. when some delta message was lost
* send the delta versions in the full state gossip to sync up the
  expected versions after dropped deltas
* implementation of deltas for ORSet
* refactoring of the delta types to allow for different type for the
  delta and the full state
* extensive tests
* mima filter
* performance optimizations
* simple pruning of deltas
* Java API
* update documentation
* KeyId type alias
* Use InternalApi annotation

akka-distributed-data/src/multi-jvm/scala/akka/cluster/ddata/ReplicatorORSetDeltaSpec.scala

@@ -0,0 +1,163 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster.ddata
+
+import scala.concurrent.duration._
+
+import akka.cluster.Cluster
+import akka.remote.testconductor.RoleName
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.remote.transport.ThrottlerTransportAdapter.Direction
+import akka.testkit._
+import com.typesafe.config.ConfigFactory
+
+object ReplicatorORSetDeltaSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+
+  commonConfig(ConfigFactory.parseString("""
+    akka.loglevel = DEBUG
+    akka.actor.provider = "cluster"
+    akka.log-dead-letters-during-shutdown = off
+    """))
+
+  testTransport(on = true)
+}
+
+class ReplicatorORSetDeltaSpecMultiJvmNode1 extends ReplicatorORSetDeltaSpec
+class ReplicatorORSetDeltaSpecMultiJvmNode2 extends ReplicatorORSetDeltaSpec
+class ReplicatorORSetDeltaSpecMultiJvmNode3 extends ReplicatorORSetDeltaSpec
+
+class ReplicatorORSetDeltaSpec extends MultiNodeSpec(ReplicatorORSetDeltaSpec) with STMultiNodeSpec with ImplicitSender {
+  import Replicator._
+  import ReplicatorORSetDeltaSpec._
+
+  override def initialParticipants = roles.size
+
+  implicit val cluster = Cluster(system)
+  val replicator = system.actorOf(Replicator.props(
+    ReplicatorSettings(system).withGossipInterval(1.second)), "replicator")
+  val timeout = 3.seconds.dilated
+
+  val KeyA = ORSetKey[String]("A")
+  val KeyB = ORSetKey[String]("B")
+  val KeyC = ORSetKey[String]("C")
+
+  def join(from: RoleName, to: RoleName): Unit = {
+    runOn(from) {
+      cluster join node(to).address
+    }
+    enterBarrier(from.name + "-joined")
+  }
+
+  def assertValue(key: Key[ReplicatedData], expected: Any): Unit =
+    within(10.seconds) {
+      awaitAssert {
+        replicator ! Get(key, ReadLocal)
+        val value = expectMsgPF() {
+          case g @ GetSuccess(`key`, _) ⇒ g.dataValue match {
+            case c: ORSet[_] ⇒ c.elements
+          }
+        }
+        value should be(expected)
+      }
+    }
+
+  "ORSet delta" must {
+
+    "replicate data in initial phase" in {
+      join(first, first)
+      join(second, first)
+      join(third, first)
+
+      replicator ! Replicator.Internal.TestFullStateGossip(enabled = false)
+
+      within(10.seconds) {
+        awaitAssert {
+          replicator ! GetReplicaCount
+          expectMsg(ReplicaCount(3))
+        }
+      }
+
+      runOn(first) {
+        replicator ! Update(KeyA, ORSet.empty[String], WriteLocal)(_ + "a")
+        expectMsg(UpdateSuccess(KeyA, None))
+      }
+
+      enterBarrier("initial-updates-done")
+
+      assertValue(KeyA, Set("a"))
+
+      enterBarrier("after-1")
+    }
+
+    "be propagated with causal consistency during network split" in {
+      runOn(first) {
+        // third is isolated
+        testConductor.blackhole(first, third, Direction.Both).await
+        testConductor.blackhole(second, third, Direction.Both).await
+      }
+      enterBarrier("split")
+
+      runOn(first) {
+        replicator ! Update(KeyA, ORSet.empty[String], WriteLocal)(_ + "b")
+        expectMsg(UpdateSuccess(KeyA, None))
+      }
+      runOn(second) {
+        replicator ! Update(KeyA, ORSet.empty[String], WriteLocal)(_ + "d")
+        expectMsg(UpdateSuccess(KeyA, None))
+      }
+      runOn(first, second) {
+        assertValue(KeyA, Set("a", "b", "d"))
+        Thread.sleep(2000) // all deltas sent
+      }
+      enterBarrier("added-b-and-d")
+
+      runOn(first) {
+        testConductor.passThrough(first, third, Direction.Both).await
+        testConductor.passThrough(second, third, Direction.Both).await
+      }
+      enterBarrier("healed")
+
+      runOn(first) {
+        // delta for "c" will be sent to third, but it has not received the previous delta for "b"
+        replicator ! Update(KeyA, ORSet.empty[String], WriteLocal)(_ + "c")
+        expectMsg(UpdateSuccess(KeyA, None))
+        // let the delta be propagated (will not fail if it takes longer)
+        Thread.sleep(1000)
+      }
+      enterBarrier("added-c")
+
+      runOn(first, second) {
+        assertValue(KeyA, Set("a", "b", "c", "d"))
+      }
+      runOn(third) {
+        // the delta for "c" should not be applied because it has not received previous delta for "b"
+        // and full gossip is turned off so far
+        assertValue(KeyA, Set("a"))
+      }
+      enterBarrier("verified-before-full-gossip")
+
+      replicator ! Replicator.Internal.TestFullStateGossip(enabled = true)
+      assertValue(KeyA, Set("a", "b", "c", "d"))
+      enterBarrier("verified-after-full-gossip")
+
+      replicator ! Replicator.Internal.TestFullStateGossip(enabled = false)
+
+      // and now the delta seqNr should be in sync again
+      runOn(first) {
+        replicator ! Update(KeyA, ORSet.empty[String], WriteLocal)(_ + "e")
+        expectMsg(UpdateSuccess(KeyA, None))
+      }
+      assertValue(KeyA, Set("a", "b", "c", "d", "e"))
+
+      enterBarrier("after-2")
+    }
+
+  }
+
+}
+