causal delivery of deltas, #22188
* keep track of delta interval versions and skip deltas that are not consequtive, i.e. when some delta message was lost * send the delta versions in the full state gossip to sync up the expected versions after dropped deltas * implementation of deltas for ORSet * refactoring of the delta types to allow for different type for the delta and the full state * extensive tests * mima filter * performance optimizations * simple pruning of deltas * Java API * update documentation * KeyId type alias * Use InternalApi annotation
This commit is contained in:
Patrik Nordwall
parent
94afbee179
commit
b700b840d1
41 changed files with 5010 additions and 1950 deletions
|
|
@ -14,17 +14,17 @@ import org.scalatest.WordSpec
|
|||
|
||||
class ORSetSpec extends WordSpec with Matchers {
|
||||
|
||||
val node1 = UniqueAddress(Address("akka.tcp", "Sys", "localhost", 2551), 1)
|
||||
val node2 = UniqueAddress(node1.address.copy(port = Some(2552)), 2)
|
||||
val node1 = UniqueAddress(Address("akka.tcp", "Sys", "localhost", 2551), 1L)
|
||||
val node2 = UniqueAddress(node1.address.copy(port = Some(2552)), 2L)
|
||||
|
||||
val nodeA = UniqueAddress(Address("akka.tcp", "Sys", "a", 2552), 1)
|
||||
val nodeB = UniqueAddress(nodeA.address.copy(host = Some("b")), 2)
|
||||
val nodeC = UniqueAddress(nodeA.address.copy(host = Some("c")), 3)
|
||||
val nodeD = UniqueAddress(nodeA.address.copy(host = Some("d")), 4)
|
||||
val nodeE = UniqueAddress(nodeA.address.copy(host = Some("e")), 5)
|
||||
val nodeF = UniqueAddress(nodeA.address.copy(host = Some("f")), 6)
|
||||
val nodeG = UniqueAddress(nodeA.address.copy(host = Some("g")), 7)
|
||||
val nodeH = UniqueAddress(nodeA.address.copy(host = Some("h")), 8)
|
||||
val nodeA = UniqueAddress(Address("akka.tcp", "Sys", "a", 2552), 1L)
|
||||
val nodeB = UniqueAddress(nodeA.address.copy(host = Some("b")), 2L)
|
||||
val nodeC = UniqueAddress(nodeA.address.copy(host = Some("c")), 3L)
|
||||
val nodeD = UniqueAddress(nodeA.address.copy(host = Some("d")), 4L)
|
||||
val nodeE = UniqueAddress(nodeA.address.copy(host = Some("e")), 5L)
|
||||
val nodeF = UniqueAddress(nodeA.address.copy(host = Some("f")), 6L)
|
||||
val nodeG = UniqueAddress(nodeA.address.copy(host = Some("g")), 7L)
|
||||
val nodeH = UniqueAddress(nodeA.address.copy(host = Some("h")), 8L)
|
||||
|
||||
val user1 = """{"username":"john","password":"coltrane"}"""
|
||||
val user2 = """{"username":"sonny","password":"rollins"}"""
|
||||
|
|
@ -226,6 +226,185 @@ class ORSetSpec extends WordSpec with Matchers {
|
|||
|
||||
}
|
||||
|
||||
"ORSet deltas" must {
|
||||
|
||||
def addDeltaOp(s: ORSet[String]): ORSet.AddDeltaOp[String] =
|
||||
asAddDeltaOp(s.delta.get)
|
||||
|
||||
def asAddDeltaOp(delta: Any): ORSet.AddDeltaOp[String] =
|
||||
delta match {
|
||||
case d: ORSet.AddDeltaOp[String] @unchecked ⇒ d
|
||||
case _ ⇒ throw new IllegalArgumentException("Expected AddDeltaOp")
|
||||
}
|
||||
|
||||
"work for additions" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.add(node1, "a")
|
||||
addDeltaOp(s2).underlying.elements should ===(Set("a"))
|
||||
s1.mergeDelta(s2.delta.get) should ===(s2)
|
||||
|
||||
val s3 = s2.resetDelta.add(node1, "b").add(node1, "c")
|
||||
addDeltaOp(s3).underlying.elements should ===(Set("b", "c"))
|
||||
s2.mergeDelta(s3.delta.get) should ===(s3)
|
||||
|
||||
// another node adds "d"
|
||||
val s4 = s3.resetDelta.add(node2, "d")
|
||||
addDeltaOp(s4).underlying.elements should ===(Set("d"))
|
||||
s3.mergeDelta(s4.delta.get) should ===(s4)
|
||||
|
||||
// concurrent update
|
||||
val s5 = s3.resetDelta.add(node1, "e")
|
||||
val s6 = s5.merge(s4)
|
||||
s5.mergeDelta(s4.delta.get) should ===(s6)
|
||||
|
||||
// concurrent add of same element
|
||||
val s7 = s3.resetDelta.add(node1, "d")
|
||||
val s8 = s7.merge(s4)
|
||||
// the dot contains both nodes
|
||||
s8.elementsMap("d").contains(node1)
|
||||
s8.elementsMap("d").contains(node2)
|
||||
// and same result when merging the deltas
|
||||
s7.mergeDelta(s4.delta.get) should ===(s8)
|
||||
s4.mergeDelta(s7.delta.get) should ===(s8)
|
||||
}
|
||||
|
||||
"handle another concurrent add scenario" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.add(node1, "a")
|
||||
val s3 = s2.add(node1, "b")
|
||||
val s4 = s2.add(node2, "c")
|
||||
|
||||
// full state merge for reference
|
||||
val s5 = s4.merge(s3)
|
||||
s5.elements should ===(Set("a", "b", "c"))
|
||||
|
||||
val s6 = s4.mergeDelta(s3.delta.get)
|
||||
s6.elements should ===(Set("a", "b", "c"))
|
||||
}
|
||||
|
||||
"merge deltas into delta groups" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.add(node1, "a")
|
||||
val d2 = s2.delta.get
|
||||
val s3 = s2.resetDelta.add(node1, "b")
|
||||
val d3 = s3.delta.get
|
||||
val d4 = d2 merge d3
|
||||
asAddDeltaOp(d4).underlying.elements should ===(Set("a", "b"))
|
||||
s1.mergeDelta(d4) should ===(s3)
|
||||
s2.mergeDelta(d4) should ===(s3)
|
||||
|
||||
val s5 = s3.resetDelta.remove(node1, "b")
|
||||
val d5 = s5.delta.get
|
||||
val d6 = (d4 merge d5).asInstanceOf[ORSet.DeltaGroup[String]]
|
||||
d6.ops.last.getClass should ===(classOf[ORSet.RemoveDeltaOp[String]])
|
||||
d6.ops.size should ===(2)
|
||||
s3.mergeDelta(d6) should ===(s5)
|
||||
|
||||
val s7 = s5.resetDelta.add(node1, "c")
|
||||
val s8 = s7.resetDelta.add(node1, "d")
|
||||
val d9 = (d6 merge s7.delta.get merge s8.delta.get).asInstanceOf[ORSet.DeltaGroup[String]]
|
||||
// the add "c" and add "d" are merged into one AddDeltaOp
|
||||
asAddDeltaOp(d9.ops.last).underlying.elements should ===(Set("c", "d"))
|
||||
d9.ops.size should ===(3)
|
||||
s5.mergeDelta(d9) should ===(s8)
|
||||
s5.mergeDelta(s7.delta.get).mergeDelta(s8.delta.get) should ===(s8)
|
||||
}
|
||||
|
||||
"work for removals" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.add(node1, "a").add(node1, "b").resetDelta
|
||||
val s3 = s2.remove(node1, "b")
|
||||
s2.merge(s3) should ===(s3)
|
||||
s2.mergeDelta(s3.delta.get) should ===(s3)
|
||||
s2.mergeDelta(s3.delta.get).elements should ===(Set("a"))
|
||||
|
||||
// concurrent update
|
||||
val s4 = s2.add(node2, "c").resetDelta
|
||||
val s5 = s4.merge(s3)
|
||||
s5.elements should ===(Set("a", "c"))
|
||||
s4.mergeDelta(s3.delta.get) should ===(s5)
|
||||
|
||||
// add "b" again
|
||||
val s6 = s5.add(node2, "b")
|
||||
// merging the old delta should not remove it
|
||||
s6.mergeDelta(s3.delta.get) should ===(s6)
|
||||
s6.mergeDelta(s3.delta.get).elements should ===(Set("a", "b", "c"))
|
||||
}
|
||||
|
||||
"work for clear" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.add(node1, "a").add(node1, "b")
|
||||
val s3 = s2.resetDelta.clear(node1)
|
||||
val s4 = s3.resetDelta.add(node1, "c")
|
||||
s2.merge(s3) should ===(s3)
|
||||
s2.mergeDelta(s3.delta.get) should ===(s3)
|
||||
val s5 = s2.mergeDelta(s3.delta.get).mergeDelta(s4.delta.get)
|
||||
s5.elements should ===(Set("c"))
|
||||
s5 should ===(s4)
|
||||
|
||||
// concurrent update
|
||||
val s6 = s2.resetDelta.add(node2, "d")
|
||||
val s7 = s6.merge(s3)
|
||||
s7.elements should ===(Set("d"))
|
||||
s6.mergeDelta(s3.delta.get) should ===(s7)
|
||||
|
||||
// add "b" again
|
||||
val s8 = s7.add(node2, "b")
|
||||
// merging the old delta should not remove it
|
||||
s8.mergeDelta(s3.delta.get) should ===(s8)
|
||||
s8.mergeDelta(s3.delta.get).elements should ===(Set("b", "d"))
|
||||
}
|
||||
|
||||
"handle a mixed add/remove scenario" in {
|
||||
val s1 = ORSet.empty[String]
|
||||
val s2 = s1.resetDelta.remove(node1, "e")
|
||||
val s3 = s2.resetDelta.add(node1, "b")
|
||||
val s4 = s3.resetDelta.add(node1, "a")
|
||||
val s5 = s4.resetDelta.remove(node1, "b")
|
||||
|
||||
val deltaGroup1 = s3.delta.get merge s4.delta.get merge s5.delta.get
|
||||
|
||||
val s7 = s2 mergeDelta deltaGroup1
|
||||
s7.elements should ===(Set("a"))
|
||||
// The above scenario was constructed from failing ReplicatorDeltaSpec,
|
||||
// some more checks...
|
||||
|
||||
val s8 = s2.resetDelta.add(node2, "z") // concurrent update from node2
|
||||
val s9 = s8 mergeDelta deltaGroup1
|
||||
s9.elements should ===(Set("a", "z"))
|
||||
}
|
||||
|
||||
"require causal delivery of deltas" in {
|
||||
// This test illustrates why we need causal delivery of deltas.
|
||||
// Otherwise the following could happen.
|
||||
|
||||
// s0 is the stable state that is initially replicated to all nodes
|
||||
val s0 = ORSet.empty[String].add(node1, "a")
|
||||
|
||||
// add element "b" and "c" at node1
|
||||
val s11 = s0.resetDelta.add(node1, "b")
|
||||
val s12 = s11.resetDelta.add(node1, "c")
|
||||
|
||||
// at the same time, add element "d" at node2
|
||||
val s21 = s0.resetDelta.add(node2, "d")
|
||||
|
||||
// node3 receives delta for "d" and "c", but the delta for "b" is lost
|
||||
val s31 = s0 mergeDelta s21.delta.get mergeDelta s12.delta.get
|
||||
s31.elements should ===(Set("a", "c", "d"))
|
||||
|
||||
// node4 receives all deltas
|
||||
val s41 = s0 mergeDelta s11.delta.get mergeDelta s12.delta.get mergeDelta s21.delta.get
|
||||
s41.elements should ===(Set("a", "b", "c", "d"))
|
||||
|
||||
// node3 and node4 sync with full state gossip
|
||||
val s32 = s31 merge s41
|
||||
// one would expect elements "a", "b", "c", "d", but "b" is removed
|
||||
// because we applied s12.delta without applying s11.delta
|
||||
s32.elements should ===(Set("a", "c", "d"))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
"ORSet unit test" must {
|
||||
"verify subtractDots" in {
|
||||
val dot = VersionVector(TreeMap(nodeA → 3L, nodeB → 2L, nodeD → 14L, nodeG → 22L))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue