causal delivery of deltas, #22188
* keep track of delta interval versions and skip deltas that are not consequtive, i.e. when some delta message was lost * send the delta versions in the full state gossip to sync up the expected versions after dropped deltas * implementation of deltas for ORSet * refactoring of the delta types to allow for different type for the delta and the full state * extensive tests * mima filter * performance optimizations * simple pruning of deltas * Java API * update documentation * KeyId type alias * Use InternalApi annotation
This commit is contained in:
Patrik Nordwall
parent
94afbee179
commit
b700b840d1
41 changed files with 5010 additions and 1950 deletions
|
|
@ -22,6 +22,7 @@ object ReplicatorDeltaSpec extends MultiNodeConfig {
|
|||
val fourth = role("fourth")
|
||||
|
||||
commonConfig(ConfigFactory.parseString("""
|
||||
akka.loglevel = DEBUG
|
||||
akka.actor.provider = "cluster"
|
||||
akka.log-dead-letters-during-shutdown = off
|
||||
"""))
|
||||
|
|
@ -32,6 +33,8 @@ object ReplicatorDeltaSpec extends MultiNodeConfig {
|
|||
final case class Delay(n: Int) extends Op
|
||||
final case class Incr(key: PNCounterKey, n: Int, consistency: WriteConsistency) extends Op
|
||||
final case class Decr(key: PNCounterKey, n: Int, consistency: WriteConsistency) extends Op
|
||||
final case class Add(key: ORSetKey[String], elem: String, consistency: WriteConsistency) extends Op
|
||||
final case class Remove(key: ORSetKey[String], elem: String, consistency: WriteConsistency) extends Op
|
||||
|
||||
val timeout = 5.seconds
|
||||
val writeTwo = WriteTo(2, timeout)
|
||||
|
|
@ -40,8 +43,11 @@ object ReplicatorDeltaSpec extends MultiNodeConfig {
|
|||
val KeyA = PNCounterKey("A")
|
||||
val KeyB = PNCounterKey("B")
|
||||
val KeyC = PNCounterKey("C")
|
||||
val KeyD = ORSetKey[String]("D")
|
||||
val KeyE = ORSetKey[String]("E")
|
||||
val KeyF = ORSetKey[String]("F")
|
||||
|
||||
def generateOperations(): Vector[Op] = {
|
||||
def generateOperations(onNode: RoleName): Vector[Op] = {
|
||||
val rnd = ThreadLocalRandom.current()
|
||||
|
||||
def consistency(): WriteConsistency = {
|
||||
|
|
@ -52,7 +58,7 @@ object ReplicatorDeltaSpec extends MultiNodeConfig {
|
|||
}
|
||||
}
|
||||
|
||||
def key(): PNCounterKey = {
|
||||
def rndPnCounterkey(): PNCounterKey = {
|
||||
rnd.nextInt(3) match {
|
||||
case 0 ⇒ KeyA
|
||||
case 1 ⇒ KeyB
|
||||
|
|
@ -60,11 +66,43 @@ object ReplicatorDeltaSpec extends MultiNodeConfig {
|
|||
}
|
||||
}
|
||||
|
||||
(0 to (20 + rnd.nextInt(10))).map { _ ⇒
|
||||
def rndOrSetkey(): ORSetKey[String] = {
|
||||
rnd.nextInt(3) match {
|
||||
case 0 ⇒ KeyD
|
||||
case 1 ⇒ KeyE
|
||||
case 2 ⇒ KeyF
|
||||
}
|
||||
}
|
||||
|
||||
var availableForRemove = Set.empty[String]
|
||||
|
||||
def rndAddElement(): String = {
|
||||
// lower case a - j
|
||||
val s = (97 + rnd.nextInt(10)).toChar.toString
|
||||
availableForRemove += s
|
||||
s
|
||||
}
|
||||
|
||||
def rndRemoveElement(): String = {
|
||||
if (availableForRemove.isEmpty)
|
||||
"a"
|
||||
else
|
||||
availableForRemove.toVector(rnd.nextInt(availableForRemove.size))
|
||||
}
|
||||
|
||||
(0 to (30 + rnd.nextInt(10))).map { _ ⇒
|
||||
rnd.nextInt(4) match {
|
||||
case 0 ⇒ Delay(rnd.nextInt(500))
|
||||
case 1 ⇒ Incr(key(), rnd.nextInt(100), consistency())
|
||||
case 2 ⇒ Decr(key(), rnd.nextInt(10), consistency())
|
||||
case 1 ⇒ Incr(rndPnCounterkey(), rnd.nextInt(100), consistency())
|
||||
case 2 ⇒ Decr(rndPnCounterkey(), rnd.nextInt(10), consistency())
|
||||
case 3 ⇒
|
||||
// ORSet
|
||||
val key = rndOrSetkey()
|
||||
// only removals for KeyF on node first
|
||||
if (key == KeyF && onNode == first && rnd.nextBoolean())
|
||||
Remove(key, rndRemoveElement(), consistency())
|
||||
else
|
||||
Add(key, rndAddElement(), consistency())
|
||||
}
|
||||
}.toVector
|
||||
}
|
||||
|
|
@ -136,21 +174,32 @@ class ReplicatorDeltaSpec extends MultiNodeSpec(ReplicatorDeltaSpec) with STMult
|
|||
enterBarrier("ready")
|
||||
|
||||
runOn(first) {
|
||||
fullStateReplicator ! Update(KeyA, PNCounter.empty, WriteLocal)(_ + 1)
|
||||
deltaReplicator ! Update(KeyA, PNCounter.empty, WriteLocal)(_ + 1)
|
||||
// by setting something for each key we don't have to worry about NotFound
|
||||
List(KeyA, KeyB, KeyC).foreach { key ⇒
|
||||
fullStateReplicator ! Update(key, PNCounter.empty, WriteLocal)(_ + 1)
|
||||
deltaReplicator ! Update(key, PNCounter.empty, WriteLocal)(_ + 1)
|
||||
}
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
fullStateReplicator ! Update(key, ORSet.empty[String], WriteLocal)(_ + "a")
|
||||
deltaReplicator ! Update(key, ORSet.empty[String], WriteLocal)(_ + "a")
|
||||
}
|
||||
}
|
||||
enterBarrier("updated-1")
|
||||
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
deltaReplicator.tell(Get(KeyA, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[PNCounter]].dataValue.getValue.intValue should be(1)
|
||||
List(KeyA, KeyB, KeyC).foreach { key ⇒
|
||||
fullStateReplicator.tell(Get(key, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[PNCounter]].dataValue.getValue.intValue should be(1)
|
||||
}
|
||||
}
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
deltaReplicator.tell(Get(KeyA, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[PNCounter]].dataValue.getValue.intValue should be(1)
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
fullStateReplicator.tell(Get(key, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[ORSet[String]]].dataValue.elements should ===(Set("a"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -158,7 +207,7 @@ class ReplicatorDeltaSpec extends MultiNodeSpec(ReplicatorDeltaSpec) with STMult
|
|||
}
|
||||
|
||||
"be eventually consistent" in {
|
||||
val operations = generateOperations()
|
||||
val operations = generateOperations(onNode = myself)
|
||||
log.debug(s"random operations on [${myself.name}]: ${operations.mkString(", ")}")
|
||||
try {
|
||||
// perform random operations with both delta and full-state replicators
|
||||
|
|
@ -170,10 +219,22 @@ class ReplicatorDeltaSpec extends MultiNodeSpec(ReplicatorDeltaSpec) with STMult
|
|||
case Delay(d) ⇒ Thread.sleep(d)
|
||||
case Incr(key, n, consistency) ⇒
|
||||
fullStateReplicator ! Update(key, PNCounter.empty, consistency)(_ + n)
|
||||
deltaReplicator ! Update(key, PNCounter.empty, WriteLocal)(_ + n)
|
||||
deltaReplicator ! Update(key, PNCounter.empty, consistency)(_ + n)
|
||||
case Decr(key, n, consistency) ⇒
|
||||
fullStateReplicator ! Update(key, PNCounter.empty, consistency)(_ - n)
|
||||
deltaReplicator ! Update(key, PNCounter.empty, WriteLocal)(_ - n)
|
||||
deltaReplicator ! Update(key, PNCounter.empty, consistency)(_ - n)
|
||||
case Add(key, elem, consistency) ⇒
|
||||
// to have an deterministic result when mixing add/remove we can only perform
|
||||
// the ORSet operations from one node
|
||||
runOn((if (key == KeyF) List(first) else List(first, second, third)): _*) {
|
||||
fullStateReplicator ! Update(key, ORSet.empty[String], consistency)(_ + elem)
|
||||
deltaReplicator ! Update(key, ORSet.empty[String], consistency)(_ + elem)
|
||||
}
|
||||
case Remove(key, elem, consistency) ⇒
|
||||
runOn(first) {
|
||||
fullStateReplicator ! Update(key, ORSet.empty[String], consistency)(_ - elem)
|
||||
deltaReplicator ! Update(key, ORSet.empty[String], consistency)(_ - elem)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -192,6 +253,19 @@ class ReplicatorDeltaSpec extends MultiNodeSpec(ReplicatorDeltaSpec) with STMult
|
|||
}
|
||||
}
|
||||
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
fullStateReplicator.tell(Get(key, ReadLocal), p.ref)
|
||||
val fullStateValue = p.expectMsgType[GetSuccess[ORSet[String]]].dataValue.elements
|
||||
deltaReplicator.tell(Get(key, ReadLocal), p.ref)
|
||||
val deltaValue = p.expectMsgType[GetSuccess[ORSet[String]]].dataValue.elements
|
||||
deltaValue should ===(fullStateValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enterBarrierAfterTestStep()
|
||||
} catch {
|
||||
case e: Throwable ⇒
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue