Merge pull request #22508 from gosubpl/wip/22350-delta-crdt-orset-friends
ORMap and friends are delta-CRDTs (#22350)
This commit is contained in:
Patrik Nordwall
GitHub
commit
edee6ae544
15 changed files with 4371 additions and 140 deletions
|
|
@ -0,0 +1,281 @@
|
|||
/**
|
||||
* Copyright (C) 2009-2016 Lightbend Inc. <http://www.lightbend.com>
|
||||
*/
|
||||
package akka.cluster.ddata
|
||||
|
||||
import java.util.concurrent.ThreadLocalRandom
|
||||
|
||||
import scala.concurrent.duration._
|
||||
|
||||
import akka.cluster.Cluster
|
||||
import akka.cluster.ddata.Replicator._
|
||||
import akka.remote.testconductor.RoleName
|
||||
import akka.remote.testkit.MultiNodeConfig
|
||||
import akka.remote.testkit.MultiNodeSpec
|
||||
import akka.testkit._
|
||||
import com.typesafe.config.ConfigFactory
|
||||
|
||||
object ReplicatorMapDeltaSpec extends MultiNodeConfig {
|
||||
val first = role("first")
|
||||
val second = role("second")
|
||||
val third = role("third")
|
||||
val fourth = role("fourth")
|
||||
|
||||
commonConfig(ConfigFactory.parseString("""
|
||||
akka.loglevel = DEBUG
|
||||
akka.actor.provider = "cluster"
|
||||
akka.log-dead-letters-during-shutdown = off
|
||||
"""))
|
||||
|
||||
testTransport(on = true)
|
||||
|
||||
sealed trait Op
|
||||
final case class Delay(n: Int) extends Op
|
||||
final case class Incr(ki: (PNCounterMapKey[String], String), n: Int, consistency: WriteConsistency) extends Op
|
||||
final case class Decr(ki: (PNCounterMapKey[String], String), n: Int, consistency: WriteConsistency) extends Op
|
||||
final case class Add(ki: (ORMultiMapKey[String, String], String), elem: String, consistency: WriteConsistency) extends Op
|
||||
final case class Remove(ki: (ORMultiMapKey[String, String], String), elem: String, consistency: WriteConsistency) extends Op
|
||||
|
||||
val timeout = 5.seconds
|
||||
val writeTwo = WriteTo(2, timeout)
|
||||
val writeMajority = WriteMajority(timeout)
|
||||
|
||||
val KeyPN = PNCounterMapKey[String]("A")
|
||||
val KeyMM = ORMultiMapKey[String, String]("D")
|
||||
val KeyA: (PNCounterMapKey[String], String) = (KeyPN, "a")
|
||||
val KeyB: (PNCounterMapKey[String], String) = (KeyPN, "b")
|
||||
val KeyC: (PNCounterMapKey[String], String) = (KeyPN, "c")
|
||||
val KeyD: (ORMultiMapKey[String, String], String) = (KeyMM, "d")
|
||||
val KeyE: (ORMultiMapKey[String, String], String) = (KeyMM, "e")
|
||||
val KeyF: (ORMultiMapKey[String, String], String) = (KeyMM, "f")
|
||||
|
||||
def generateOperations(onNode: RoleName): Vector[Op] = {
|
||||
val rnd = ThreadLocalRandom.current()
|
||||
|
||||
def consistency(): WriteConsistency = {
|
||||
rnd.nextInt(100) match {
|
||||
case n if n < 90 ⇒ WriteLocal
|
||||
case n if n < 95 ⇒ writeTwo
|
||||
case n if n < 100 ⇒ writeMajority
|
||||
}
|
||||
}
|
||||
|
||||
def rndPnCounterkey(): (PNCounterMapKey[String], String) = {
|
||||
rnd.nextInt(3) match {
|
||||
case 0 ⇒ KeyA
|
||||
case 1 ⇒ KeyB
|
||||
case 2 ⇒ KeyC
|
||||
}
|
||||
}
|
||||
|
||||
def rndOrSetkey(): (ORMultiMapKey[String, String], String) = {
|
||||
rnd.nextInt(3) match {
|
||||
case 0 ⇒ KeyD
|
||||
case 1 ⇒ KeyE
|
||||
case 2 ⇒ KeyF
|
||||
}
|
||||
}
|
||||
|
||||
var availableForRemove = Set.empty[String]
|
||||
|
||||
def rndAddElement(): String = {
|
||||
// lower case a - j
|
||||
val s = (97 + rnd.nextInt(10)).toChar.toString
|
||||
availableForRemove += s
|
||||
s
|
||||
}
|
||||
|
||||
def rndRemoveElement(): String = {
|
||||
if (availableForRemove.isEmpty)
|
||||
"a"
|
||||
else
|
||||
availableForRemove.toVector(rnd.nextInt(availableForRemove.size))
|
||||
}
|
||||
|
||||
(0 to (30 + rnd.nextInt(10))).map { _ ⇒
|
||||
rnd.nextInt(4) match {
|
||||
case 0 ⇒ Delay(rnd.nextInt(500))
|
||||
case 1 ⇒ Incr(rndPnCounterkey(), rnd.nextInt(100), consistency())
|
||||
case 2 ⇒ Decr(rndPnCounterkey(), rnd.nextInt(10), consistency())
|
||||
case 3 ⇒
|
||||
// ORSet
|
||||
val key = rndOrSetkey()
|
||||
// only removals for KeyF on node first
|
||||
if (key == KeyF && onNode == first && rnd.nextBoolean())
|
||||
Remove(key, rndRemoveElement(), consistency())
|
||||
else
|
||||
Add(key, rndAddElement(), consistency())
|
||||
}
|
||||
}.toVector
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class ReplicatorMapDeltaSpecMultiJvmNode1 extends ReplicatorMapDeltaSpec
|
||||
class ReplicatorMapDeltaSpecMultiJvmNode2 extends ReplicatorMapDeltaSpec
|
||||
class ReplicatorMapDeltaSpecMultiJvmNode3 extends ReplicatorMapDeltaSpec
|
||||
class ReplicatorMapDeltaSpecMultiJvmNode4 extends ReplicatorMapDeltaSpec
|
||||
|
||||
class ReplicatorMapDeltaSpec extends MultiNodeSpec(ReplicatorMapDeltaSpec) with STMultiNodeSpec with ImplicitSender {
|
||||
import Replicator._
|
||||
import ReplicatorMapDeltaSpec._
|
||||
|
||||
override def initialParticipants = roles.size
|
||||
|
||||
implicit val cluster = Cluster(system)
|
||||
val fullStateReplicator = system.actorOf(Replicator.props(
|
||||
ReplicatorSettings(system).withGossipInterval(1.second).withDeltaCrdtEnabled(false)), "fullStateReplicator")
|
||||
val deltaReplicator = {
|
||||
val r = system.actorOf(Replicator.props(ReplicatorSettings(system)), "deltaReplicator")
|
||||
r ! Replicator.Internal.TestFullStateGossip(enabled = false)
|
||||
r
|
||||
}
|
||||
|
||||
var afterCounter = 0
|
||||
def enterBarrierAfterTestStep(): Unit = {
|
||||
afterCounter += 1
|
||||
enterBarrier("after-" + afterCounter)
|
||||
}
|
||||
|
||||
def join(from: RoleName, to: RoleName): Unit = {
|
||||
runOn(from) {
|
||||
cluster join node(to).address
|
||||
}
|
||||
enterBarrier(from.name + "-joined")
|
||||
}
|
||||
|
||||
"delta-CRDT" must {
|
||||
"join cluster" in {
|
||||
join(first, first)
|
||||
join(second, first)
|
||||
join(third, first)
|
||||
join(fourth, first)
|
||||
|
||||
within(15.seconds) {
|
||||
awaitAssert {
|
||||
fullStateReplicator ! GetReplicaCount
|
||||
expectMsg(ReplicaCount(4))
|
||||
}
|
||||
}
|
||||
|
||||
enterBarrierAfterTestStep()
|
||||
}
|
||||
|
||||
"propagate delta" in {
|
||||
join(first, first)
|
||||
join(second, first)
|
||||
join(third, first)
|
||||
join(fourth, first)
|
||||
|
||||
within(15.seconds) {
|
||||
awaitAssert {
|
||||
fullStateReplicator ! GetReplicaCount
|
||||
expectMsg(ReplicaCount(4))
|
||||
}
|
||||
}
|
||||
enterBarrier("ready")
|
||||
|
||||
runOn(first) {
|
||||
// by setting something for each key we don't have to worry about NotFound
|
||||
List(KeyA, KeyB, KeyC).foreach { key ⇒
|
||||
fullStateReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ increment key._2)
|
||||
deltaReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ increment key._2)
|
||||
}
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
fullStateReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ + (key._2, Set("a")))
|
||||
deltaReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ + (key._2, Set("a")))
|
||||
}
|
||||
}
|
||||
enterBarrier("updated-1")
|
||||
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
List(KeyA, KeyB, KeyC).foreach { key ⇒
|
||||
fullStateReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[PNCounterMap[String]]].dataValue.get(key._2).get.intValue should be(1)
|
||||
}
|
||||
}
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
fullStateReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
p.expectMsgType[GetSuccess[ORMultiMap[String, String]]].dataValue.get(key._2) should ===(Some(Set("a")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enterBarrierAfterTestStep()
|
||||
}
|
||||
|
||||
"be eventually consistent" in {
|
||||
val operations = generateOperations(onNode = myself)
|
||||
log.debug(s"random operations on [${myself.name}]: ${operations.mkString(", ")}")
|
||||
try {
|
||||
// perform random operations with both delta and full-state replicators
|
||||
// and compare that the end result is the same
|
||||
|
||||
for (op ← operations) {
|
||||
log.debug("operation: {}", op)
|
||||
op match {
|
||||
case Delay(d) ⇒ Thread.sleep(d)
|
||||
case Incr(key, n, consistency) ⇒
|
||||
fullStateReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ increment (key._2, n))
|
||||
deltaReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ increment (key._2, n))
|
||||
case Decr(key, n, consistency) ⇒
|
||||
fullStateReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ decrement (key._2, n))
|
||||
deltaReplicator ! Update(key._1, PNCounterMap.empty[String], WriteLocal)(_ decrement (key._2, n))
|
||||
case Add(key, elem, consistency) ⇒
|
||||
// to have an deterministic result when mixing add/remove we can only perform
|
||||
// the ORSet operations from one node
|
||||
runOn((if (key == KeyF) List(first) else List(first, second, third)): _*) {
|
||||
fullStateReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ addBinding (key._2, elem))
|
||||
deltaReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ addBinding (key._2, elem))
|
||||
}
|
||||
case Remove(key, elem, consistency) ⇒
|
||||
runOn(first) {
|
||||
fullStateReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ removeBinding (key._2, elem))
|
||||
deltaReplicator ! Update(key._1, ORMultiMap.emptyWithValueDeltas[String, String], WriteLocal)(_ removeBinding (key._2, elem))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enterBarrier("updated-2")
|
||||
|
||||
List(KeyA, KeyB, KeyC).foreach { key ⇒
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
fullStateReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
val fullStateValue = p.expectMsgType[GetSuccess[PNCounterMap[String]]].dataValue.get(key._2).get.intValue
|
||||
deltaReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
val deltaValue = p.expectMsgType[GetSuccess[PNCounterMap[String]]].dataValue.get(key._2).get.intValue
|
||||
deltaValue should ===(fullStateValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List(KeyD, KeyE, KeyF).foreach { key ⇒
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
val p = TestProbe()
|
||||
fullStateReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
val fullStateValue = p.expectMsgType[GetSuccess[ORMultiMap[String, String]]].dataValue.get(key._2)
|
||||
deltaReplicator.tell(Get(key._1, ReadLocal), p.ref)
|
||||
val deltaValue = p.expectMsgType[GetSuccess[ORMultiMap[String, String]]].dataValue.get(key._2)
|
||||
deltaValue should ===(fullStateValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enterBarrierAfterTestStep()
|
||||
} catch {
|
||||
case e: Throwable ⇒
|
||||
info(s"random operations on [${myself.name}]: ${operations.mkString(", ")}")
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -47,6 +47,8 @@ class ReplicatorPruningSpec extends MultiNodeSpec(ReplicatorPruningSpec) with ST
|
|||
val KeyA = GCounterKey("A")
|
||||
val KeyB = ORSetKey[String]("B")
|
||||
val KeyC = PNCounterMapKey[String]("C")
|
||||
val KeyD = ORMultiMapKey[String, String]("D")
|
||||
val KeyE = ORMapKey[String, GSet[String]]("E")
|
||||
|
||||
def join(from: RoleName, to: RoleName): Unit = {
|
||||
runOn(from) {
|
||||
|
|
@ -89,6 +91,12 @@ class ReplicatorPruningSpec extends MultiNodeSpec(ReplicatorPruningSpec) with ST
|
|||
replicator ! Update(KeyC, PNCounterMap.empty[String], WriteAll(timeout)) { _ increment "x" increment "y" }
|
||||
expectMsg(UpdateSuccess(KeyC, None))
|
||||
|
||||
replicator ! Update(KeyD, ORMultiMap.empty[String, String], WriteAll(timeout)) { _ + ("a", Set("A")) }
|
||||
expectMsg(UpdateSuccess(KeyD, None))
|
||||
|
||||
replicator ! Update(KeyE, ORMap.empty[String, GSet[String]], WriteAll(timeout)) { _ + ("a", GSet.empty[String].add("A")) }
|
||||
expectMsg(UpdateSuccess(KeyE, None))
|
||||
|
||||
enterBarrier("updates-done")
|
||||
|
||||
replicator ! Get(KeyA, ReadLocal)
|
||||
|
|
@ -104,8 +112,24 @@ class ReplicatorPruningSpec extends MultiNodeSpec(ReplicatorPruningSpec) with ST
|
|||
oldMap.get("x") should be(Some(3))
|
||||
oldMap.get("y") should be(Some(3))
|
||||
|
||||
replicator ! Get(KeyD, ReadLocal)
|
||||
val oldMultiMap = expectMsgType[GetSuccess[ORMultiMap[String, String]]].dataValue
|
||||
oldMultiMap.get("a") should be(Some(Set("A")))
|
||||
|
||||
replicator ! Get(KeyE, ReadLocal)
|
||||
val oldORMap = expectMsgType[GetSuccess[ORMap[String, GSet[String]]]].dataValue
|
||||
val GSet(d) = oldORMap.entries("a")
|
||||
d should be(Set("A"))
|
||||
|
||||
enterBarrier("get-old")
|
||||
|
||||
runOn(third) {
|
||||
replicator ! Update(KeyE, ORMap.empty[String, GSet[String]], WriteLocal) { _ - "a" }
|
||||
expectMsg(UpdateSuccess(KeyE, None))
|
||||
}
|
||||
|
||||
enterBarrier("remove-element")
|
||||
|
||||
runOn(first) {
|
||||
cluster.leave(node(third).address)
|
||||
}
|
||||
|
|
@ -155,6 +179,25 @@ class ReplicatorPruningSpec extends MultiNodeSpec(ReplicatorPruningSpec) with ST
|
|||
}
|
||||
}
|
||||
}
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
replicator ! Get(KeyD, ReadLocal)
|
||||
expectMsgPF() {
|
||||
case g @ GetSuccess(KeyD, _) ⇒
|
||||
g.get(KeyD).entries("a") should be(Set("A"))
|
||||
g.get(KeyD).needPruningFrom(thirdUniqueAddress) should be(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
within(5.seconds) {
|
||||
awaitAssert {
|
||||
replicator ! Get(KeyE, ReadLocal)
|
||||
expectMsgPF() {
|
||||
case g @ GetSuccess(KeyE, _) ⇒
|
||||
g.get(KeyE).needPruningFrom(thirdUniqueAddress) should be(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
enterBarrier("pruning-done")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue