raboof/pekko
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions 11

Improve performance of DData delta updates, #25310

Browse source 
* Use deterministic order of the target nodes for the writes when
  type RequiresCausalDeliveryOfDeltas, otherwise the random pick
  of targets caused that delta sequence numbers were missing for
  susequent updates
* Resend immediately when receiving DeltaNack instead of waiting
  for timeout. DeltaNack can happen when there are multiple
  concurrent updates from same node because each starts a WriteAggregator
  and a later Update might bypass an earlier
This commit is contained in:
Patrik Nordwall 2018-07-11 13:14:25 +02:00 • committed by Johan Andrén
parent db48514b88
commit 180ef934bc
3 changed files with 59 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

29
akka-distributed-data/src/main/scala/akka/cluster/ddata/Replicator.scala
View file

@ -1948,13 +1948,17 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
def doneWhenRemainingSize: Int
lazy val (primaryNodes, secondaryNodes) = {
def primaryAndSecondaryNodes(requiresCausalDeliveryOfDeltas: Boolean): (Vector[Address], Vector[Address]) = {
val primarySize = nodes.size - doneWhenRemainingSize
if (primarySize >= nodes.size)
(nodes, Set.empty[Address])
(nodes.toVector, Vector.empty[Address])
else {
// Prefer to use reachable nodes over the unreachable nodes first
val orderedNodes = scala.util.Random.shuffle(reachableNodes.toVector) ++ scala.util.Random.shuffle(unreachable.toVector)
// Prefer to use reachable nodes over the unreachable nodes first.
// When RequiresCausalDeliveryOfDeltas use deterministic order to so that sequence numbers of subsequent
// updates are in sync on the destination nodes.
val orderedNodes =
if (requiresCausalDeliveryOfDeltas) reachableNodes.toVector.sorted ++ unreachable.toVector.sorted
else scala.util.Random.shuffle(reachableNodes.toVector) ++ scala.util.Random.shuffle(unreachable.toVector)
val (p, s) = orderedNodes.splitAt(primarySize)
(p, s.take(MaxSecondaryNodes))
}
@ -2030,6 +2034,14 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
var gotLocalStoreReply = !durable
var gotWriteNackFrom = Set.empty[Address]
private val (primaryNodes, secondaryNodes) = {
val requiresCausalDeliveryOfDeltas = delta match {
case None ⇒ false
case Some(d) ⇒ d.dataEnvelope.data.isInstanceOf[RequiresCausalDeliveryOfDeltas]
}
primaryAndSecondaryNodes(requiresCausalDeliveryOfDeltas)
}
override def preStart(): Unit = {
val msg = deltaMsg match {
case Some(d) ⇒ d
@ -2048,7 +2060,10 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
gotWriteNackFrom += senderAddress()
if (isDone) reply(isTimeout = false)
case DeltaNack ⇒
// ok, will be retried with full state
// Deltas must be applied in order and we can't keep track of ordering of
// simultaneous updates so there is a chance that the delta could not be applied.
// Try again with the full state
sender() ! writeMsg
case _: Replicator.UpdateSuccess[_] ⇒
gotLocalStoreReply = true
@ -2148,6 +2163,10 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
val readMsg = Read(key.id)
private val (primaryNodes, secondaryNodes) = {
primaryAndSecondaryNodes(requiresCausalDeliveryOfDeltas = false)
}
override def preStart(): Unit = {
primaryNodes.foreach { replica(_) ! readMsg }