raboof/pekko
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions 11

Avoid false removals in ClusterReceptionist, #26284

Browse source 
* The scenario was (probably) that a node was restarted with
  same host:port and then didn't join the same cluster. The DData
  Replicator in the original cluster would continue sending messages
  to the new incarnation resulting in false removals.
* The fix is that DData Replicator includes the system uid of the sending
  or target system in messages and if recipient gets a message that is from/to
  unknown it will discard it and thereby not spreading information across
  different clusters.
* Reproduced in ClusterReceptionistSpec
* Much hardening of other things in ClusterReceptionistSpec
* There are also some improvements to ClusterReceptionist to not leak
  Listing with refs of removed nodes.
* use ClusterShuttingDown
* The reason for using sender system uid instead of target uid in messages
  like Read and Write is that then the optimization for sending same message
  to many destinations can remain.
This commit is contained in:
Patrik Nordwall 2019-02-21 09:09:20 +01:00
parent 3cbda93496
commit 825d90bf63
16 changed files with 1714 additions and 396 deletions
Download patch file Download diff file Expand all files Collapse all files

42
akka-distributed-data/src/test/scala/akka/cluster/ddata/DeltaPropagationSelectorSpec.scala
View file

@ -16,7 +16,7 @@ import org.scalatest.Matchers
import org.scalatest.WordSpec
object DeltaPropagationSelectorSpec {
class TestSelector(val selfUniqueAddress: UniqueAddress, override val allNodes: Vector[Address])
class TestSelector(val selfUniqueAddress: UniqueAddress, override val allNodes: Vector[UniqueAddress])
extends DeltaPropagationSelector {
override val gossipIntervalDivisor = 5
override def createDeltaPropagation(deltas: Map[KeyId, (ReplicatedData, Long, Long)]): DeltaPropagation =
@ -33,14 +33,14 @@ object DeltaPropagationSelectorSpec {
class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheckedTripleEquals {
import DeltaPropagationSelectorSpec._
val selfUniqueAddress = UniqueAddress(Address("akka", "Sys", "localhost", 4999), 1L)
val nodes = (2500 until 2600).map(n => Address("akka", "Sys", "localhost", n)).toVector
val selfUniqueAddress = UniqueAddress(Address("akka", "Sys", "localhost", 4999), 17L)
val nodes = (2500 until 2600).map(n => UniqueAddress(Address("akka", "Sys", "localhost", n), 17L)).toVector
"DeltaPropagationSelector" must {
"collect none when no nodes" in {
val selector = new TestSelector(selfUniqueAddress, Vector.empty)
selector.update("A", deltaA)
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("A") should ===(false)
}
@ -56,9 +56,9 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
DeltaPropagation(
selfUniqueAddress,
false,
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" -> Delta(DataEnvelope(deltaB), 1L, 1L)))
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" → Delta(DataEnvelope(deltaB), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected))
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("A") should ===(false)
selector.hasDeltaEntries("B") should ===(false)
@ -72,13 +72,13 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
DeltaPropagation(
selfUniqueAddress,
false,
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" -> Delta(DataEnvelope(deltaB), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected, nodes(1) -> expected))
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" → Delta(DataEnvelope(deltaB), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected, nodes(1) → expected))
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("A") should ===(true)
selector.hasDeltaEntries("B") should ===(true)
selector.collectPropagations() should ===(Map(nodes(2) -> expected))
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("A") should ===(false)
selector.hasDeltaEntries("B") should ===(false)
@ -92,8 +92,8 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
DeltaPropagation(
selfUniqueAddress,
false,
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" -> Delta(DataEnvelope(deltaB), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected1, nodes(1) -> expected1))
Map("A" -> Delta(DataEnvelope(deltaA), 1L, 1L), "B" → Delta(DataEnvelope(deltaB), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected1, nodes(1) → expected1))
// new update before previous was propagated to all nodes
selector.update("C", deltaC)
val expected2 = DeltaPropagation(
@ -103,14 +103,15 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
"A" -> Delta(DataEnvelope(deltaA), 1L, 1L),
"B" -> Delta(DataEnvelope(deltaB), 1L, 1L),
"C" -> Delta(DataEnvelope(deltaC), 1L, 1L)))
val expected3 = DeltaPropagation(selfUniqueAddress, false, Map("C" -> Delta(DataEnvelope(deltaC), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(2) -> expected2, nodes(0) -> expected3))
val expected3 =
DeltaPropagation(selfUniqueAddress, false, Map("C" -> Delta(DataEnvelope(deltaC), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(2) -> expected2, nodes(0) → expected3))
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("A") should ===(false)
selector.hasDeltaEntries("B") should ===(false)
selector.hasDeltaEntries("C") should ===(true)
selector.collectPropagations() should ===(Map(nodes(1) -> expected3))
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
selector.cleanupDeltaEntries()
selector.hasDeltaEntries("C") should ===(false)
}
@ -129,9 +130,10 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
selector.collectPropagations() should ===(Map(nodes(0) -> expected1))
selector.update("A", delta3)
selector.currentVersion("A") should ===(3L)
val expected2 = DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta3), 3L, 3L)))
val expected2 =
DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta3), 3L, 3L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected2))
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
}
"merge deltas" in {
@ -142,7 +144,8 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
override def nodesSliceSize(allNodesSize: Int): Int = 1
}
selector.update("A", delta1)
val expected1 = DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta1), 1L, 1L)))
val expected1 =
DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta1), 1L, 1L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected1))
selector.update("A", delta2)
@ -161,10 +164,11 @@ class DeltaPropagationSelectorSpec extends WordSpec with Matchers with TypeCheck
DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta2.merge(delta3)), 2L, 3L)))
selector.collectPropagations() should ===(Map(nodes(0) -> expected4))
val expected5 = DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta3), 3L, 3L)))
val expected5 =
DeltaPropagation(selfUniqueAddress, false, Map("A" -> Delta(DataEnvelope(delta3), 3L, 3L)))
selector.collectPropagations() should ===(Map(nodes(1) -> expected5))
selector.collectPropagations() should ===(Map.empty[Address, DeltaPropagation])
selector.collectPropagations() should ===(Map.empty[UniqueAddress, DeltaPropagation])
}
"discard too large deltas" in {