Stop singleton and shards when self MemberDowned, #26336 (#26339)

* Stop singleton when self MemberDowned, #26336
  * It's safer to stop singleton instance early in case of downing.
  * Instead of waiting for MemberRemoved and trying to hand over.
* Stop ShardRegion when self MemberDowned, #26336
* Upper bound when waiting for seen in shutdownSelfWhenDown, #26336
This commit is contained in:
Patrik Nordwall 2019-02-12 15:05:33 +01:00 committed by GitHub
parent 8e2d378228
commit ddada9a8e1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 165 additions and 2 deletions

View file

@ -511,7 +511,7 @@ class ClusterSingletonManager(
require(!cluster.isTerminated, "Cluster node must not be terminated")
// subscribe to cluster changes, re-subscribe when restart
cluster.subscribe(self, ClusterEvent.InitialStateAsEvents, classOf[MemberRemoved])
cluster.subscribe(self, ClusterEvent.InitialStateAsEvents, classOf[MemberRemoved], classOf[MemberDowned])
setTimer(CleanupTimer, Cleanup, 1.minute, repeat = true)
@ -573,6 +573,10 @@ class ClusterSingletonManager(
stay using YoungerData(oldestOption)
}
case Event(MemberDowned(m), _) if m.uniqueAddress == cluster.selfUniqueAddress
logInfo("Self downed, stopping ClusterSingletonManager")
stop()
case Event(MemberRemoved(m, _), _) if m.uniqueAddress == cluster.selfUniqueAddress
logInfo("Self removed, stopping ClusterSingletonManager")
stop()
@ -612,6 +616,10 @@ class ClusterSingletonManager(
stay
}
case Event(MemberDowned(m), _) if m.uniqueAddress == cluster.selfUniqueAddress
logInfo("Self downed, stopping ClusterSingletonManager")
stop()
case Event(MemberRemoved(m, _), _) if m.uniqueAddress == cluster.selfUniqueAddress
logInfo("Self removed, stopping ClusterSingletonManager")
stop()
@ -722,6 +730,15 @@ class ClusterSingletonManager(
// complete memberExitingProgress when handOverDone
sender() ! Done // reply to ask
stay
case Event(MemberDowned(m), OldestData(singleton, singletonTerminated)) if m.uniqueAddress == cluster.selfUniqueAddress
if (singletonTerminated) {
logInfo("Self downed, stopping ClusterSingletonManager")
stop()
} else {
logInfo("Self downed, stopping")
gotoStopping(singleton)
}
}
when(WasOldest) {
@ -761,6 +778,15 @@ class ClusterSingletonManager(
sender() ! Done // reply to ask
stay
case Event(MemberDowned(m), OldestData(singleton, singletonTerminated)) if m.uniqueAddress == cluster.selfUniqueAddress
if (singletonTerminated) {
logInfo("Self downed, stopping ClusterSingletonManager")
stop()
} else {
logInfo("Self downed, stopping")
gotoStopping(singleton)
}
}
def gotoHandingOver(singleton: ActorRef, singletonTerminated: Boolean, handOverTo: Option[ActorRef]): State = {
@ -853,6 +879,10 @@ class ClusterSingletonManager(
case Event(Cleanup, _)
cleanupOverdueNotMemberAnyMore()
stay
case Event(MemberDowned(m), _)
if (m.uniqueAddress == cluster.selfUniqueAddress)
logInfo("Self downed, waiting for removal")
stay
}
onTransition {