raboof/pekko
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions 11

=per #15943 Avoid initite restart loop when recovery fails

Browse source 
* also include the failing message and sequenceNr in the RecoveryFailure
  message
* remove the "putting back" the message first in the mailbox
This commit is contained in:
Patrik Nordwall 2014-12-13 15:35:12 +01:00
parent da7991a3d5
commit 72d54626f3
6 changed files with 123 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

47
akka-persistence/src/main/scala/akka/persistence/PersistentView.scala
View file

@ -8,7 +8,6 @@ import scala.concurrent.duration._
import scala.util.control.NonFatal
import akka.actor.AbstractActor
import akka.actor.Actor
import akka.actor.ActorCell
import akka.actor.ActorKilledException
import akka.actor.Cancellable
import akka.actor.Stash
@ -237,7 +236,7 @@ trait PersistentView extends Actor with Snapshotter with Stash with StashFactory
/**
* Processes a loaded snapshot, if any. A loaded snapshot is offered with a `SnapshotOffer`
* message to the actor's `receiveRecover`. Then initiates a message replay, either starting
* message to the actor's `receive`. Then initiates a message replay, either starting
* from the loaded snapshot or from scratch, and switches to `replayStarted` state.
* All incoming messages are stashed.
*
@ -264,7 +263,7 @@ trait PersistentView extends Actor with Snapshotter with Stash with StashFactory
}
/**
* Processes replayed messages, if any. The actor's `receiveRecover` is invoked with the replayed
* Processes replayed messages, if any. The actor's `receive` is invoked with the replayed
* events.
*
* If replay succeeds it switches to `initializing` state and requests the highest stored sequence
@ -272,7 +271,7 @@ trait PersistentView extends Actor with Snapshotter with Stash with StashFactory
* If replay succeeds the `onReplaySuccess` callback method is called, otherwise `onReplayFailure`.
*
* If processing of a replayed event fails, the exception is caught and
* stored for being thrown later and state is changed to `recoveryFailed`.
* stored for later `RecoveryFailure` message and state is changed to `recoveryFailed`.
*
* All incoming messages are stashed.
*/
@ -294,15 +293,14 @@ trait PersistentView extends Actor with Snapshotter with Stash with StashFactory
PersistentView.super.aroundReceive(receive, p.payload)
} catch {
case NonFatal(t) ⇒
val currentMsg = context.asInstanceOf[ActorCell].currentMessage
changeState(replayFailed(t, currentMsg)) // delay throwing exception to prepareRestart
changeState(replayFailed(t, p))
}
case ReplayMessagesSuccess ⇒
onReplayComplete(await)
case ReplayMessagesFailure(cause) ⇒
onReplayComplete(await)
// FIXME what happens if RecoveryFailure is handled, i.e. actor is not stopped?
PersistentView.super.aroundReceive(receive, RecoveryFailure(cause))
PersistentView.super.aroundReceive(receive, RecoveryFailure(cause)(None))
case other ⇒
internalStash.stash()
}
@ -312,50 +310,37 @@ trait PersistentView extends Actor with Snapshotter with Stash with StashFactory
*/
private def onReplayComplete(await: Boolean): Unit = {
changeState(idle)
if (autoUpdate) schedule = Some(context.system.scheduler.scheduleOnce(autoUpdateInterval, self, Update(await = false, autoUpdateReplayMax)))
if (autoUpdate) schedule = Some(context.system.scheduler.scheduleOnce(autoUpdateInterval, self,
Update(await = false, autoUpdateReplayMax)))
if (await) internalStash.unstashAll()
}
}
/**
* Consumes remaining replayed messages and then changes to `prepareRestart`. The
* message that caused the exception during replay, is re-added to the mailbox and
* re-received in `prepareRestart` state.
* Consumes remaining replayed messages and then emits RecoveryFailure to the
* `receive` behavior.
*/
private def replayFailed(cause: Throwable, failureMessage: Envelope) = new State {
private def replayFailed(cause: Throwable, failed: PersistentRepr) = new State {
override def toString: String = "replay failed"
override def recoveryRunning: Boolean = true
override def stateReceive(receive: Receive, message: Any) = message match {
case ReplayedMessage(p) ⇒ updateLastSequenceNr(p)
case ReplayMessagesFailure(_) ⇒
replayCompleted()
replayCompleted(receive)
// journal couldn't tell the maximum stored sequence number, hence the next
// replay must be a full replay (up to the highest stored sequence number)
// Recover(lastSequenceNr) is sent by preRestart
setLastSequenceNr(Long.MaxValue)
case ReplayMessagesSuccess ⇒ replayCompleted()
case ReplayedMessage(p) ⇒ updateLastSequenceNr(p)
case ReplayMessagesSuccess ⇒ replayCompleted(receive)
case r: Recover ⇒ // ignore
case _ ⇒ internalStash.stash()
}
def replayCompleted(): Unit = {
changeState(prepareRestart(cause))
mailbox.enqueueFirst(self, failureMessage)
}
}
/**
* Re-receives the replayed message that caused an exception and re-throws that exception.
*/
private def prepareRestart(cause: Throwable) = new State {
override def toString: String = "prepare restart"
override def recoveryRunning: Boolean = true
override def stateReceive(receive: Receive, message: Any) = message match {
case ReplayedMessage(_) ⇒ throw cause
case _ ⇒ // ignore
def replayCompleted(receive: Receive): Unit = {
// FIXME what happens if RecoveryFailure is handled, i.e. actor is not stopped?
PersistentView.super.aroundReceive(receive, RecoveryFailure(cause)(Some((failed.sequenceNr, failed.payload))))
}
}