pekko/akka-persistence/src/test/scala/akka/persistence/FailureSpec.scala

/**
 * Copyright (C) 2009-2013 Typesafe Inc. <http://www.typesafe.com>
 */

package akka.persistence

import scala.concurrent.duration._
import scala.concurrent.forkjoin.ThreadLocalRandom
import scala.language.postfixOps

import com.typesafe.config.ConfigFactory

import akka.actor._
import akka.testkit._

object FailureSpec {
  val config = ConfigFactory.parseString(
    s"""
      akka.persistence.processor.chaos.live-processing-failure-rate = 0.3
      akka.persistence.processor.chaos.replay-processing-failure-rate = 0.1
      akka.persistence.journal.plugin = "akka.persistence.journal.chaos"
      akka.persistence.journal.chaos.write-failure-rate = 0.3
      akka.persistence.journal.chaos.delete-failure-rate = 0.3
      akka.persistence.journal.chaos.replay-failure-rate = 0.3
      akka.persistence.journal.chaos.class = akka.persistence.journal.chaos.ChaosJournal
      akka.persistence.snapshot-store.local.dir = "target/snapshots-failure-spec/"
    """)

  val numMessages = 10

  case object Start
  case class Done(ints: Vector[Int])

  case class ProcessingFailure(i: Int)
  case class JournalingFailure(i: Int)

  class ChaosProcessor extends Processor with ActorLogging {
    val config = context.system.settings.config.getConfig("akka.persistence.processor.chaos")
    val liveProcessingFailureRate = config.getDouble("live-processing-failure-rate")
    val replayProcessingFailureRate = config.getDouble("replay-processing-failure-rate")

    // processor state
    var ints = Vector.empty[Int]

    override def processorId = "chaos"

    def random = ThreadLocalRandom.current

    def receive = {
      case Persistent(i: Int, _) ⇒
        val failureRate = if (recoveryRunning) replayProcessingFailureRate else liveProcessingFailureRate
        if (ints.contains(i)) {
          log.debug(debugMessage(s"ignored duplicate ${i}"))
        } else if (shouldFail(failureRate)) {
          throw new TestException(debugMessage(s"rejected payload ${i}"))
        } else {
          ints :+= i
          if (ints.length == numMessages) sender ! Done(ints)
          log.debug(debugMessage(s"processed payload ${i}"))
        }
      case PersistenceFailure(i: Int, _, _) ⇒
        // inform sender about journaling failure so that it can resend
        sender ! JournalingFailure(i)
      case RecoveryFailure(_) ⇒
        // journal failed during recovery, throw exception to re-recover processor
        throw new TestException(debugMessage("recovery failed"))
    }

    override def preRestart(reason: Throwable, message: Option[Any]): Unit = {
      message match {
        case Some(p @ Persistent(i: Int, _)) if !recoveryRunning ⇒
          deleteMessage(p.sequenceNr)
          log.debug(debugMessage(s"requested deletion of payload ${i}"))
          // inform sender about processing failure so that it can resend
          sender ! ProcessingFailure(i)
        case _ ⇒
      }
      super.preRestart(reason, message)
    }

    private def shouldFail(rate: Double) =
      random.nextDouble() < rate

    private def debugMessage(msg: String): String =
      s"${msg} (mode = ${if (recoveryRunning) "replay" else "live"} snr = ${lastSequenceNr} state = ${ints.sorted})"
  }

  class ChaosProcessorApp(probe: ActorRef) extends Actor with ActorLogging {
    val processor = context.actorOf(Props[ChaosProcessor])

    def receive = {
      case Start      ⇒ 1 to numMessages foreach (processor ! Persistent(_))
      case Done(ints) ⇒ probe ! Done(ints)
      case ProcessingFailure(i) ⇒
        processor ! Persistent(i)
        log.debug(s"resent ${i} after processing failure")
      case JournalingFailure(i) ⇒
        processor ! Persistent(i)
        log.debug(s"resent ${i} after journaling failure")
    }
  }
}

class FailureSpec extends AkkaSpec(FailureSpec.config) with Cleanup with ImplicitSender {
  import FailureSpec._

  "The journaling protocol (= conversation between a processor and a journal)" must {
    "tolerate and recover from random failures" in {
      system.actorOf(Props(classOf[ChaosProcessorApp], testActor)) ! Start
      expectMsgPF(numMessages seconds) { case Done(ints) ⇒ ints.sorted must be(1 to numMessages toVector) }

      system.actorOf(Props(classOf[ChaosProcessorApp], testActor)) // recovery of new instance must have same outcome
      expectMsgPF(numMessages seconds) { case Done(ints) ⇒ ints.sorted must be(1 to numMessages toVector) }
    }
  }
}