4e8a5d333b
* some cluster logging improvements * most logger names are actually good, when using ActorLogging since config can be setup on the package (prefix) * override logSource when StageLogging is used * replace system.log with more specific logger
419 lines
15 KiB
Scala
419 lines
15 KiB
Scala
/*
|
|
* Copyright (C) 2015-2019 Lightbend Inc. <https://www.lightbend.com>
|
|
*/
|
|
|
|
package akka.stream.scaladsl
|
|
|
|
import akka.NotUsed
|
|
import akka.event.Logging
|
|
import akka.pattern.BackoffSupervisor
|
|
import akka.stream.Attributes.Attribute
|
|
import akka.stream._
|
|
import akka.stream.impl.fusing.GraphStages.SimpleLinearGraphStage
|
|
import akka.stream.scaladsl.RestartWithBackoffFlow.Delay
|
|
import akka.stream.stage._
|
|
|
|
import scala.concurrent.duration._
|
|
import akka.stream.Attributes.LogLevels
|
|
import akka.util.OptionVal
|
|
|
|
/**
|
|
* A RestartFlow wraps a [[Flow]] that gets restarted when it completes or fails.
|
|
*
|
|
* They are useful for graphs that need to run for longer than the [[Flow]] can necessarily guarantee it will, for
|
|
* example, for [[Flow]] streams that depend on a remote server that may crash or become partitioned. The
|
|
* RestartFlow ensures that the graph can continue running while the [[Flow]] restarts.
|
|
*/
|
|
object RestartFlow {
|
|
|
|
/**
|
|
* Wrap the given [[Flow]] with a [[Flow]] that will restart it when it fails or complete using an exponential
|
|
* backoff.
|
|
*
|
|
* This [[Flow]] will not cancel, complete or emit a failure, until the opposite end of it has been cancelled or
|
|
* completed. Any termination by the [[Flow]] before that time will be handled by restarting it. Any termination
|
|
* signals sent to this [[Flow]] however will terminate the wrapped [[Flow]], if it's running, and then the [[Flow]]
|
|
* will be allowed to terminate without being restarted.
|
|
*
|
|
* The restart process is inherently lossy, since there is no coordination between cancelling and the sending of
|
|
* messages. A termination signal from either end of the wrapped [[Flow]] will cause the other end to be terminated,
|
|
* and any in transit messages will be lost. During backoff, this [[Flow]] will backpressure.
|
|
*
|
|
* This uses the same exponential backoff algorithm as [[akka.pattern.Backoff]].
|
|
*
|
|
* @param minBackoff minimum (initial) duration until the child actor will
|
|
* started again, if it is terminated
|
|
* @param maxBackoff the exponential back-off is capped to this duration
|
|
* @param randomFactor after calculation of the exponential back-off an additional
|
|
* random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay.
|
|
* In order to skip this additional delay pass in `0`.
|
|
* @param flowFactory A factory for producing the [[Flow]] to wrap.
|
|
*/
|
|
def withBackoff[In, Out](minBackoff: FiniteDuration, maxBackoff: FiniteDuration, randomFactor: Double)(
|
|
flowFactory: () => Flow[In, Out, _]): Flow[In, Out, NotUsed] = {
|
|
Flow.fromGraph(
|
|
new RestartWithBackoffFlow(
|
|
flowFactory,
|
|
minBackoff,
|
|
maxBackoff,
|
|
randomFactor,
|
|
onlyOnFailures = false,
|
|
Int.MaxValue))
|
|
}
|
|
|
|
/**
|
|
* Wrap the given [[Flow]] with a [[Flow]] that will restart it when it fails or complete using an exponential
|
|
* backoff.
|
|
*
|
|
* This [[Flow]] will not cancel, complete or emit a failure, until the opposite end of it has been cancelled or
|
|
* completed. Any termination by the [[Flow]] before that time will be handled by restarting it as long as maxRestarts
|
|
* is not reached. Any termination signals sent to this [[Flow]] however will terminate the wrapped [[Flow]], if it's
|
|
* running, and then the [[Flow]] will be allowed to terminate without being restarted.
|
|
*
|
|
* The restart process is inherently lossy, since there is no coordination between cancelling and the sending of
|
|
* messages. A termination signal from either end of the wrapped [[Flow]] will cause the other end to be terminated,
|
|
* and any in transit messages will be lost. During backoff, this [[Flow]] will backpressure.
|
|
*
|
|
* This uses the same exponential backoff algorithm as [[akka.pattern.Backoff]].
|
|
*
|
|
* @param minBackoff minimum (initial) duration until the child actor will
|
|
* started again, if it is terminated
|
|
* @param maxBackoff the exponential back-off is capped to this duration
|
|
* @param randomFactor after calculation of the exponential back-off an additional
|
|
* random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay.
|
|
* In order to skip this additional delay pass in `0`.
|
|
* @param maxRestarts the amount of restarts is capped to this amount within a time frame of minBackoff.
|
|
* Passing `0` will cause no restarts and a negative number will not cap the amount of restarts.
|
|
* @param flowFactory A factory for producing the [[Flow]] to wrap.
|
|
*/
|
|
def withBackoff[In, Out](
|
|
minBackoff: FiniteDuration,
|
|
maxBackoff: FiniteDuration,
|
|
randomFactor: Double,
|
|
maxRestarts: Int)(flowFactory: () => Flow[In, Out, _]): Flow[In, Out, NotUsed] = {
|
|
Flow.fromGraph(
|
|
new RestartWithBackoffFlow(
|
|
flowFactory,
|
|
minBackoff,
|
|
maxBackoff,
|
|
randomFactor,
|
|
onlyOnFailures = false,
|
|
maxRestarts))
|
|
}
|
|
|
|
/**
|
|
* Wrap the given [[Flow]] with a [[Flow]] that will restart it when it fails using an exponential
|
|
* backoff. Notice that this [[Flow]] will not restart on completion of the wrapped flow.
|
|
*
|
|
* This [[Flow]] will not emit any failure
|
|
* The failures by the wrapped [[Flow]] will be handled by
|
|
* restarting the wrapping [[Flow]] as long as maxRestarts is not reached.
|
|
* Any termination signals sent to this [[Flow]] however will terminate the wrapped [[Flow]], if it's
|
|
* running, and then the [[Flow]] will be allowed to terminate without being restarted.
|
|
*
|
|
* The restart process is inherently lossy, since there is no coordination between cancelling and the sending of
|
|
* messages. A termination signal from either end of the wrapped [[Flow]] will cause the other end to be terminated,
|
|
* and any in transit messages will be lost. During backoff, this [[Flow]] will backpressure.
|
|
*
|
|
* This uses the same exponential backoff algorithm as [[akka.pattern.Backoff]].
|
|
*
|
|
* @param minBackoff minimum (initial) duration until the child actor will
|
|
* started again, if it is terminated
|
|
* @param maxBackoff the exponential back-off is capped to this duration
|
|
* @param randomFactor after calculation of the exponential back-off an additional
|
|
* random delay based on this factor is added, e.g. `0.2` adds up to `20%` delay.
|
|
* In order to skip this additional delay pass in `0`.
|
|
* @param maxRestarts the amount of restarts is capped to this amount within a time frame of minBackoff.
|
|
* Passing `0` will cause no restarts and a negative number will not cap the amount of restarts.
|
|
* @param flowFactory A factory for producing the [[Flow]] to wrap.
|
|
*/
|
|
def onFailuresWithBackoff[In, Out](
|
|
minBackoff: FiniteDuration,
|
|
maxBackoff: FiniteDuration,
|
|
randomFactor: Double,
|
|
maxRestarts: Int)(flowFactory: () => Flow[In, Out, _]): Flow[In, Out, NotUsed] = {
|
|
Flow.fromGraph(
|
|
new RestartWithBackoffFlow(flowFactory, minBackoff, maxBackoff, randomFactor, onlyOnFailures = true, maxRestarts))
|
|
}
|
|
|
|
}
|
|
|
|
private final class RestartWithBackoffFlow[In, Out](
|
|
flowFactory: () => Flow[In, Out, _],
|
|
minBackoff: FiniteDuration,
|
|
maxBackoff: FiniteDuration,
|
|
randomFactor: Double,
|
|
onlyOnFailures: Boolean,
|
|
maxRestarts: Int)
|
|
extends GraphStage[FlowShape[In, Out]] { self =>
|
|
|
|
val in = Inlet[In]("RestartWithBackoffFlow.in")
|
|
val out = Outlet[Out]("RestartWithBackoffFlow.out")
|
|
|
|
override def shape = FlowShape(in, out)
|
|
|
|
override def createLogic(inheritedAttributes: Attributes) =
|
|
new RestartWithBackoffLogic(
|
|
"Flow",
|
|
shape,
|
|
inheritedAttributes,
|
|
minBackoff,
|
|
maxBackoff,
|
|
randomFactor,
|
|
onlyOnFailures,
|
|
maxRestarts) {
|
|
val delay = inheritedAttributes.get[Delay](Delay(50.millis)).duration
|
|
|
|
var activeOutIn: Option[(SubSourceOutlet[In], SubSinkInlet[Out])] = None
|
|
|
|
override protected def logSource = self.getClass
|
|
|
|
override protected def startGraph() = {
|
|
val sourceOut: SubSourceOutlet[In] = createSubOutlet(in)
|
|
val sinkIn: SubSinkInlet[Out] = createSubInlet(out)
|
|
|
|
Source
|
|
.fromGraph(sourceOut.source)
|
|
// Temp fix while waiting cause of cancellation. See #23909
|
|
.via(RestartWithBackoffFlow.delayCancellation[In](delay))
|
|
.via(flowFactory())
|
|
.runWith(sinkIn.sink)(subFusingMaterializer)
|
|
|
|
if (isAvailable(out)) {
|
|
sinkIn.pull()
|
|
}
|
|
activeOutIn = Some((sourceOut, sinkIn))
|
|
}
|
|
|
|
override protected def backoff() = {
|
|
setHandler(in, new InHandler {
|
|
override def onPush() = ()
|
|
})
|
|
setHandler(out, new OutHandler {
|
|
override def onPull() = ()
|
|
})
|
|
|
|
// We need to ensure that the other end of the sub flow is also completed, so that we don't
|
|
// receive any callbacks from it.
|
|
activeOutIn.foreach {
|
|
case (sourceOut, sinkIn) =>
|
|
if (!sourceOut.isClosed) {
|
|
sourceOut.complete()
|
|
}
|
|
if (!sinkIn.isClosed) {
|
|
sinkIn.cancel()
|
|
}
|
|
activeOutIn = None
|
|
}
|
|
}
|
|
|
|
backoff()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Shared logic for all restart with backoff logics.
|
|
*/
|
|
private abstract class RestartWithBackoffLogic[S <: Shape](
|
|
name: String,
|
|
shape: S,
|
|
inheritedAttributes: Attributes,
|
|
minBackoff: FiniteDuration,
|
|
maxBackoff: FiniteDuration,
|
|
randomFactor: Double,
|
|
onlyOnFailures: Boolean,
|
|
maxRestarts: Int)
|
|
extends TimerGraphStageLogicWithLogging(shape) {
|
|
var restartCount = 0
|
|
var resetDeadline = minBackoff.fromNow
|
|
|
|
// This is effectively only used for flows, if either the main inlet or outlet of this stage finishes, then we
|
|
// don't want to restart the sub inlet when it finishes, we just finish normally.
|
|
var finishing = false
|
|
|
|
override protected def logSource: Class[_] = classOf[RestartWithBackoffLogic[_]]
|
|
|
|
protected def startGraph(): Unit
|
|
protected def backoff(): Unit
|
|
|
|
private def loggingEnabled = inheritedAttributes.get[LogLevels] match {
|
|
case Some(levels) => levels.onFailure != LogLevels.Off
|
|
case None => true
|
|
}
|
|
|
|
/**
|
|
* @param out The permanent outlet
|
|
* @return A sub sink inlet that's sink is attached to the wrapped operator
|
|
*/
|
|
protected final def createSubInlet[T](out: Outlet[T]): SubSinkInlet[T] = {
|
|
val sinkIn = new SubSinkInlet[T](s"RestartWithBackoff$name.subIn")
|
|
|
|
sinkIn.setHandler(new InHandler {
|
|
override def onPush() = push(out, sinkIn.grab())
|
|
|
|
override def onUpstreamFinish() = {
|
|
if (finishing || maxRestartsReached() || onlyOnFailures) {
|
|
complete(out)
|
|
} else {
|
|
scheduleRestartTimer()
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Upstream in this context is the wrapped stage.
|
|
*/
|
|
override def onUpstreamFailure(ex: Throwable) = {
|
|
if (finishing || maxRestartsReached()) {
|
|
fail(out, ex)
|
|
} else {
|
|
if (loggingEnabled)
|
|
log.warning("Restarting graph due to failure. stack_trace: {}", Logging.stackTraceFor(ex))
|
|
scheduleRestartTimer()
|
|
}
|
|
}
|
|
})
|
|
|
|
setHandler(out, new OutHandler {
|
|
override def onPull() = sinkIn.pull()
|
|
override def onDownstreamFinish(cause: Throwable) = {
|
|
finishing = true
|
|
sinkIn.cancel(cause)
|
|
}
|
|
})
|
|
sinkIn
|
|
}
|
|
|
|
/**
|
|
* @param in The permanent inlet for this operator
|
|
* @return Temporary SubSourceOutlet for this "restart"
|
|
*/
|
|
protected final def createSubOutlet[T](in: Inlet[T]): SubSourceOutlet[T] = {
|
|
val sourceOut = new SubSourceOutlet[T](s"RestartWithBackoff$name.subOut")
|
|
|
|
sourceOut.setHandler(new OutHandler {
|
|
override def onPull() =
|
|
if (isAvailable(in)) {
|
|
sourceOut.push(grab(in))
|
|
} else {
|
|
if (!hasBeenPulled(in)) {
|
|
pull(in)
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Downstream in this context is the wrapped stage.
|
|
*
|
|
* Can either be a failure or a cancel in the wrapped state.
|
|
* onlyOnFailures is thus racy so a delay to cancellation is added in the case of a flow.
|
|
*/
|
|
override def onDownstreamFinish(cause: Throwable) = {
|
|
if (finishing || maxRestartsReached() || onlyOnFailures) {
|
|
cancel(in, cause)
|
|
} else {
|
|
scheduleRestartTimer()
|
|
}
|
|
}
|
|
})
|
|
|
|
setHandler(
|
|
in,
|
|
new InHandler {
|
|
override def onPush() = if (sourceOut.isAvailable) {
|
|
sourceOut.push(grab(in))
|
|
}
|
|
override def onUpstreamFinish() = {
|
|
finishing = true
|
|
sourceOut.complete()
|
|
}
|
|
override def onUpstreamFailure(ex: Throwable) = {
|
|
finishing = true
|
|
sourceOut.fail(ex)
|
|
}
|
|
})
|
|
|
|
sourceOut
|
|
}
|
|
|
|
protected final def maxRestartsReached(): Boolean = {
|
|
// Check if the last start attempt was more than the minimum backoff
|
|
if (resetDeadline.isOverdue()) {
|
|
log.debug("Last restart attempt was more than {} ago, resetting restart count", minBackoff)
|
|
restartCount = 0
|
|
}
|
|
restartCount == maxRestarts
|
|
}
|
|
|
|
// Set a timer to restart after the calculated delay
|
|
protected final def scheduleRestartTimer(): Unit = {
|
|
val restartDelay = BackoffSupervisor.calculateDelay(restartCount, minBackoff, maxBackoff, randomFactor)
|
|
log.debug("Restarting graph in {}", restartDelay)
|
|
scheduleOnce("RestartTimer", restartDelay)
|
|
restartCount += 1
|
|
// And while we wait, we go into backoff mode
|
|
backoff()
|
|
}
|
|
|
|
// Invoked when the backoff timer ticks
|
|
override protected def onTimer(timerKey: Any) = {
|
|
startGraph()
|
|
resetDeadline = minBackoff.fromNow
|
|
}
|
|
|
|
// When the stage starts, start the source
|
|
override def preStart() = startGraph()
|
|
}
|
|
|
|
object RestartWithBackoffFlow {
|
|
|
|
/**
|
|
* Temporary attribute that can override the time a [[RestartWithBackoffFlow]] waits
|
|
* for a failure before cancelling.
|
|
*
|
|
* See https://github.com/akka/akka/issues/24529
|
|
*
|
|
* Will be removed if/when cancellation can include a cause.
|
|
*/
|
|
case class Delay(duration: FiniteDuration) extends Attribute
|
|
|
|
/**
|
|
* Returns a flow that is almost identity but delays propagation of cancellation from downstream to upstream.
|
|
*
|
|
* Once the down stream is finish calls to onPush are ignored.
|
|
*/
|
|
private def delayCancellation[T](duration: FiniteDuration): Flow[T, T, NotUsed] =
|
|
Flow.fromGraph(new DelayCancellationStage(duration))
|
|
|
|
private final class DelayCancellationStage[T](delay: FiniteDuration) extends SimpleLinearGraphStage[T] {
|
|
|
|
override def createLogic(inheritedAttributes: Attributes): GraphStageLogic =
|
|
new TimerGraphStageLogic(shape) with InHandler with OutHandler with StageLogging {
|
|
override protected def logSource: Class[_] = classOf[DelayCancellationStage[_]]
|
|
|
|
private var cause: OptionVal[Throwable] = OptionVal.None
|
|
|
|
setHandlers(in, out, this)
|
|
|
|
def onPush(): Unit = push(out, grab(in))
|
|
def onPull(): Unit = pull(in)
|
|
|
|
override def onDownstreamFinish(cause: Throwable): Unit = {
|
|
this.cause = OptionVal.Some(cause)
|
|
scheduleOnce("CompleteState", delay)
|
|
setHandler(in, new InHandler {
|
|
def onPush(): Unit = {}
|
|
})
|
|
}
|
|
|
|
override protected def onTimer(timerKey: Any): Unit = {
|
|
log.debug(s"Stage was canceled after delay of $delay")
|
|
cause match {
|
|
case OptionVal.Some(ex) =>
|
|
cancelStage(ex)
|
|
case OptionVal.None =>
|
|
throw new IllegalStateException("Timer hitting without first getting a cancel cannot happen")
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
}
|