From 8c4cc9a93fe4d354445325fd96a7f69711e58e8a Mon Sep 17 00:00:00 2001 From: Konrad Malawski Date: Wed, 19 Aug 2015 18:20:13 +0200 Subject: [PATCH] =doc #18128 explain PoisonPill interaction with PA --- .../general/message-delivery-reliability.rst | 8 +- .../persistence/LambdaPersistenceDocTest.java | 70 ++++++++++++ .../docs/persistence/PersistenceDocTest.java | 73 +++++++++++- akka-docs/rst/java/lambda-actors.rst | 2 + akka-docs/rst/java/lambda-persistence.rst | 30 ++++- akka-docs/rst/java/persistence-query.rst | 4 +- akka-docs/rst/java/persistence.rst | 26 +++++ .../migration-guide-eventsourced-2.3.x.rst | 2 +- ...e-persistence-experimental-2.3.x-2.4.x.rst | 4 +- akka-docs/rst/scala/actors.rst | 2 +- .../docs/persistence/PersistenceDocSpec.scala | 105 +++++++++++++++++- akka-docs/rst/scala/persistence-query.rst | 4 +- akka-docs/rst/scala/persistence.rst | 29 ++++- akka-docs/rst/scala/testing.rst | 4 +- 14 files changed, 341 insertions(+), 22 deletions(-) diff --git a/akka-docs/rst/general/message-delivery-reliability.rst b/akka-docs/rst/general/message-delivery-reliability.rst index 0a3dadd13a..6f48c900a0 100644 --- a/akka-docs/rst/general/message-delivery-reliability.rst +++ b/akka-docs/rst/general/message-delivery-reliability.rst @@ -291,13 +291,13 @@ delivery is an explicit ACK–RETRY protocol. In its simplest form this requires The third becomes necessary by virtue of the acknowledgements not being guaranteed to arrive either. An ACK-RETRY protocol with business-level acknowledgements is -supported by :ref:`at-least-once-delivery` of the Akka Persistence module. Duplicates can be -detected by tracking the identifiers of messages sent via :ref:`at-least-once-delivery`. +supported by :ref:`at-least-once-delivery-scala` of the Akka Persistence module. Duplicates can be +detected by tracking the identifiers of messages sent via :ref:`at-least-once-delivery-scala`. Another way of implementing the third part would be to make processing the messages idempotent on the level of the business logic. Another example of implementing all three requirements is shown at -:ref:`reliable-proxy` (which is now superseded by :ref:`at-least-once-delivery`). +:ref:`reliable-proxy` (which is now superseded by :ref:`at-least-once-delivery-scala`). Event Sourcing -------------- @@ -313,7 +313,7 @@ components may consume the event stream as a means to replicate the component’ state on a different continent or to react to changes). If the component’s state is lost—due to a machine failure or by being pushed out of a cache—it can easily be reconstructed by replaying the event stream (usually employing -snapshots to speed up the process). :ref:`event-sourcing` is supported by +snapshots to speed up the process). :ref:`event-sourcing-scala` is supported by Akka Persistence. Mailbox with Explicit Acknowledgement diff --git a/akka-docs/rst/java/code/docs/persistence/LambdaPersistenceDocTest.java b/akka-docs/rst/java/code/docs/persistence/LambdaPersistenceDocTest.java index 48a0629c42..c130db275c 100644 --- a/akka-docs/rst/java/code/docs/persistence/LambdaPersistenceDocTest.java +++ b/akka-docs/rst/java/code/docs/persistence/LambdaPersistenceDocTest.java @@ -542,4 +542,74 @@ public class LambdaPersistenceDocTest { //#view-update } }; + + static Object o14 = new Object() { + //#safe-shutdown + final class Shutdown { + } + + class MyPersistentActor extends AbstractPersistentActor { + @Override + public String persistenceId() { + return "some-persistence-id"; + } + + @Override + public PartialFunction receiveCommand() { + return ReceiveBuilder + .match(Shutdown.class, shutdown -> { + context().stop(self()); + }) + .match(String.class, msg -> { + System.out.println(msg); + persist("handle-" + msg, e -> System.out.println(e)); + }) + .build(); + } + + @Override + public PartialFunction receiveRecover() { + return ReceiveBuilder.matchAny(any -> {}).build(); + } + + } + //#safe-shutdown + + + public void usage() { + final ActorSystem system = ActorSystem.create("example"); + final ActorRef persistentActor = system.actorOf(Props.create(MyPersistentActor.class)); + //#safe-shutdown-example-bad + // UN-SAFE, due to PersistentActor's command stashing: + persistentActor.tell("a", ActorRef.noSender()); + persistentActor.tell("b", ActorRef.noSender()); + persistentActor.tell(PoisonPill.getInstance(), ActorRef.noSender()); + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // PoisonPill is an AutoReceivedMessage, is handled automatically + // !! stop !! + // Actor is stopped without handling `b` nor the `a` handler! + //#safe-shutdown-example-bad + + //#safe-shutdown-example-good + // SAFE: + persistentActor.tell("a", ActorRef.noSender()); + persistentActor.tell("b", ActorRef.noSender()); + persistentActor.tell(new Shutdown(), ActorRef.noSender()); + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // handle-a + // # unstashing; internal-stash = [Shutdown] + // b + // handle-b + // # unstashing; internal-stash = [] + // Shutdown + // -- stop -- + //#safe-shutdown-example-good + } + }; } diff --git a/akka-docs/rst/java/code/docs/persistence/PersistenceDocTest.java b/akka-docs/rst/java/code/docs/persistence/PersistenceDocTest.java index eedf1d346b..506b31ec9c 100644 --- a/akka-docs/rst/java/code/docs/persistence/PersistenceDocTest.java +++ b/akka-docs/rst/java/code/docs/persistence/PersistenceDocTest.java @@ -13,7 +13,6 @@ import akka.japi.Function; import akka.japi.Procedure; import akka.persistence.*; -import scala.Option; import java.io.Serializable; public class PersistenceDocTest { @@ -504,7 +503,7 @@ public class PersistenceDocTest { } }; - static Object o13 = new Object() { + static Object o14 = new Object() { //#view class MyView extends UntypedPersistentView { @Override @@ -534,4 +533,74 @@ public class PersistenceDocTest { //#view-update } }; + + static Object o13 = new Object() { + //#safe-shutdown + final class Shutdown {} + + class MyPersistentActor extends UntypedPersistentActor { + @Override + public String persistenceId() { + return "some-persistence-id"; + } + + @Override + public void onReceiveCommand(Object msg) throws Exception { + if (msg instanceof Shutdown) { + context().stop(self()); + } else if (msg instanceof String) { + System.out.println(msg); + persist("handle-" + msg, new Procedure() { + @Override + public void apply(String param) throws Exception { + System.out.println(param); + } + }); + } else unhandled(msg); + } + + @Override + public void onReceiveRecover(Object msg) throws Exception { + // handle recovery... + } + } + //#safe-shutdown + + + public void usage() { + final ActorSystem system = ActorSystem.create("example"); + final ActorRef persistentActor = system.actorOf(Props.create(MyPersistentActor.class)); + //#safe-shutdown-example-bad + // UN-SAFE, due to PersistentActor's command stashing: + persistentActor.tell("a", ActorRef.noSender()); + persistentActor.tell("b", ActorRef.noSender()); + persistentActor.tell(PoisonPill.getInstance(), ActorRef.noSender()); + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // PoisonPill is an AutoReceivedMessage, is handled automatically + // !! stop !! + // Actor is stopped without handling `b` nor the `a` handler! + //#safe-shutdown-example-bad + + //#safe-shutdown-example-good + // SAFE: + persistentActor.tell("a", ActorRef.noSender()); + persistentActor.tell("b", ActorRef.noSender()); + persistentActor.tell(new Shutdown(), ActorRef.noSender()); + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // handle-a + // # unstashing; internal-stash = [Shutdown] + // b + // handle-b + // # unstashing; internal-stash = [] + // Shutdown + // -- stop -- + //#safe-shutdown-example-good + } + }; } diff --git a/akka-docs/rst/java/lambda-actors.rst b/akka-docs/rst/java/lambda-actors.rst index cd43b3f8e1..d15c731a98 100644 --- a/akka-docs/rst/java/lambda-actors.rst +++ b/akka-docs/rst/java/lambda-actors.rst @@ -537,6 +537,8 @@ different one. Outside of an actor and if no reply is needed the second argument can be ``null``; if a reply is needed outside of an actor you can use the ask-pattern described next.. +.. _actors-ask-lambda: + Ask: Send-And-Receive-Future ---------------------------- diff --git a/akka-docs/rst/java/lambda-persistence.rst b/akka-docs/rst/java/lambda-persistence.rst index 15b10b11d2..34e7c48dfe 100644 --- a/akka-docs/rst/java/lambda-persistence.rst +++ b/akka-docs/rst/java/lambda-persistence.rst @@ -376,10 +376,36 @@ restarts of the persistent actor. .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem +.. _safe-shutdown-lambda: + +Safely shutting down persistent actors +-------------------------------------- + +Special care should be given when when shutting down persistent actors from the outside. +With normal Actors it is often acceptable to use the special :ref:`PoisonPill ` message +to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled +automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill. + +This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while +the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used. +Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the +confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill +before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor. + +.. warning:: + Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors. + +The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing +mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used: + +.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown +.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown-example-bad +.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown-example-good + .. _persistent-views-java-lambda: -Views -===== +Persistent Views +================ .. warning:: diff --git a/akka-docs/rst/java/persistence-query.rst b/akka-docs/rst/java/persistence-query.rst index b35f934325..f373b99938 100644 --- a/akka-docs/rst/java/persistence-query.rst +++ b/akka-docs/rst/java/persistence-query.rst @@ -82,7 +82,7 @@ If your usage does not require a live stream, you can disable refreshing by usin .. includecode:: code/docs/persistence/PersistenceQueryDocTest.java#all-persistence-ids-snap -``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor `, +``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor `, however, since it is a stream it is possible to keep it alive and watch for additional incoming events persisted by the persistent actor identified by the given ``persistenceId``. Most journals will have to revert to polling in order to achieve this, which can be configured using the ``RefreshInterval`` query hint: @@ -135,7 +135,7 @@ specialised query object, as demonstrated in the sample below: Performance and denormalization =============================== -When building systems using :ref:`event-sourcing` and CQRS (`Command & Query Responsibility Segragation`_) techniques +When building systems using :ref:`event-sourcing-scala` and CQRS (`Command & Query Responsibility Segragation`_) techniques it is tremendously important to realise that the write-side has completely different needs from the read-side, and separating those concerns into datastores that are optimised for either side makes it possible to offer the best expirience for the write and read sides independently. diff --git a/akka-docs/rst/java/persistence.rst b/akka-docs/rst/java/persistence.rst index 8bb7029c53..80cadedf29 100644 --- a/akka-docs/rst/java/persistence.rst +++ b/akka-docs/rst/java/persistence.rst @@ -379,6 +379,32 @@ restarts of the persistent actor. .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem +.. _safe-shutdown-java: + +Safely shutting down persistent actors +-------------------------------------- + +Special care should be given when when shutting down persistent actors from the outside. +With normal Actors it is often acceptable to use the special :ref:`PoisonPill ` message +to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled +automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill. + +This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while +the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used. +Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the +confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill +before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor. + +.. warning:: + Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors. + +The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing +mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used: + +.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown +.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown-example-bad +.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown-example-good + .. _persistent-views-java: Persistent Views diff --git a/akka-docs/rst/project/migration-guide-eventsourced-2.3.x.rst b/akka-docs/rst/project/migration-guide-eventsourced-2.3.x.rst index 850dfa5cc7..f2968d5f07 100644 --- a/akka-docs/rst/project/migration-guide-eventsourced-2.3.x.rst +++ b/akka-docs/rst/project/migration-guide-eventsourced-2.3.x.rst @@ -72,7 +72,7 @@ Processors / PersistentActor **Akka Persistence:** ``PersistentActor`` -- Trait that adds journaling to actors (see :ref:`event-sourcing`) and used by applications for +- Trait that adds journaling to actors (see :ref:`event-sourcing-scala`) and used by applications for *event sourcing* or *command sourcing*. Corresponds to ``Eventsourced`` processors in Eventsourced but is not a stackable trait. - Automatically recovers on start and re-start, by default. :ref:`recovery` can be customized or turned off by overriding actor life cycle hooks ``preStart`` and ``preRestart``. ``Processor`` takes care that new messages diff --git a/akka-docs/rst/project/migration-guide-persistence-experimental-2.3.x-2.4.x.rst b/akka-docs/rst/project/migration-guide-persistence-experimental-2.3.x-2.4.x.rst index b7435d47dd..f8060314a1 100644 --- a/akka-docs/rst/project/migration-guide-persistence-experimental-2.3.x-2.4.x.rst +++ b/akka-docs/rst/project/migration-guide-persistence-experimental-2.3.x-2.4.x.rst @@ -34,7 +34,7 @@ To extend ``PersistentActor``:: /*...*/ } -Read more about the persistent actor in the :ref:`documentation for Scala ` and +Read more about the persistent actor in the :ref:`documentation for Scala ` and :ref:`documentation for Java `. Changed processorId to (abstract) persistenceId @@ -183,7 +183,7 @@ acknowledgement from the channel is needed to guarantee safe hand-off. Therefore delivery is provided in a new ``AtLeastOnceDelivery`` trait that is mixed-in to the persistent actor on the sending side. -Read more about at-least-once delivery in the :ref:`documentation for Scala ` and +Read more about at-least-once delivery in the :ref:`documentation for Scala ` and :ref:`documentation for Java `. Default persistence plugins diff --git a/akka-docs/rst/scala/actors.rst b/akka-docs/rst/scala/actors.rst index 8f7e56c9b0..a895cc9bf7 100644 --- a/akka-docs/rst/scala/actors.rst +++ b/akka-docs/rst/scala/actors.rst @@ -426,7 +426,7 @@ result: It is always preferable to communicate with other Actors using their ActorRef instead of relying upon ActorSelection. Exceptions are - * sending messages using the :ref:`at-least-once-delivery` facility + * sending messages using the :ref:`at-least-once-delivery-scala` facility * initiating first contact with a remote system In all other cases ActorRefs can be provided during Actor creation or diff --git a/akka-docs/rst/scala/code/docs/persistence/PersistenceDocSpec.scala b/akka-docs/rst/scala/code/docs/persistence/PersistenceDocSpec.scala index 423cccbbc2..dddf53e0ae 100644 --- a/akka-docs/rst/scala/code/docs/persistence/PersistenceDocSpec.scala +++ b/akka-docs/rst/scala/code/docs/persistence/PersistenceDocSpec.scala @@ -4,7 +4,7 @@ package docs.persistence -import akka.actor.{ Actor, ActorRef, ActorSystem, Props } +import akka.actor._ import akka.pattern.BackoffSupervisor import akka.persistence._ @@ -102,7 +102,7 @@ object PersistenceDocSpec { object AtLeastOnce { //#at-least-once-example - import akka.actor.{ Actor, ActorPath, ActorSelection } + import akka.actor.{ Actor, ActorSelection } import akka.persistence.AtLeastOnceDelivery case class Msg(deliveryId: Long, s: String) @@ -358,6 +358,107 @@ object PersistenceDocSpec { //#nested-persistAsync-persistAsync-caller } + object AvoidPoisonPill { + + //#safe-shutdown + /** Explicit shutdown message */ + case object Shutdown + + class SafePersistentActor extends PersistentActor { + override def persistenceId = "safe-actor" + + override def receiveCommand: Receive = { + case c: String => + println(c) + persist(s"handle-$c") { println(_) } + case Shutdown => + context.stop(self) + } + + override def receiveRecover: Receive = { + case _ => // handle recovery here + } + } + //#safe-shutdown + + //#safe-shutdown-example-bad + // UN-SAFE, due to PersistentActor's command stashing: + persistentActor ! "a" + persistentActor ! "b" + persistentActor ! PoisonPill + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // PoisonPill is an AutoReceivedMessage, is handled automatically + // !! stop !! + // Actor is stopped without handling `b` nor the `a` handler! + //#safe-shutdown-example-bad + + //#safe-shutdown-example-good + // SAFE: + persistentActor ! "a" + persistentActor ! "b" + persistentActor ! Shutdown + // order of received messages: + // a + // # b arrives at mailbox, stashing; internal-stash = [b] + // # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown] + // handle-a + // # unstashing; internal-stash = [Shutdown] + // b + // handle-b + // # unstashing; internal-stash = [] + // Shutdown + // -- stop -- + //#safe-shutdown-example-good + + class MyPersistAsyncActor extends PersistentActor { + override def persistenceId = "my-stable-persistence-id" + + override def receiveRecover: Receive = { + case _ => // handle recovery here + } + + //#nested-persistAsync-persistAsync + override def receiveCommand: Receive = { + case c: String => + sender() ! c + persistAsync(c + "-outer-1") { outer ⇒ + sender() ! outer + persistAsync(c + "-inner-1") { inner ⇒ sender() ! inner } + } + persistAsync(c + "-outer-2") { outer ⇒ + sender() ! outer + persistAsync(c + "-inner-2") { inner ⇒ sender() ! inner } + } + } + //#nested-persistAsync-persistAsync + } + + //#nested-persistAsync-persistAsync-caller + persistentActor ! "a" + persistentActor ! "b" + + // order of received messages: + // a + // b + // a-outer-1 + // a-outer-2 + // b-outer-1 + // b-outer-2 + // a-inner-1 + // a-inner-2 + // b-inner-1 + // b-inner-2 + + // which can be seen as the following causal relationship: + // a -> a-outer-1 -> a-outer-2 -> a-inner-1 -> a-inner-2 + // b -> b-outer-1 -> b-outer-2 -> b-inner-1 -> b-inner-2 + + //#nested-persistAsync-persistAsync-caller + } + object View { import akka.actor.Props diff --git a/akka-docs/rst/scala/persistence-query.rst b/akka-docs/rst/scala/persistence-query.rst index b5e9023f18..2dce052344 100644 --- a/akka-docs/rst/scala/persistence-query.rst +++ b/akka-docs/rst/scala/persistence-query.rst @@ -82,7 +82,7 @@ If your usage does not require a live stream, you can disable refreshing by usin .. includecode:: code/docs/persistence/query/PersistenceQueryDocSpec.scala#all-persistence-ids-snap -``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor `, +``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor `, however, since it is a stream it is possible to keep it alive and watch for additional incoming events persisted by the persistent actor identified by the given ``persistenceId``. Most journals will have to revert to polling in order to achieve this, which can be configured using the ``RefreshInterval`` query hint: @@ -134,7 +134,7 @@ specialised query object, as demonstrated in the sample below: Performance and denormalization =============================== -When building systems using :ref:`event-sourcing` and CQRS (`Command & Query Responsibility Segragation`_) techniques +When building systems using :ref:`event-sourcing-scala` and CQRS (`Command & Query Responsibility Segragation`_) techniques it is tremendously important to realise that the write-side has completely different needs from the read-side, and separating those concerns into datastores that are optimised for either side makes it possible to offer the best expirience for the write and read sides independently. diff --git a/akka-docs/rst/scala/persistence.rst b/akka-docs/rst/scala/persistence.rst index 39e561118d..eeb10e3e4d 100644 --- a/akka-docs/rst/scala/persistence.rst +++ b/akka-docs/rst/scala/persistence.rst @@ -61,7 +61,7 @@ Architecture .. _Community plugins: http://akka.io/community/ -.. _event-sourcing: +.. _event-sourcing-scala: Event sourcing ============== @@ -366,6 +366,31 @@ restarts of the persistent actor. .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem +.. _safe-shutdown-scala: + +Safely shutting down persistent actors +-------------------------------------- + +Special care should be given when when shutting down persistent actors from the outside. +With normal Actors it is often acceptable to use the special :ref:`PoisonPill ` message +to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled +automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill. + +This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while +the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used. +Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the +confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill +before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor. + +.. warning:: + Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors. + +The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing +mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used: + +.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown +.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown-example-bad +.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown-example-good .. _persistent-views: @@ -519,7 +544,7 @@ If failure messages are left unhandled by the actor, a default warning log messa No default action is performed on the success messages, however you're free to handle them e.g. in order to delete an in memory representation of the snapshot, or in the case of failure to attempt save the snapshot again. -.. _at-least-once-delivery: +.. _at-least-once-delivery-scala: At-Least-Once Delivery ====================== diff --git a/akka-docs/rst/scala/testing.rst b/akka-docs/rst/scala/testing.rst index 9e23992a6d..090fe0e6c7 100644 --- a/akka-docs/rst/scala/testing.rst +++ b/akka-docs/rst/scala/testing.rst @@ -62,8 +62,8 @@ section below. .. warning:: Due to the synchronous nature of ``TestActorRef`` it will **not** work with some support traits that Akka provides as they require asynchronous behaviours to function properly. - Examples of traits that do not mix well with test actor refs are :ref:`PersistentActor ` - and :ref:`AtLeastOnceDelivery ` provided by :ref:`Akka Persistence `. + Examples of traits that do not mix well with test actor refs are :ref:`PersistentActor ` + and :ref:`AtLeastOnceDelivery ` provided by :ref:`Akka Persistence `. Obtaining a Reference to an :class:`Actor` ------------------------------------------