=doc #18128 explain PoisonPill interaction with PA

akka-docs/rst/general/message-delivery-reliability.rst

@@ -291,13 +291,13 @@ delivery is an explicit ACK–RETRY protocol. In its simplest form this requires
 
 The third becomes necessary by virtue of the acknowledgements not being guaranteed
 to arrive either. An ACK-RETRY protocol with business-level acknowledgements is
-supported by :ref:`at-least-once-delivery` of the Akka Persistence module. Duplicates can be
-detected by tracking the identifiers of messages sent via :ref:`at-least-once-delivery`.
+supported by :ref:`at-least-once-delivery-scala` of the Akka Persistence module. Duplicates can be
+detected by tracking the identifiers of messages sent via :ref:`at-least-once-delivery-scala`.
 Another way of implementing the third part would be to make processing the messages
 idempotent on the level of the business logic.
 
 Another example of implementing all three requirements is shown at
-:ref:`reliable-proxy` (which is now superseded by :ref:`at-least-once-delivery`).
+:ref:`reliable-proxy` (which is now superseded by :ref:`at-least-once-delivery-scala`).
 
 Event Sourcing
 --------------
@@ -313,7 +313,7 @@ components may consume the event stream as a means to replicate the component’
 state on a different continent or to react to changes). If the component’s
 state is lost—due to a machine failure or by being pushed out of a cache—it can
 easily be reconstructed by replaying the event stream (usually employing
-snapshots to speed up the process). :ref:`event-sourcing` is supported by
+snapshots to speed up the process). :ref:`event-sourcing-scala` is supported by
 Akka Persistence.
 
 Mailbox with Explicit Acknowledgement

akka-docs/rst/java/code/docs/persistence/LambdaPersistenceDocTest.java

@@ -542,4 +542,74 @@ public class LambdaPersistenceDocTest {
       //#view-update
     }
   };
+
+  static Object o14 = new Object() {
+    //#safe-shutdown
+    final class Shutdown {
+    }
+
+    class MyPersistentActor extends AbstractPersistentActor {
+      @Override
+      public String persistenceId() {
+        return "some-persistence-id";
+      }
+
+      @Override
+      public PartialFunction<Object, BoxedUnit> receiveCommand() {
+        return ReceiveBuilder
+          .match(Shutdown.class, shutdown -> {
+            context().stop(self());
+          })
+          .match(String.class, msg -> {
+            System.out.println(msg);
+            persist("handle-" + msg, e -> System.out.println(e));
+          })
+          .build();
+      }
+
+      @Override
+      public PartialFunction<Object, BoxedUnit> receiveRecover() {
+        return ReceiveBuilder.matchAny(any -> {}).build();
+      }
+
+    }
+    //#safe-shutdown
+
+
+    public void usage() {
+      final ActorSystem system = ActorSystem.create("example");
+      final ActorRef persistentActor = system.actorOf(Props.create(MyPersistentActor.class));
+      //#safe-shutdown-example-bad
+      // UN-SAFE, due to PersistentActor's command stashing:
+      persistentActor.tell("a", ActorRef.noSender());
+      persistentActor.tell("b", ActorRef.noSender());
+      persistentActor.tell(PoisonPill.getInstance(), ActorRef.noSender());
+      // order of received messages:
+      // a
+      //   # b arrives at mailbox, stashing;        internal-stash = [b]
+      //   # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+      // PoisonPill is an AutoReceivedMessage, is handled automatically
+      // !! stop !!
+      // Actor is stopped without handling `b` nor the `a` handler!
+      //#safe-shutdown-example-bad
+
+      //#safe-shutdown-example-good
+      // SAFE:
+      persistentActor.tell("a", ActorRef.noSender());
+      persistentActor.tell("b", ActorRef.noSender());
+      persistentActor.tell(new Shutdown(), ActorRef.noSender());
+      // order of received messages:
+      // a
+      //   # b arrives at mailbox, stashing;        internal-stash = [b]
+      //   # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+      // handle-a
+      //   # unstashing;                            internal-stash = [Shutdown]
+      // b
+      // handle-b
+      //   # unstashing;                            internal-stash = []
+      // Shutdown
+      // -- stop --
+      //#safe-shutdown-example-good
+    }
+  };
 }

akka-docs/rst/java/code/docs/persistence/PersistenceDocTest.java

@@ -13,7 +13,6 @@ import akka.japi.Function;
 import akka.japi.Procedure;
 import akka.persistence.*;
 
-import scala.Option;
 import java.io.Serializable;
 
 public class PersistenceDocTest {
@@ -504,7 +503,7 @@ public class PersistenceDocTest {
         }
     };
 
-    static Object o13 = new Object() {
+    static Object o14 = new Object() {
         //#view
         class MyView extends UntypedPersistentView {
             @Override
@@ -534,4 +533,74 @@ public class PersistenceDocTest {
             //#view-update
         }
     };
+
+    static Object o13 = new Object() {
+        //#safe-shutdown
+        final class Shutdown {}
+
+        class MyPersistentActor extends UntypedPersistentActor {
+            @Override
+            public String persistenceId() {
+                return "some-persistence-id";
+            }
+
+            @Override
+            public void onReceiveCommand(Object msg) throws Exception {
+                if (msg instanceof Shutdown) {
+                    context().stop(self());
+                } else if (msg instanceof String) {
+                    System.out.println(msg);
+                    persist("handle-" + msg, new Procedure<String>() {
+                        @Override
+                        public void apply(String param) throws Exception {
+                            System.out.println(param);
+                        }
+                    });
+                } else unhandled(msg);
+            }
+
+            @Override
+            public void onReceiveRecover(Object msg) throws Exception {
+                // handle recovery...
+            }
+        }
+        //#safe-shutdown
+
+
+        public void usage() {
+            final ActorSystem system = ActorSystem.create("example");
+            final ActorRef persistentActor = system.actorOf(Props.create(MyPersistentActor.class));
+            //#safe-shutdown-example-bad
+            // UN-SAFE, due to PersistentActor's command stashing:
+            persistentActor.tell("a", ActorRef.noSender());
+            persistentActor.tell("b", ActorRef.noSender());
+            persistentActor.tell(PoisonPill.getInstance(), ActorRef.noSender());
+            // order of received messages:
+            // a
+            //   # b arrives at mailbox, stashing;        internal-stash = [b]
+            //   # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+            // PoisonPill is an AutoReceivedMessage, is handled automatically
+            // !! stop !!
+            // Actor is stopped without handling `b` nor the `a` handler!
+            //#safe-shutdown-example-bad
+
+            //#safe-shutdown-example-good
+            // SAFE:
+            persistentActor.tell("a", ActorRef.noSender());
+            persistentActor.tell("b", ActorRef.noSender());
+            persistentActor.tell(new Shutdown(), ActorRef.noSender());
+            // order of received messages:
+            // a
+            //   # b arrives at mailbox, stashing;        internal-stash = [b]
+            //   # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+            // handle-a
+            //   # unstashing;                            internal-stash = [Shutdown]
+            // b
+            // handle-b
+            //   # unstashing;                            internal-stash = []
+            // Shutdown
+            // -- stop --
+            //#safe-shutdown-example-good
+        }
+    };
 }

akka-docs/rst/java/lambda-actors.rst

@@ -537,6 +537,8 @@ different one. Outside of an actor and if no reply is needed the second
 argument can be ``null``; if a reply is needed outside of an actor you can use
 the ask-pattern described next..
 
+.. _actors-ask-lambda:
+
 Ask: Send-And-Receive-Future
 ----------------------------
 

akka-docs/rst/java/lambda-persistence.rst

@@ -376,10 +376,36 @@ restarts of the persistent actor.
 
 .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem
 
+.. _safe-shutdown-lambda:
+
+Safely shutting down persistent actors
+--------------------------------------
+
+Special care should be given when when shutting down persistent actors from the outside.
+With normal Actors it is often acceptable to use the special :ref:`PoisonPill <poison-pill-java>` message
+to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled
+automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill.
+
+This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while
+the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used.
+Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the
+confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill
+before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor.
+
+.. warning::
+  Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors.
+
+The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing
+mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used:
+
+.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown
+.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown-example-bad
+.. includecode:: code/docs/persistence/LambdaPersistenceDocTest.java#safe-shutdown-example-good
+
 .. _persistent-views-java-lambda:
 
-Views
-=====
+Persistent Views
+================
 
 .. warning::
 

akka-docs/rst/java/persistence-query.rst

@@ -82,7 +82,7 @@ If your usage does not require a live stream, you can disable refreshing by usin
 
 .. includecode:: code/docs/persistence/PersistenceQueryDocTest.java#all-persistence-ids-snap
 
-``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor <event-sourcing>`,
+``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor <event-sourcing-scala>`,
 however, since it is a stream it is possible to keep it alive and watch for additional incoming events persisted by the
 persistent actor identified by the given ``persistenceId``. Most journals will have to revert to polling in order to achieve
 this, which can be configured using the ``RefreshInterval`` query hint:
@@ -135,7 +135,7 @@ specialised query object, as demonstrated in the sample below:
 
 Performance and denormalization
 ===============================
-When building systems using :ref:`event-sourcing` and CQRS (`Command & Query Responsibility Segragation`_) techniques
+When building systems using :ref:`event-sourcing-scala` and CQRS (`Command & Query Responsibility Segragation`_) techniques
 it is tremendously important to realise that the write-side has completely different needs from the read-side,
 and separating those concerns into datastores that are optimised for either side makes it possible to offer the best
 expirience for the write and read sides independently.

akka-docs/rst/java/persistence.rst

@@ -379,6 +379,32 @@ restarts of the persistent actor.
 
 .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem
 
+.. _safe-shutdown-java:
+
+Safely shutting down persistent actors
+--------------------------------------
+
+Special care should be given when when shutting down persistent actors from the outside.
+With normal Actors it is often acceptable to use the special :ref:`PoisonPill <poison-pill-java>` message
+to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled
+automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill.
+
+This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while
+the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used.
+Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the
+confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill
+before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor.
+
+.. warning::
+  Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors.
+
+The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing
+mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used:
+
+.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown
+.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown-example-bad
+.. includecode:: code/docs/persistence/PersistenceDocTest.java#safe-shutdown-example-good
+
 .. _persistent-views-java:
 
 Persistent Views

akka-docs/rst/project/migration-guide-eventsourced-2.3.x.rst

@@ -72,7 +72,7 @@ Processors / PersistentActor
 
 **Akka Persistence:** ``PersistentActor``
 
-- Trait that adds journaling to actors (see :ref:`event-sourcing`) and used by applications for
+- Trait that adds journaling to actors (see :ref:`event-sourcing-scala`) and used by applications for
   *event sourcing* or *command sourcing*. Corresponds to ``Eventsourced`` processors in Eventsourced but is not a stackable trait.
 - Automatically recovers on start and re-start, by default. :ref:`recovery` can be customized or turned off by
   overriding actor life cycle hooks ``preStart`` and ``preRestart``. ``Processor`` takes care that new messages

akka-docs/rst/project/migration-guide-persistence-experimental-2.3.x-2.4.x.rst

@@ -34,7 +34,7 @@ To extend ``PersistentActor``::
       /*...*/
     }
 
-Read more about the persistent actor in the :ref:`documentation for Scala <event-sourcing>` and 
+Read more about the persistent actor in the :ref:`documentation for Scala <event-sourcing-scala>` and
 :ref:`documentation for Java <event-sourcing-java>`.
 
 Changed processorId to (abstract) persistenceId
@@ -183,7 +183,7 @@ acknowledgement from the channel is needed to guarantee safe hand-off. Therefore
 delivery is provided in a new ``AtLeastOnceDelivery`` trait that is mixed-in to the
 persistent actor on the sending side. 
 
-Read more about at-least-once delivery in the :ref:`documentation for Scala <at-least-once-delivery>` and 
+Read more about at-least-once delivery in the :ref:`documentation for Scala <at-least-once-delivery-scala>` and
 :ref:`documentation for Java <at-least-once-delivery-java>`.  
 
 Default persistence plugins

akka-docs/rst/scala/actors.rst

@@ -426,7 +426,7 @@ result:
   It is always preferable to communicate with other Actors using their ActorRef
   instead of relying upon ActorSelection. Exceptions are
 
-    * sending messages using the :ref:`at-least-once-delivery` facility
+    * sending messages using the :ref:`at-least-once-delivery-scala` facility
     * initiating first contact with a remote system
 
   In all other cases ActorRefs can be provided during Actor creation or

akka-docs/rst/scala/code/docs/persistence/PersistenceDocSpec.scala

@@ -4,7 +4,7 @@
 
 package docs.persistence
 
-import akka.actor.{ Actor, ActorRef, ActorSystem, Props }
+import akka.actor._
 import akka.pattern.BackoffSupervisor
 import akka.persistence._
 
@@ -102,7 +102,7 @@ object PersistenceDocSpec {
 
   object AtLeastOnce {
     //#at-least-once-example
-    import akka.actor.{ Actor, ActorPath, ActorSelection }
+    import akka.actor.{ Actor, ActorSelection }
     import akka.persistence.AtLeastOnceDelivery
 
     case class Msg(deliveryId: Long, s: String)
@@ -358,6 +358,107 @@ object PersistenceDocSpec {
     //#nested-persistAsync-persistAsync-caller
   }
 
+  object AvoidPoisonPill {
+
+    //#safe-shutdown
+    /** Explicit shutdown message */
+    case object Shutdown
+
+    class SafePersistentActor extends PersistentActor {
+      override def persistenceId = "safe-actor"
+
+      override def receiveCommand: Receive = {
+        case c: String =>
+          println(c)
+          persist(s"handle-$c") { println(_) }
+        case Shutdown =>
+          context.stop(self)
+      }
+
+      override def receiveRecover: Receive = {
+        case _ => // handle recovery here
+      }
+    }
+    //#safe-shutdown
+
+    //#safe-shutdown-example-bad
+    // UN-SAFE, due to PersistentActor's command stashing:
+    persistentActor ! "a"
+    persistentActor ! "b"
+    persistentActor ! PoisonPill
+    // order of received messages:
+    // a
+    //   # b arrives at mailbox, stashing;        internal-stash = [b]
+    //   # PoisonPill arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+    // PoisonPill is an AutoReceivedMessage, is handled automatically
+    // !! stop !!
+    // Actor is stopped without handling `b` nor the `a` handler!
+    //#safe-shutdown-example-bad
+
+    //#safe-shutdown-example-good
+    // SAFE:
+    persistentActor ! "a"
+    persistentActor ! "b"
+    persistentActor ! Shutdown
+    // order of received messages:
+    // a
+    //   # b arrives at mailbox, stashing;        internal-stash = [b]
+    //   # Shutdown arrives at mailbox, stashing; internal-stash = [b, Shutdown]
+    // handle-a
+    //   # unstashing;                            internal-stash = [Shutdown]
+    // b
+    // handle-b
+    //   # unstashing;                            internal-stash = []
+    // Shutdown
+    // -- stop --
+    //#safe-shutdown-example-good
+
+    class MyPersistAsyncActor extends PersistentActor {
+      override def persistenceId = "my-stable-persistence-id"
+
+      override def receiveRecover: Receive = {
+        case _ => // handle recovery here
+      }
+
+      //#nested-persistAsync-persistAsync
+      override def receiveCommand: Receive = {
+        case c: String =>
+          sender() ! c
+          persistAsync(c + "-outer-1") { outer ⇒
+            sender() ! outer
+            persistAsync(c + "-inner-1") { inner ⇒ sender() ! inner }
+          }
+          persistAsync(c + "-outer-2") { outer ⇒
+            sender() ! outer
+            persistAsync(c + "-inner-2") { inner ⇒ sender() ! inner }
+          }
+      }
+      //#nested-persistAsync-persistAsync
+    }
+
+    //#nested-persistAsync-persistAsync-caller
+    persistentActor ! "a"
+    persistentActor ! "b"
+
+    // order of received messages:
+    // a
+    // b
+    // a-outer-1
+    // a-outer-2
+    // b-outer-1
+    // b-outer-2
+    // a-inner-1
+    // a-inner-2
+    // b-inner-1
+    // b-inner-2
+
+    // which can be seen as the following causal relationship:
+    // a -> a-outer-1 -> a-outer-2 -> a-inner-1 -> a-inner-2
+    // b -> b-outer-1 -> b-outer-2 -> b-inner-1 -> b-inner-2
+
+    //#nested-persistAsync-persistAsync-caller
+  }
+
   object View {
     import akka.actor.Props
 

akka-docs/rst/scala/persistence-query.rst

@@ -82,7 +82,7 @@ If your usage does not require a live stream, you can disable refreshing by usin
 
 .. includecode:: code/docs/persistence/query/PersistenceQueryDocSpec.scala#all-persistence-ids-snap
 
-``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor <event-sourcing>`,
+``EventsByPersistenceId`` is a query equivalent to replaying a :ref:`PersistentActor <event-sourcing-scala>`,
 however, since it is a stream it is possible to keep it alive and watch for additional incoming events persisted by the
 persistent actor identified by the given ``persistenceId``. Most journals will have to revert to polling in order to achieve
 this, which can be configured using the ``RefreshInterval`` query hint:
@@ -134,7 +134,7 @@ specialised query object, as demonstrated in the sample below:
 
 Performance and denormalization
 ===============================
-When building systems using :ref:`event-sourcing` and CQRS (`Command & Query Responsibility Segragation`_) techniques
+When building systems using :ref:`event-sourcing-scala` and CQRS (`Command & Query Responsibility Segragation`_) techniques
 it is tremendously important to realise that the write-side has completely different needs from the read-side,
 and separating those concerns into datastores that are optimised for either side makes it possible to offer the best
 expirience for the write and read sides independently.

akka-docs/rst/scala/persistence.rst

@@ -61,7 +61,7 @@ Architecture
 
 .. _Community plugins: http://akka.io/community/
 
-.. _event-sourcing:
+.. _event-sourcing-scala:
 
 Event sourcing
 ==============
@@ -366,6 +366,31 @@ restarts of the persistent actor.
 
 .. _Thundering herd problem: https://en.wikipedia.org/wiki/Thundering_herd_problem
 
+.. _safe-shutdown-scala:
+
+Safely shutting down persistent actors
+--------------------------------------
+
+Special care should be given when when shutting down persistent actors from the outside.
+With normal Actors it is often acceptable to use the special :ref:`PoisonPill <poison-pill-scala>` message
+to signal to an Actor that it should stop itself once it receives this message – in fact this message is handled
+automatically by Akka, leaving the target actor no way to refuse stopping itself when given a poison pill.
+
+This can be dangerous when used with :class:`PersistentActor` due to the fact that incoming commands are *stashed* while
+the persistent actor is awaiting confirmation from the Journal that events have been written when ``persist()`` was used.
+Since the incoming commands will be drained from the Actor's mailbox and put into it's internal stash while awaiting the
+confirmation (thus, before calling the persist handlers) the Actor **may receive and (auto)handle the PoisonPill
+before it processes the other messages which have been put into its stash**, causing a pre-mature shutdown of the Actor.
+
+.. warning::
+  Consider using explicit shut-down messages instead of :class:`PoisonPill` when working with persistent actors.
+
+The example below highlights how messages arrive in the Actor's mailbox and how they interact with it's internal stashing
+mechanism when ``persist()`` is used, notice the early stop behaviour that occurs when ``PoisonPill`` is used:
+
+.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown
+.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown-example-bad
+.. includecode:: code/docs/persistence/PersistenceDocSpec.scala#safe-shutdown-example-good
 
 .. _persistent-views:
 
@@ -519,7 +544,7 @@ If failure messages are left unhandled by the actor, a default warning log messa
 No default action is performed on the success messages, however you're free to handle them e.g. in order to delete
 an in memory representation of the snapshot, or in the case of failure to attempt save the snapshot again.
 
-.. _at-least-once-delivery:
+.. _at-least-once-delivery-scala:
 
 At-Least-Once Delivery
 ======================

akka-docs/rst/scala/testing.rst

@@ -62,8 +62,8 @@ section below.
 .. warning::
   Due to the synchronous nature of ``TestActorRef`` it will **not** work with some support
   traits that Akka provides as they require asynchronous behaviours to function properly.
-  Examples of traits that do not mix well with test actor refs are :ref:`PersistentActor <event-sourcing>`
-  and :ref:`AtLeastOnceDelivery <at-least-once-delivery>` provided by :ref:`Akka Persistence <persistence-scala>`.
+  Examples of traits that do not mix well with test actor refs are :ref:`PersistentActor <event-sourcing-scala>`
+  and :ref:`AtLeastOnceDelivery <at-least-once-delivery-scala>` provided by :ref:`Akka Persistence <persistence-scala>`.
 
 Obtaining a Reference to an :class:`Actor`
 ------------------------------------------