add CoordinatedShutdown, #21537

* CoordinatedShutdown that can run tasks for configured phases in order (DAG)
* coordinate handover/shutdown of singleton with cluster exiting/shutdown
* phase config obj with depends-on list
* integrate graceful leaving of sharding in coordinated shutdown
* add timeout and recover
* add some missing artery ports to tests
* leave via CoordinatedShutdown.run
* optionally exit-jvm in last phase
* run via jvm shutdown hook
* send ExitingConfirmed to leader before shutdown of Exiting
  to not have to wait for failure detector to mark it as
  unreachable before removing
* the unreachable signal is still kept as a safe guard if
  message is lost or leader dies
* PhaseClusterExiting vs MemberExited in ClusterSingletonManager
* terminate ActorSystem when cluster shutdown (via Down)
* add more predefined and custom phases
* reference documentation
* migration guide
* problem when the leader order was sys2, sys1, sys3,
  then sys3 could not perform it's duties and move Leving sys1 to
  Exiting because it was observing sys1 as unreachable
* exclude Leaving with exitingConfirmed from convergence condidtion

akka-docs/rst/java/code/docs/actorlambda/ActorDocTest.java

@@ -12,11 +12,17 @@ import akka.testkit.TestEvent;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import docs.AbstractJavaTest;
+import docs.actor.ActorDocTest.FirstActor;
 import scala.PartialFunction;
 import scala.runtime.BoxedUnit;
 import static docs.actorlambda.Messages.Swap.Swap;
 import static docs.actorlambda.Messages.*;
 import static akka.japi.Util.immutableSeq;
+import akka.actor.CoordinatedShutdown;
+import static akka.pattern.PatternsCS.ask;
+import akka.util.Timeout;
+import akka.Done;
+import java.util.concurrent.CompletionStage;
 
 import java.util.concurrent.TimeUnit;
 
@@ -677,5 +683,31 @@ public class ActorDocTest extends AbstractJavaTest {
       system.eventStream().publish(new TestEvent.UnMute(immutableSeq(ignoreExceptions)));
     }
   }
+  
+  @Test
+  public void coordinatedShutdown() {
+    final ActorRef someActor = system.actorOf(Props.create(FirstActor.class));
+    //#coordinated-shutdown-addTask
+    CoordinatedShutdown.get(system).addTask(
+      CoordinatedShutdown.PhaseBeforeServiceUnbind(), "someTaskName",
+      () -> {
+        return ask(someActor, "stop", new Timeout(5, TimeUnit.SECONDS))
+          .thenApply(reply -> Done.getInstance());
+    });
+    //#coordinated-shutdown-addTask
+
+    //#coordinated-shutdown-jvm-hook
+    CoordinatedShutdown.get(system).addJvmShutdownHook(() ->
+      System.out.println("custom JVM shutdown hook...")
+    );
+    //#coordinated-shutdown-jvm-hook
+
+    // don't run this
+    if (false) {
+      //#coordinated-shutdown-run
+      CompletionStage<Done> done = CoordinatedShutdown.get(system).runAll();
+      //#coordinated-shutdown-run
+    }
+  }
 
 }