+act,mul #3948 add MultiNodeSpec.startNewSsytem() and system.abort()

abort() currently only changes that remote-deployed child actors are not
waited for during termination (because that would not change anything);
it is still a different operation than shutdown() since it changes what
you are guaranteed to observe after termination.

testConductor.shutdown(..., abort = true) uses this mode of termination.

improve MultiNodeSpec to allow injection of deployment configuration
into arbitrary actor systems

include number of received elements in the timeout failure message for
TestKit.receiveN

akka-multi-node-testkit/src/main/scala/akka/remote/testkit/MultiNodeSpec.scala

@@ -327,7 +327,7 @@ abstract class MultiNodeSpec(val myself: RoleName, _system: ActorSystem, _roles:
    * been started either in Conductor or Player mode when the constructor of
    * MultiNodeSpec finishes, i.e. do not call the start*() methods yourself!
    */
-  val testConductor: TestConductorExt = TestConductor(system)
+  var testConductor: TestConductorExt = null
 
   /**
    * Execute the given block of code only on the given nodes (names according
@@ -376,52 +376,85 @@ abstract class MultiNodeSpec(val myself: RoleName, _system: ActorSystem, _roles:
    */
 
   private val controllerAddr = new InetSocketAddress(serverName, serverPort)
-  if (selfIndex == 0) {
-    Await.result(testConductor.startController(initialParticipants, myself, controllerAddr),
-      testConductor.Settings.BarrierTimeout.duration)
-  } else {
-    Await.result(testConductor.startClient(myself, controllerAddr),
-      testConductor.Settings.BarrierTimeout.duration)
+
+  protected def attachConductor(tc: TestConductorExt): Unit = {
+    val timeout = tc.Settings.BarrierTimeout.duration
+    val startFuture =
+      if (selfIndex == 0) tc.startController(initialParticipants, myself, controllerAddr)
+      else tc.startClient(myself, controllerAddr)
+    try Await.result(startFuture, timeout)
+    catch {
+      case NonFatal(x) ⇒ throw new RuntimeException("failure while attaching new conductor", x)
+    }
+    testConductor = tc
   }
 
+  attachConductor(TestConductor(system))
+
   // now add deployments, if so desired
 
   private final case class Replacement(tag: String, role: RoleName) {
     lazy val addr = node(role).address.toString
   }
+
   private val replacements = roles map (r ⇒ Replacement("@" + r.name + "@", r))
-  private val deployer = system.asInstanceOf[ExtendedActorSystem].provider.deployer
-  deployments(myself) foreach { str ⇒
-    val deployString = (str /: replacements) {
-      case (base, r @ Replacement(tag, _)) ⇒
-        base.indexOf(tag) match {
-          case -1 ⇒ base
-          case start ⇒
-            val replaceWith = try
-              r.addr
-            catch {
-              case NonFatal(e) ⇒
-                // might happen if all test cases are ignored (excluded) and
-                // controller node is finished/exited before r.addr is run
-                // on the other nodes
-                val unresolved = "akka://unresolved-replacement-" + r.role.name
-                log.warning(unresolved + " due to: " + e.getMessage)
-                unresolved
-            }
-            base.replace(tag, replaceWith)
-        }
-    }
-    import scala.collection.JavaConverters._
-    ConfigFactory.parseString(deployString).root.asScala foreach {
-      case (key, value: ConfigObject) ⇒ deployer.parseConfig(key, value.toConfig) foreach deployer.deploy
-      case (key, x)                   ⇒ throw new IllegalArgumentException(s"key $key must map to deployment section, not simple value $x")
+
+  protected def injectDeployments(sys: ActorSystem, role: RoleName): Unit = {
+    val deployer = sys.asInstanceOf[ExtendedActorSystem].provider.deployer
+    deployments(role) foreach { str ⇒
+      val deployString = (str /: replacements) {
+        case (base, r @ Replacement(tag, _)) ⇒
+          base.indexOf(tag) match {
+            case -1 ⇒ base
+            case start ⇒
+              val replaceWith = try
+                r.addr
+              catch {
+                case NonFatal(e) ⇒
+                  // might happen if all test cases are ignored (excluded) and
+                  // controller node is finished/exited before r.addr is run
+                  // on the other nodes
+                  val unresolved = "akka://unresolved-replacement-" + r.role.name
+                  log.warning(unresolved + " due to: " + e.getMessage)
+                  unresolved
+              }
+              base.replace(tag, replaceWith)
+          }
+      }
+      import scala.collection.JavaConverters._
+      ConfigFactory.parseString(deployString).root.asScala foreach {
+        case (key, value: ConfigObject) ⇒ deployer.parseConfig(key, value.toConfig) foreach deployer.deploy
+        case (key, x)                   ⇒ throw new IllegalArgumentException(s"key $key must map to deployment section, not simple value $x")
+      }
     }
   }
 
-  // useful to see which jvm is running which role, used by LogRoleReplace utility
-  log.info("Role [{}] started with address [{}]", myself.name,
-    system.asInstanceOf[ExtendedActorSystem].provider.asInstanceOf[RemoteActorRefProvider].transport.defaultAddress)
+  injectDeployments(system, myself)
 
+  protected val myAddress = system.asInstanceOf[ExtendedActorSystem].provider.getDefaultAddress
+
+  // useful to see which jvm is running which role, used by LogRoleReplace utility
+  log.info("Role [{}] started with address [{}]", myself.name, myAddress)
+
+  /**
+   * This method starts a new ActorSystem with the same configuration as the
+   * previous one on the current node, including deployments. It also creates
+   * a new TestConductor client and registers itself with the conductor so
+   * that it is possible to use barriers etc. normally after this method has
+   * been called.
+   *
+   * NOTICE: you MUST start a new system before trying to enter a barrier or
+   * otherwise using the TestConductor after having terminated this node’s
+   * system.
+   */
+  protected def startNewSystem(): ActorSystem = {
+    val config = ConfigFactory.parseString(s"akka.remote.netty.tcp{port=${myAddress.port.get}\nhostname=${myAddress.host.get}}")
+      .withFallback(system.settings.config)
+    val sys = ActorSystem(system.name, config)
+    injectDeployments(sys, myself)
+    attachConductor(TestConductor(sys))
+    sys
+  }
 }
 
 /**