Race in creating new remote child, see #3423

* RemoteSystemDaemon removes its child when sendSystemMessage
  DeathWatchNotification is called from tellWatchersWeDied.
  That can happen after Terminated is delivered to other watcher,
  which create a new child with same name, and then the new child
  can then be removed instead of the old one.
* RemoteSystemDaemon must only remove child with exact match of
  full ActorRef

akka-actor/src/main/scala/akka/actor/ActorRef.scala

@@ -550,13 +550,28 @@ private[akka] class VirtualPathContainer(
   def addChild(name: String, ref: InternalActorRef): Unit = {
     children.put(name, ref) match {
       case null ⇒ // okay
-      case old  ⇒ log.warning("{} replacing child {} ({} -> {})", path, name, old, ref)
+      case old ⇒
+        // this can happen from RemoteSystemDaemon if a new child is created
+        // before the old is removed from RemoteSystemDaemon children
+        log.debug("{} replacing child {} ({} -> {})", path, name, old, ref)
     }
   }
 
   def removeChild(name: String): Unit =
     if (children.remove(name) eq null) log.warning("{} trying to remove non-child {}", path, name)
 
+  /**
+   * Remove a named child if it matches the ref.
+   */
+  protected def removeChild(name: String, ref: ActorRef): Unit = {
+    val current = getChild(name)
+    if (current eq null)
+      log.warning("{} trying to remove non-child {}", path, name)
+    else if (current == ref)
+      children.remove(name, current) // remove when same value
+
+  }
+
   def getChild(name: String): InternalActorRef = children.get(name)
 
   override def getChild(name: Iterator[String]): InternalActorRef = {

akka-remote/src/main/scala/akka/remote/RemoteDaemon.scala

@@ -83,7 +83,7 @@ private[akka] class RemoteSystemDaemon(
   override def sendSystemMessage(message: SystemMessage): Unit = message match {
     case DeathWatchNotification(child: ActorRefWithCell with ActorRefScope, _, _) if child.isLocal ⇒
       terminating.locked {
-        removeChild(child.path.elements.drop(1).mkString("/"))
+        removeChild(child.path.elements.drop(1).mkString("/"), child)
         terminationHookDoneWhenNoChildren()
       }
     case _ ⇒ super.sendSystemMessage(message)