=3974 per Persist (serialize) actor refs with transport info

* The reason for the problem with NoSuchElementException in ClusterSharding was
  that actor references were not serialized with full address information. In
  certain fail over scenarios the references could not be resolved and therefore
  the ShardRegionTerminated did not match corresponding ShardRegionRegistered.
* Wrap serialization with transport information from defaultAddress

(cherry picked from commit 3e73ae5925cf1293a9a5d61e48919b1708e84df2)

akka-contrib/src/main/scala/akka/contrib/pattern/ClusterSharding.scala

@@ -1198,7 +1198,13 @@ class ShardCoordinator(handOffTimeout: FiniteDuration, rebalanceInterval: Finite
       case ShardRegionProxyRegistered(proxy) ⇒
         persistentState = persistentState.updated(evt)
       case ShardRegionTerminated(region) ⇒
+        if (persistentState.regions.contains(region))
           persistentState = persistentState.updated(evt)
+        else {
+          log.debug("ShardRegionTerminated, but region {} was not registered. This inconsistency is due to that " +
+            " some stored ActorRef in Akka v2.3.0 and v2.3.1 did not contain full address information. It will be " +
+            "removed by later watch.", region)
+        }
       case ShardRegionProxyTerminated(proxy) ⇒
         persistentState = persistentState.updated(evt)
       case _: ShardHomeAllocated ⇒

akka-persistence/src/main/scala/akka/persistence/serialization/MessageSerializer.scala

@@ -36,6 +36,12 @@ class MessageSerializer(val system: ExtendedActorSystem) extends Serializer {
   def identifier: Int = 7
   def includeManifest: Boolean = true
 
+  private lazy val transportInformation: Option[Serialization.Information] = {
+    val address = system.provider.getDefaultAddress
+    if (address.hasLocalScope) None
+    else Some(Serialization.Information(address, system))
+  }
+
   /**
    * Serializes [[PersistentBatch]], [[PersistentRepr]] and [[Deliver]] messages. Delegates
    * serialization of a persistent message's payload to a matching `akka.serialization.Serializer`.
@@ -103,6 +109,7 @@ class MessageSerializer(val system: ExtendedActorSystem) extends Serializer {
   }
 
   private def persistentPayloadBuilder(payload: AnyRef) = {
+    def payloadBuilder() = {
       val serializer = SerializationExtension(system).findSerializerFor(payload)
       val builder = PersistentPayload.newBuilder()
 
@@ -113,6 +120,13 @@ class MessageSerializer(val system: ExtendedActorSystem) extends Serializer {
       builder
     }
 
+    // serialize actor references with full address information (defaultAddress) 
+    transportInformation match {
+      case Some(ti) ⇒ Serialization.currentTransportInformation.withValue(ti) { payloadBuilder() }
+      case None     ⇒ payloadBuilder()
+    }
+  }
+
   private def deliveredMessageBuilder(delivered: Delivered) = {
     val builder = DeliveredMessage.newBuilder
 

akka-persistence/src/main/scala/akka/persistence/serialization/SnapshotSerializer.scala

@@ -6,9 +6,9 @@
 package akka.persistence.serialization
 
 import java.io._
-
 import akka.actor._
 import akka.serialization.{ Serializer, SerializationExtension }
+import akka.serialization.Serialization
 
 /**
  * Wrapper for snapshot `data`. Snapshot `data` are the actual snapshot objects captured by
@@ -32,6 +32,12 @@ class SnapshotSerializer(system: ExtendedActorSystem) extends Serializer {
   def identifier: Int = 8
   def includeManifest: Boolean = false
 
+  private lazy val transportInformation: Option[Serialization.Information] = {
+    val address = system.provider.getDefaultAddress
+    if (address.hasLocalScope) None
+    else Some(Serialization.Information(address, system))
+  }
+
   /**
    * Serializes a [[Snapshot]]. Delegates serialization of snapshot `data` to a matching
    * `akka.serialization.Serializer`.
@@ -49,6 +55,7 @@ class SnapshotSerializer(system: ExtendedActorSystem) extends Serializer {
     Snapshot(snapshotFromBinary(bytes))
 
   private def snapshotToBinary(snapshot: AnyRef): Array[Byte] = {
+    def serialize() = {
       val extension = SerializationExtension(system)
 
       val snapshotSerializer = extension.findSerializerFor(snapshot)
@@ -67,6 +74,13 @@ class SnapshotSerializer(system: ExtendedActorSystem) extends Serializer {
       out.toByteArray
     }
 
+    // serialize actor references with full address information (defaultAddress) 
+    transportInformation match {
+      case Some(ti) ⇒ Serialization.currentTransportInformation.withValue(ti) { serialize() }
+      case None     ⇒ serialize()
+    }
+  }
+
   private def snapshotFromBinary(bytes: Array[Byte]): AnyRef = {
     val extension = SerializationExtension(system)