catch NotSerializableException from deserialization, #20641

* to be able to introduce new messages and still support rolling upgrades,
  i.e. a cluster of mixed versions
* note that it's only catching NotSerializableException, which we already
  use for unknown serializer ids and class manifests
* note that it is not catching for system messages, since that could result
  in infinite resending

akka-remote/src/main/scala/akka/remote/Endpoint.scala

@@ -984,7 +984,18 @@ private[remote] class EndpointReader(
           if (msg.reliableDeliveryEnabled) {
             ackedReceiveBuffer = ackedReceiveBuffer.receive(msg)
             deliverAndAck()
-          } else msgDispatch.dispatch(msg.recipient, msg.recipientAddress, msg.serializedMessage, msg.senderOption)
+          } else try
+            msgDispatch.dispatch(msg.recipient, msg.recipientAddress, msg.serializedMessage, msg.senderOption)
+          catch {
+            case e: NotSerializableException ⇒
+              val sm = msg.serializedMessage
+              log.warning(
+                "Serializer not defined for message with serializer id [{}] and manifest [{}]. " +
+                  "Transient association error (association remains live). {}",
+                sm.getSerializerId,
+                if (sm.hasMessageManifest) sm.getMessageManifest.toStringUtf8 else "",
+                e.getMessage)
+          }
 
         case None ⇒
       }