Remove additional-serialization-bindings, #26684

This should be safe for a rolling update between Akka 2.5.x and 2.6.0
also if additional-bindings were disabled in 2.5.x because:
* if 2.6.0 sends one of those the serializer (akka-misc) exists in 2.5.x
  so deserialization will work via the serializerId
* if 2.5.x sends one of those with java serialization it can be
  deserialized in 2.6.0 since the java serializer exists
  (disabling java serialization by default is another ticket)

* historical problems with serialization of remote Deploy messages
  in Akka 2.4.x should be gone now
This commit is contained in:
Patrik Nordwall 2019-07-03 13:00:47 +02:00
parent 2db03309ce
commit 3f4179c316
12 changed files with 48 additions and 236 deletions

View file

@ -116,10 +116,7 @@ object SerializationTests {
def mostlyReferenceSystem: ActorSystem = {
val referenceConf = ConfigFactory.defaultReference()
// we are checking the old Java serialization formats here
val mostlyReferenceConf = ConfigFactory.parseString("""
akka.actor.enable-additional-serialization-bindings = off
""").withFallback(AkkaSpec.testConf.withFallback(referenceConf))
val mostlyReferenceConf = AkkaSpec.testConf.withFallback(referenceConf)
ActorSystem("SerializationSystem", mostlyReferenceConf)
}

View file

@ -114,10 +114,6 @@ akka {
# this is only intended for testing.
serialize-messages = off
# Additional serialization bindings which are enabled automatically when allow-java-serialization is disabled.
# settings are provided
java-serialization-disabled-additional-serialization-bindings = {}
# Serializes and deserializes creators (in Props) to ensure that they can be
# sent over the network, this is only intended for testing. Purely local deployments
# as marked with deploy.scope == LocalScope are exempt from verification.
@ -697,7 +693,6 @@ akka {
#
# This setting is a short-cut to
# - using DisabledJavaSerializer instead of JavaSerializer
# - enable-additional-serialization-bindings = on
#
# Completely disable the use of `akka.serialization.JavaSerialization` by the
# Akka Serialization extension, instead DisabledJavaSerializer will
@ -731,21 +726,6 @@ akka {
"java.io.Serializable" = java
}
# Additional serialization-bindings that are replacing Java serialization are
# defined in this section for backwards compatibility reasons. They are included
# by default but can be excluded for backwards compatibility with Akka 2.4.x.
# They can be disabled with enable-additional-serialization-bindings=off.
#
# This should only be needed for backwards compatibility reasons.
enable-additional-serialization-bindings = on
# Additional serialization-bindings that are replacing Java serialization are
# defined in this section for backwards compatibility reasons. They are included
# by default but can be excluded for backwards compatibility with Akka 2.4.x.
# They can be disabled with enable-additional-serialization-bindings=off.
additional-serialization-bindings {
}
# Log warnings when the default Java serialization is used to serialize messages.
# The default serializer uses Java serialization which is not very performant and should not
# be used in production environments unless you don't care about performance. In that case

View file

@ -370,8 +370,8 @@ object ActorSystem {
final val UnstartedPushTimeout: Timeout = Timeout(config.getMillisDuration("akka.actor.unstarted-push-timeout"))
final val AllowJavaSerialization: Boolean = getBoolean("akka.actor.allow-java-serialization")
final val EnableAdditionalSerializationBindings: Boolean =
!AllowJavaSerialization || getBoolean("akka.actor.enable-additional-serialization-bindings")
@deprecated("Always enabled from Akka 2.6.0", "2.6.0")
final val EnableAdditionalSerializationBindings: Boolean = true
final val SerializeAllMessages: Boolean = getBoolean("akka.actor.serialize-messages")
final val SerializeAllCreators: Boolean = getBoolean("akka.actor.serialize-creators")

View file

@ -42,23 +42,7 @@ object Serialization {
class Settings(val config: Config) {
val Serializers: Map[String, String] = configToMap(config.getConfig("akka.actor.serializers"))
val SerializationBindings: Map[String, String] = {
val defaultBindings = config.getConfig("akka.actor.serialization-bindings")
val bindings = {
if (config.getBoolean("akka.actor.enable-additional-serialization-bindings") ||
!config.getBoolean("akka.actor.allow-java-serialization") ||
config.hasPath("akka.remote.artery.enabled") && config.getBoolean("akka.remote.artery.enabled")) {
val bs = defaultBindings.withFallback(config.getConfig("akka.actor.additional-serialization-bindings"))
// in addition to the additional settings, we also enable even more bindings if java serialization is disabled:
val additionalWhenJavaOffKey = "akka.actor.java-serialization-disabled-additional-serialization-bindings"
if (!config.getBoolean("akka.actor.allow-java-serialization")) {
bs.withFallback(config.getConfig(additionalWhenJavaOffKey))
} else bs
} else {
defaultBindings
}
}
val bindings = config.getConfig("akka.actor.serialization-bindings")
configToMap(bindings)
}

View file

@ -14,7 +14,6 @@ class MessageSerializerSpec extends AkkaSpec("""
akka.actor.provider = cluster
akka.actor.serialize-messages = off
akka.actor.allow-java-serialization = off
akka.actor.enable-additional-serialization-bindings = on
""") {
val serializer = new MessageSerializer(system.asInstanceOf[ExtendedActorSystem])

View file

@ -45,14 +45,11 @@ akka.actor {
}
serialization-bindings {
"akka.cluster.pubsub.DistributedPubSubMessage" = akka-pubsub
"akka.cluster.pubsub.DistributedPubSubMediator$Internal$SendToOneSubscriber" = akka-pubsub
}
serialization-identifiers {
"akka.cluster.pubsub.protobuf.DistributedPubSubMessageSerializer" = 9
}
# adds the protobuf serialization of pub sub messages to groups
additional-serialization-bindings {
"akka.cluster.pubsub.DistributedPubSubMediator$Internal$SendToOneSubscriber" = akka-pubsub
}
}

View file

@ -146,7 +146,3 @@ manual exception overrides may be put in place if the change happened to be in a
Scala does not maintain serialization compatibility across major versions. This means that if Java serialization is used
there is no guarantee objects can be cleanly deserialized if serialized with a different version of Scala.
The internal Akka Protobuf serializers that can be enabled explicitly with `enable-additional-serialization-bindings`
or implicitly with `akka.actor.allow-java-serialization = off` (which is preferable from a security standpoint)
does not suffer from this problem.

View file

@ -55,26 +55,6 @@ implements `java.io.Serializable`, protobuf messages will always be
serialized using the protobuf protocol unless specifically overridden. In order
to disable a default serializer, see @ref:[Disabling the Java Serializer](remoting-artery.md#disable-java-serializer)
### Enable additional bindings
A few types in Akka are, for backwards-compatibility reasons, still serialized by using Java serializer by default.
You can switch them to using protocol buffers instead by adding the following bindings or set `akka.actor.allow-java-serialization=off`, which will make them serialized using protocol buffers instead.
Refer to @ref[Rolling Upgrades](#rolling-upgrades) to understand how it is possible to turn and start using these new
serializers in your clustered applications.
You can enable them one by one adding by adding their bindings to the misc serializer, like this:
```
akka.actor.serialization-bindings {
"akka.Done" = akka-misc
"akka.NotUsed" = akka-misc
"akka.actor.Address" = akka-misc
"akka.remote.UniqueAddress" = akka-misc
}
```
Alternatively, you can disable all Java serialization which then automatically will add the `java-serialization-disabled-additional-serialization-bindings` bindings to the active bindings.
### Verification
Normally, messages sent between local actors (i.e. same JVM) do not undergo serialization. For testing, sometimes, it may be desirable to force serialization on all messages (both remote and local). If you want to do this in order to verify that your messages are serializable you can enable the following config option:
@ -258,57 +238,14 @@ The recommended approach to do deep serialization of internal actor state is to
<a id="disable-java-serializer"></a>
## Disabling the Java Serializer
Since the `2.4.11` release of Akka it is possible to entirely disable the default Java Serialization mechanism.
For compatibility reasons, the current (non-Artery) @ref:[Remoting](remoting.md) still uses Java
serialization for some classes, however you can disable it in this remoting implementation as well by following
the steps below.
The first step is to enable some additional serializers that replace previous Java serialization of some internal
messages. This is recommended also when you can't disable Java serialization completely. Those serializers are
enabled with this configuration:
```ruby
akka.actor {
# Set this to on to enable serialization-bindings defined in
# additional-serialization-bindings. Those are by default not included
# for backwards compatibility reasons. They are enabled by default if
# akka.remote.artery.enabled=on.
enable-additional-serialization-bindings = on
}
```
The reason these are not enabled by default is wire-level compatibility between any 2.4.x Actor Systems.
If you roll out a new cluster, all on the same Akka version that can enable these serializers it is recommended to
enable this setting. When using @ref:[Remoting (codename Artery)](remoting-artery.md) these serializers are enabled by default.
@@@ warning
Please note that when enabling the `additional-serialization-bindings` when using the old remoting,
you must do so on all nodes participating in a cluster, otherwise the mis-aligned serialization
configurations will cause deserialization errors on the receiving nodes.
@@@
Java serialization is known to be slow and [prone to attacks](https://community.hpe.com/t5/Security-Research/The-perils-of-Java-deserialization/ba-p/6838995)
of various kinds - it never was designed for high throughput messaging after all. However, it is very
convenient to use, thus it remained the default serialization mechanism that Akka used to
serialize user messages as well as some of its internal messages in previous versions.
Since the release of Artery, Akka internals do not rely on Java serialization anymore (one exception being `java.lang.Throwable`).
@@@ warning
Please note Akka 2.5 by default does not use any Java Serialization for its own internal messages, unlike 2.4 where
by default it still did for a few of the messages. If you want an 2.4.x system to communicate with a 2.5.x series, for
example during a rolling deployment you should first enable `additional-serialization-bindings` on the old systems.
You must do so on all nodes participating in a cluster, otherwise the mis-aligned serialization
configurations will cause deserialization errors on the receiving nodes. These additional serialization bindings are
enabled by default in Akka 2.5.x.
@@@
@@@ note
When using the new remoting implementation (codename Artery), Akka does not use Java Serialization for any of its internal messages.
Akka does not use Java Serialization for any of its internal messages.
It is highly encouraged to disable java serialization, so please plan to do so at the earliest possibility you have in your project.
One may think that network bandwidth and latency limit the performance of remote messaging, but serialization is a more typical bottleneck.
@ -332,8 +269,6 @@ This will completely disable the use of `akka.serialization.JavaSerialization` b
Akka Serialization extension, instead `DisabledJavaSerializer` will
be inserted which will fail explicitly if attempts to use java serialization are made.
It will also enable the above mentioned `enable-additional-serialization-bindings`.
The log messages emitted by such serializer SHOULD be treated as potential
attacks which the serializer prevented, as they MAY indicate an external operator
attempting to send malicious messages intending to use java serialization as attack vector.
@ -346,11 +281,6 @@ Please note that this option does not stop you from manually invoking java seria
It is not safe to mix major Scala versions when using the Java serialization as Scala does not guarantee compatibility
and this could lead to very surprising errors.
If using the Akka Protobuf serializers (implicitly with `akka.actor.allow-java-serialization = off` or explicitly with
`enable-additional-serialization-bindings = true`) for the internal Akka messages those will not require the same major
Scala version however you must also ensure the serializers used for your own types does not introduce the same
incompatibility as Java serialization does.
## Rolling upgrades
A serialized remote message (or persistent event) consists of serializer-id, the manifest, and the binary payload.

View file

@ -48,28 +48,11 @@ akka {
# i.e. com.google.protobuf dependency has been added in the application project.
"com.google.protobuf.GeneratedMessage" = proto
"java.util.Optional" = akka-misc
# The following are handled by the MiscMessageSerializer, but they are not enabled for
# compatibility reasons (it was added in Akka 2.5.[8,9,12]). Enable them by adding:
# akka.actor.serialization-bindings {
# "akka.Done" = akka-misc
# "akka.NotUsed" = akka-misc
# "akka.actor.Address" = akka-misc
# "akka.remote.UniqueAddress" = akka-misc
# }
}
# Additional serialization-bindings that are replacing Java serialization are
# defined in this section for backwards compatibility reasons. They are included
# by default but can be excluded for backwards compatibility with Akka 2.4.x.
# They can be disabled with enable-additional-serialization-bindings=off.
additional-serialization-bindings {
"akka.actor.Identify" = akka-misc
"akka.actor.ActorIdentity" = akka-misc
"scala.Some" = akka-misc
"scala.None$" = akka-misc
"java.util.Optional" = akka-misc
"akka.actor.Status$Success" = akka-misc
"akka.actor.Status$Failure" = akka-misc
"akka.actor.ActorRef" = akka-misc
@ -77,28 +60,12 @@ akka {
"akka.actor.Kill$" = akka-misc
"akka.remote.RemoteWatcher$Heartbeat$" = akka-misc
"akka.remote.RemoteWatcher$HeartbeatRsp" = akka-misc
"akka.Done" = akka-misc
"akka.NotUsed" = akka-misc
"akka.actor.Address" = akka-misc
"akka.remote.UniqueAddress" = akka-misc
"akka.actor.ActorInitializationException" = akka-misc
"akka.dispatch.sysmsg.SystemMessage" = akka-system-msg
"java.lang.String" = primitive-string
"akka.util.ByteString$ByteString1C" = primitive-bytestring
"akka.util.ByteString$ByteString1" = primitive-bytestring
"akka.util.ByteString$ByteStrings" = primitive-bytestring
"java.lang.Long" = primitive-long
"scala.Long" = primitive-long
"java.lang.Integer" = primitive-int
"scala.Int" = primitive-int
# Java Serializer is by default used for exceptions.
# It's recommended that you implement custom serializer for exceptions that are
# sent remotely, e.g. in akka.actor.Status.Failure for ask replies. You can add
# binding to akka-misc (MiscMessageSerializerSpec) for the exceptions that have
# a constructor with single message String or constructor with message String as
# first parameter and cause Throwable as second parameter. Note that it's not
# safe to add this binding for general exceptions such as IllegalArgumentException
# because it may have a subclass without required constructor.
"java.lang.Throwable" = java
"akka.actor.IllegalActorStateException" = akka-misc
"akka.actor.ActorKilledException" = akka-misc
"akka.actor.InvalidActorNameException" = akka-misc
@ -125,14 +92,27 @@ akka {
"akka.routing.TailChoppingGroup" = akka-misc
"akka.routing.TailChoppingPool" = akka-misc
"akka.remote.routing.RemoteRouterConfig" = akka-misc
}
# Additional serialization bindings which are enabled automatically when allow-java-serialization is disabled.
java-serialization-disabled-additional-serialization-bindings = {
"akka.Done" = akka-misc
"akka.NotUsed" = akka-misc
"akka.actor.Address" = akka-misc
"akka.remote.UniqueAddress" = akka-misc
"akka.dispatch.sysmsg.SystemMessage" = akka-system-msg
"java.lang.String" = primitive-string
"akka.util.ByteString$ByteString1C" = primitive-bytestring
"akka.util.ByteString$ByteString1" = primitive-bytestring
"akka.util.ByteString$ByteStrings" = primitive-bytestring
"java.lang.Long" = primitive-long
"scala.Long" = primitive-long
"java.lang.Integer" = primitive-int
"scala.Int" = primitive-int
# Java Serializer is by default used for exceptions.
# It's recommended that you implement custom serializer for exceptions that are
# sent remotely, e.g. in akka.actor.Status.Failure for ask replies. You can add
# binding to akka-misc (MiscMessageSerializerSpec) for the exceptions that have
# a constructor with single message String or constructor with message String as
# first parameter and cause Throwable as second parameter. Note that it's not
# safe to add this binding for general exceptions such as IllegalArgumentException
# because it may have a subclass without required constructor.
"java.lang.Throwable" = java
}
serialization-identifiers {

View file

@ -76,13 +76,10 @@ class AllowJavaSerializationOffSpec
}
val addedJavaSerializationProgramaticallyButDisabledSettings = BootstrapSetup(
None,
Some(
ConfigFactory.parseString(
"""
Some(ConfigFactory.parseString("""
akka {
loglevel = debug
actor {
enable-additional-serialization-bindings = off # this should be overridden by the setting below, which should force it to be on
allow-java-serialization = off
# this is by default on, but tests are running with off, use defaults here
warn-about-java-serializer-usage = on
@ -110,15 +107,6 @@ class AllowJavaSerializationOffSpec
}
}
"enable additional-serialization-bindings" in {
val some = Some("foo")
val ser = SerializationExtension(dontAllowJavaSystem).findSerializerFor(some).asInstanceOf[MiscMessageSerializer]
val bytes = ser.toBinary(some)
ser.fromBinary(bytes, ser.manifest(some)) should ===(Some("foo"))
SerializationExtension(dontAllowJavaSystem).deserialize(bytes, ser.identifier, ser.manifest(some)).get should ===(
Some("foo"))
}
"have replaced java serializer" in {
val p = TestProbe()(dontAllowJavaSystem) // only receiver has the serialization disabled

View file

@ -4,12 +4,18 @@
package akka.remote.serialization
import akka.actor.ActorSystem
import akka.testkit.TestKit
import akka.actor.{ Actor, ActorRef, Address, Deploy, ExtendedActorSystem, Props, SupervisorStrategy }
import akka.remote.{ DaemonMsgCreate, RemoteScope }
import akka.routing.{ FromConfig, RoundRobinPool }
import akka.serialization.{ Serialization, SerializationExtension }
import akka.actor.Actor
import akka.actor.ActorRef
import akka.actor.Address
import akka.actor.Deploy
import akka.actor.Props
import akka.actor.SupervisorStrategy
import akka.remote.DaemonMsgCreate
import akka.remote.RemoteScope
import akka.routing.FromConfig
import akka.routing.RoundRobinPool
import akka.serialization.Serialization
import akka.serialization.SerializationExtension
import akka.testkit.AkkaSpec
import akka.util.unused
import com.typesafe.config.ConfigFactory
@ -99,53 +105,6 @@ class DaemonMsgCreateSerializerSpec extends AkkaSpec with SerializationVerificat
}
}
"deserialize the old wire format with just class and field for props parameters (if possible)" in {
val system = ActorSystem(
"DaemonMsgCreateSerializer-old-wire-format",
ConfigFactory.parseString("""
# old hex bytes contain actor ref with akka.tcp
akka.remote.artery.enabled = off
# in 2.4 this is off by default, but in 2.5+ its on so we wouldn't
# get the right set of serializers (and since the old wire protocol doesn't
# contain serializer ids that will go unnoticed with unpleasant consequences)
akka.actor.enable-additional-serialization-bindings = off
"""))
try {
val serializer = new DaemonMsgCreateSerializer(system.asInstanceOf[ExtendedActorSystem])
// the oldSnapshot was created with the version of DemonMsgCreateSerializer in Akka 2.4.17. See issue #22224.
// It was created with:
/*
import org.apache.commons.codec.binary.Hex.encodeHex
val bytes = serializer.toBinary(
DaemonMsgCreate(Props(classOf[MyActorWithParam], "a string"), Deploy.local, "/user/test", system.actorFor("/user")))
println(String.valueOf(encodeHex(bytes)))
*/
val oldBytesHex =
"0a7112020a001a48616b6b612e72656d6f74652e73657269616c697a617" +
"4696f6e2e4461656d6f6e4d736743726561746553657269616c697a6572" +
"53706563244d794163746f7257697468506172616d220faced000574000" +
"86120737472696e672a106a6176612e6c616e672e537472696e67122f0a" +
"00222baced000573720016616b6b612e6163746f722e4c6f63616c53636" +
"f706524000000000000000102000078701a0a2f757365722f7465737422" +
"2b0a29616b6b613a2f2f4461656d6f6e4d7367437265617465536572696" +
"16c697a6572537065632f75736572"
import org.apache.commons.codec.binary.Hex.decodeHex
val oldBytes = decodeHex(oldBytesHex.toCharArray)
val result = serializer.fromBinary(oldBytes, classOf[DaemonMsgCreate])
result match {
case dmc: DaemonMsgCreate =>
dmc.props.args should ===(Seq("a string": Any))
}
} finally {
TestKit.shutdownActorSystem(system)
}
}
"serialize and de-serialize DaemonMsgCreate with Deploy and RouterConfig" in {
verifySerialization {
// Duration.Inf doesn't equal Duration.Inf, so we use another for test

View file

@ -23,7 +23,9 @@ object MiscMessageSerializerSpec {
val serializationTestOverrides =
s"""
akka.actor {
serialization-bindings = { "akka.remote.serialization.MiscMessageSerializerSpec$$TestException" = akka-misc } $${akka.actor.java-serialization-disabled-additional-serialization-bindings}
serialization-bindings = {
"akka.remote.serialization.MiscMessageSerializerSpec$$TestException" = akka-misc
}
}
"""