Merge pull request #23583 from akka/wip-multi-dc-merge-master-patriknw

merge wip-multi-dc-dev back to master

akka-cluster-sharding/src/main/mima-filters/2.5.4.backwards.excludes

@@ -0,0 +1,4 @@
+# #23231 multi-DC Sharding
+ProblemFilters.exclude[Problem]("akka.cluster.sharding.ClusterShardingGuardian*")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.sharding.ShardRegion.proxyProps")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.sharding.ShardRegion.this")

akka-cluster-sharding/src/main/scala/akka/cluster/sharding/ClusterSharding.scala

@@ -29,6 +29,8 @@ import akka.cluster.ddata.ReplicatorSettings
 import akka.cluster.ddata.Replicator
 import scala.util.control.NonFatal
 import akka.actor.Status
+import akka.cluster.ClusterSettings
+import akka.cluster.ClusterSettings.DataCenter
 
 /**
  * This extension provides sharding functionality of actors in a cluster.
@@ -341,16 +343,53 @@ class ClusterSharding(system: ExtendedActorSystem) extends Extension {
     typeName:        String,
     role:            Option[String],
     extractEntityId: ShardRegion.ExtractEntityId,
+    extractShardId:  ShardRegion.ExtractShardId): ActorRef =
+    startProxy(typeName, role, dataCenter = None, extractEntityId, extractShardId)
+
+  /**
+   * Scala API: Register a named entity type `ShardRegion` on this node that will run in proxy only mode,
+   * i.e. it will delegate messages to other `ShardRegion` actors on other nodes, but not host any
+   * entity actors itself. The [[ShardRegion]] actor for this type can later be retrieved with the
+   * [[#shardRegion]] method.
+   *
+   * Some settings can be configured as described in the `akka.cluster.sharding` section
+   * of the `reference.conf`.
+   *
+   * @param typeName the name of the entity type
+   * @param role specifies that this entity type is located on cluster nodes with a specific role.
+   *   If the role is not specified all nodes in the cluster are used.
+   * @param dataCenter The data center of the cluster nodes where the cluster sharding is running.
+   *   If None then the same data center as current node.
+   * @param extractEntityId partial function to extract the entity id and the message to send to the
+   *   entity from the incoming message, if the partial function does not match the message will
+   *   be `unhandled`, i.e. posted as `Unhandled` messages on the event stream
+   * @param extractShardId function to determine the shard id for an incoming message, only messages
+   *   that passed the `extractEntityId` will be used
+   * @return the actor ref of the [[ShardRegion]] that is to be responsible for the shard
+   */
+  def startProxy(
+    typeName:        String,
+    role:            Option[String],
+    dataCenter:      Option[DataCenter],
+    extractEntityId: ShardRegion.ExtractEntityId,
     extractShardId:  ShardRegion.ExtractShardId): ActorRef = {
 
     implicit val timeout = system.settings.CreationTimeout
     val settings = ClusterShardingSettings(system).withRole(role)
-    val startMsg = StartProxy(typeName, settings, extractEntityId, extractShardId)
+    val startMsg = StartProxy(typeName, dataCenter, settings, extractEntityId, extractShardId)
     val Started(shardRegion) = Await.result(guardian ? startMsg, timeout.duration)
-    regions.put(typeName, shardRegion)
+    // it must be possible to start several proxies, one per data center
+    regions.put(proxyName(typeName, dataCenter), shardRegion)
     shardRegion
   }
 
+  private def proxyName(typeName: String, dataCenter: Option[DataCenter]): String = {
+    dataCenter match {
+      case None    ⇒ typeName
+      case Some(t) ⇒ typeName + "-" + t
+    }
+  }
+
   /**
    * Java/Scala API: Register a named entity type `ShardRegion` on this node that will run in proxy only mode,
    * i.e. it will delegate messages to other `ShardRegion` actors on other nodes, but not host any
@@ -370,9 +409,34 @@ class ClusterSharding(system: ExtendedActorSystem) extends Extension {
   def startProxy(
     typeName:         String,
     role:             Optional[String],
+    messageExtractor: ShardRegion.MessageExtractor): ActorRef =
+    startProxy(typeName, role, dataCenter = Optional.empty(), messageExtractor)
+
+  /**
+   * Java/Scala API: Register a named entity type `ShardRegion` on this node that will run in proxy only mode,
+   * i.e. it will delegate messages to other `ShardRegion` actors on other nodes, but not host any
+   * entity actors itself. The [[ShardRegion]] actor for this type can later be retrieved with the
+   * [[#shardRegion]] method.
+   *
+   * Some settings can be configured as described in the `akka.cluster.sharding` section
+   * of the `reference.conf`.
+   *
+   * @param typeName the name of the entity type
+   * @param role specifies that this entity type is located on cluster nodes with a specific role.
+   *   If the role is not specified all nodes in the cluster are used.
+   * @param dataCenter The data center of the cluster nodes where the cluster sharding is running.
+   *   If None then the same data center as current node.
+   * @param messageExtractor functions to extract the entity id, shard id, and the message to send to the
+   *   entity from the incoming message
+   * @return the actor ref of the [[ShardRegion]] that is to be responsible for the shard
+   */
+  def startProxy(
+    typeName:         String,
+    role:             Optional[String],
+    dataCenter:       Optional[String],
     messageExtractor: ShardRegion.MessageExtractor): ActorRef = {
 
-    startProxy(typeName, Option(role.orElse(null)),
+    startProxy(typeName, Option(role.orElse(null)), Option(dataCenter.orElse(null)),
       extractEntityId = {
       case msg if messageExtractor.entityId(msg) ne null ⇒
         (messageExtractor.entityId(msg), messageExtractor.entityMessage(msg))
@@ -383,14 +447,28 @@ class ClusterSharding(system: ExtendedActorSystem) extends Extension {
 
   /**
    * Retrieve the actor reference of the [[ShardRegion]] actor responsible for the named entity type.
-   * The entity type must be registered with the [[#start]] method before it can be used here.
-   * Messages to the entity is always sent via the `ShardRegion`.
+   * The entity type must be registered with the [[#start]] or [[#startProxy]] method before it
+   * can be used here. Messages to the entity is always sent via the `ShardRegion`.
    */
   def shardRegion(typeName: String): ActorRef = regions.get(typeName) match {
     case null ⇒ throw new IllegalArgumentException(s"Shard type [$typeName] must be started first")
     case ref  ⇒ ref
   }
 
+  /**
+   * Retrieve the actor reference of the [[ShardRegion]] actor that will act as a proxy to the
+   * named entity type running in another data center. A proxy within the same data center can be accessed
+   * with [[#shardRegion]] instead of this method. The entity type must be registered with the
+   * [[#startProxy]] method before it can be used here. Messages to the entity is always sent
+   * via the `ShardRegion`.
+   */
+  def shardRegionProxy(typeName: String, dataCenter: DataCenter): ActorRef = {
+    regions.get(proxyName(typeName, Some(dataCenter))) match {
+      case null ⇒ throw new IllegalArgumentException(s"Shard type [$typeName] must be started first")
+      case ref  ⇒ ref
+    }
+  }
+
 }
 
 /**
@@ -402,7 +480,7 @@ private[akka] object ClusterShardingGuardian {
                          extractEntityId: ShardRegion.ExtractEntityId, extractShardId: ShardRegion.ExtractShardId,
                          allocationStrategy: ShardAllocationStrategy, handOffStopMessage: Any)
     extends NoSerializationVerificationNeeded
-  final case class StartProxy(typeName: String, settings: ClusterShardingSettings,
+  final case class StartProxy(typeName: String, dataCenter: Option[DataCenter], settings: ClusterShardingSettings,
                               extractEntityId: ShardRegion.ExtractEntityId, extractShardId: ShardRegion.ExtractShardId)
     extends NoSerializationVerificationNeeded
   final case class Started(shardRegion: ActorRef) extends NoSerializationVerificationNeeded
@@ -441,7 +519,9 @@ private[akka] class ClusterShardingGuardian extends Actor {
             case Some(r) ⇒ URLEncoder.encode(r, ByteString.UTF_8) + "Replicator"
             case None    ⇒ "replicator"
           }
-          val ref = context.actorOf(Replicator.props(replicatorSettings.withRole(settings.role)), name)
+          // Use members within the data center and with the given role (if any)
+          val replicatorRoles = Set(ClusterSettings.DcRolePrefix + cluster.settings.SelfDataCenter) ++ settings.role
+          val ref = context.actorOf(Replicator.props(replicatorSettings.withRoles(replicatorRoles)), name)
           replicatorByRole = replicatorByRole.updated(settings.role, ref)
           ref
       }
@@ -505,22 +585,29 @@ private[akka] class ClusterShardingGuardian extends Actor {
           sender() ! Status.Failure(e)
       }
 
-    case StartProxy(typeName, settings, extractEntityId, extractShardId) ⇒
+    case StartProxy(typeName, dataCenter, settings, extractEntityId, extractShardId) ⇒
       try {
+
         val encName = URLEncoder.encode(typeName, ByteString.UTF_8)
         val cName = coordinatorSingletonManagerName(encName)
         val cPath = coordinatorPath(encName)
-        val shardRegion = context.child(encName).getOrElse {
+        // it must be possible to start several proxies, one per data center
+        val actorName = dataCenter match {
+          case None    ⇒ encName
+          case Some(t) ⇒ URLEncoder.encode(typeName + "-" + t, ByteString.UTF_8)
+        }
+        val shardRegion = context.child(actorName).getOrElse {
           context.actorOf(
             ShardRegion.proxyProps(
               typeName = typeName,
+              dataCenter = dataCenter,
               settings = settings,
               coordinatorPath = cPath,
               extractEntityId = extractEntityId,
               extractShardId = extractShardId,
               replicator = context.system.deadLetters,
               majorityMinCap).withDispatcher(context.props.dispatcher),
-            name = encName)
+            name = actorName)
         }
         sender() ! Started(shardRegion)
       } catch {

akka-cluster-sharding/src/main/scala/akka/cluster/sharding/ShardRegion.scala

@@ -20,6 +20,8 @@ import scala.concurrent.Future
 import scala.reflect.ClassTag
 import scala.concurrent.Promise
 import akka.Done
+import akka.cluster.ClusterSettings
+import akka.cluster.ClusterSettings.DataCenter
 
 /**
  * @see [[ClusterSharding$ ClusterSharding extension]]
@@ -40,7 +42,7 @@ object ShardRegion {
     handOffStopMessage: Any,
     replicator:         ActorRef,
     majorityMinCap:     Int): Props =
-    Props(new ShardRegion(typeName, Some(entityProps), settings, coordinatorPath, extractEntityId,
+    Props(new ShardRegion(typeName, Some(entityProps), dataCenter = None, settings, coordinatorPath, extractEntityId,
       extractShardId, handOffStopMessage, replicator, majorityMinCap)).withDeploy(Deploy.local)
 
   /**
@@ -50,13 +52,14 @@ object ShardRegion {
    */
   private[akka] def proxyProps(
     typeName:        String,
+    dataCenter:      Option[DataCenter],
     settings:        ClusterShardingSettings,
     coordinatorPath: String,
     extractEntityId: ShardRegion.ExtractEntityId,
     extractShardId:  ShardRegion.ExtractShardId,
     replicator:      ActorRef,
     majorityMinCap:  Int): Props =
-    Props(new ShardRegion(typeName, None, settings, coordinatorPath, extractEntityId, extractShardId,
+    Props(new ShardRegion(typeName, None, dataCenter, settings, coordinatorPath, extractEntityId, extractShardId,
       PoisonPill, replicator, majorityMinCap)).withDeploy(Deploy.local)
 
   /**
@@ -365,6 +368,7 @@ object ShardRegion {
 private[akka] class ShardRegion(
   typeName:           String,
   entityProps:        Option[Props],
+  dataCenter:         Option[DataCenter],
   settings:           ClusterShardingSettings,
   coordinatorPath:    String,
   extractEntityId:    ShardRegion.ExtractEntityId,
@@ -419,11 +423,15 @@ private[akka] class ShardRegion(
     retryTask.cancel()
   }
 
-  def matchingRole(member: Member): Boolean = role match {
-    case None    ⇒ true
-    case Some(r) ⇒ member.hasRole(r)
+  // when using proxy the data center can be different from the own data center
+  private val targetDcRole = dataCenter match {
+    case Some(t) ⇒ ClusterSettings.DcRolePrefix + t
+    case None    ⇒ ClusterSettings.DcRolePrefix + cluster.settings.SelfDataCenter
   }
 
+  def matchingRole(member: Member): Boolean =
+    member.hasRole(targetDcRole) && role.forall(member.hasRole)
+
   def coordinatorSelection: Option[ActorSelection] =
     membersByAge.headOption.map(m ⇒ context.actorSelection(RootActorPath(m.address) + coordinatorPath))
 
@@ -471,13 +479,22 @@ private[akka] class ShardRegion(
   def receiveClusterEvent(evt: ClusterDomainEvent): Unit = evt match {
     case MemberUp(m) ⇒
       if (matchingRole(m))
-        changeMembers(membersByAge - m + m) // replace
+        changeMembers {
+          // replace, it's possible that the upNumber is changed
+          membersByAge = membersByAge.filterNot(_.uniqueAddress == m.uniqueAddress)
+          membersByAge += m
+          membersByAge
+        }
 
     case MemberRemoved(m, _) ⇒
       if (m.uniqueAddress == cluster.selfUniqueAddress)
         context.stop(self)
       else if (matchingRole(m))
-        changeMembers(membersByAge - m)
+        changeMembers {
+          // filter, it's possible that the upNumber is changed
+          membersByAge = membersByAge.filterNot(_.uniqueAddress == m.uniqueAddress)
+          membersByAge
+        }
 
     case _: MemberEvent ⇒ // these are expected, no need to warn about them
 

akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sharding/ClusterShardingSpec.scala

@@ -460,6 +460,7 @@ abstract class ClusterShardingSpec(config: ClusterShardingSpecConfig) extends Mu
         val proxy = system.actorOf(
           ShardRegion.proxyProps(
             typeName = "counter",
+            dataCenter = None,
             settings,
             coordinatorPath = "/user/counterCoordinator/singleton/coordinator",
             extractEntityId = extractEntityId,
@@ -698,6 +699,21 @@ abstract class ClusterShardingSpec(config: ClusterShardingSpecConfig) extends Mu
 
   }
 
+  "demonstrate API for DC proxy" in within(50.seconds) {
+    runOn(sixth) {
+      // #proxy-dc
+      val counterProxyDcB: ActorRef = ClusterSharding(system).startProxy(
+        typeName = "Counter",
+        role = None,
+        dataCenter = Some("B"),
+        extractEntityId = extractEntityId,
+        extractShardId = extractShardId)
+      // #proxy-dc
+    }
+    enterBarrier("after-dc-proxy")
+
+  }
+
   "Persistent Cluster Shards" must {
     "recover entities upon restart" in within(50.seconds) {
       runOn(third, fourth, fifth) {

akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sharding/MultiDcClusterShardingSpec.scala

@@ -0,0 +1,220 @@
+/**
+ * Copyright (C) 2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster.sharding
+
+import scala.concurrent.duration._
+
+import akka.actor.Actor
+import akka.actor.ActorRef
+import akka.actor.Address
+import akka.actor.Props
+import akka.cluster.Cluster
+import akka.cluster.ClusterEvent._
+import akka.cluster.MemberStatus
+import akka.cluster.sharding.ShardRegion.CurrentRegions
+import akka.cluster.sharding.ShardRegion.GetCurrentRegions
+import akka.remote.testconductor.RoleName
+import akka.remote.testkit.MultiNodeConfig
+import akka.remote.testkit.MultiNodeSpec
+import akka.remote.testkit.STMultiNodeSpec
+import akka.testkit._
+import com.typesafe.config.ConfigFactory
+
+object MultiDcClusterShardingSpec {
+  sealed trait EntityMsg {
+    def id: String
+  }
+  final case class Ping(id: String) extends EntityMsg
+  final case class GetCount(id: String) extends EntityMsg
+
+  class Entity extends Actor {
+    var count = 0
+    def receive = {
+      case Ping(_) ⇒
+        count += 1
+        sender() ! self
+      case GetCount(_) ⇒
+        sender() ! count
+    }
+  }
+
+  val extractEntityId: ShardRegion.ExtractEntityId = {
+    case m: EntityMsg ⇒ (m.id, m)
+  }
+
+  val extractShardId: ShardRegion.ExtractShardId = {
+    case m: EntityMsg ⇒ m.id.charAt(0).toString
+  }
+}
+
+object MultiDcClusterShardingSpecConfig extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+
+  commonConfig(ConfigFactory.parseString(s"""
+    # DEBUG because of failing test, issue #23582
+    akka.loglevel = DEBUG
+    akka.cluster.debug.verbose-heartbeat-logging = on
+    akka.actor.provider = "cluster"
+    akka.remote.log-remote-lifecycle-events = off
+    akka.cluster.auto-down-unreachable-after = 0s
+    """))
+
+  nodeConfig(first, second) {
+    ConfigFactory.parseString("akka.cluster.multi-data-center.self-data-center = DC1")
+  }
+
+  nodeConfig(third, fourth) {
+    ConfigFactory.parseString("akka.cluster.multi-data-center.self-data-center = DC2")
+  }
+}
+
+class MultiDcClusterShardingMultiJvmNode1 extends MultiDcClusterShardingSpec
+class MultiDcClusterShardingMultiJvmNode2 extends MultiDcClusterShardingSpec
+class MultiDcClusterShardingMultiJvmNode3 extends MultiDcClusterShardingSpec
+class MultiDcClusterShardingMultiJvmNode4 extends MultiDcClusterShardingSpec
+
+abstract class MultiDcClusterShardingSpec extends MultiNodeSpec(MultiDcClusterShardingSpecConfig)
+  with STMultiNodeSpec with ImplicitSender {
+  import MultiDcClusterShardingSpec._
+  import MultiDcClusterShardingSpecConfig._
+
+  override def initialParticipants = roles.size
+
+  val cluster = Cluster(system)
+
+  def join(from: RoleName, to: RoleName): Unit = {
+    runOn(from) {
+      cluster join node(to).address
+      startSharding()
+      within(15.seconds) {
+        awaitAssert(cluster.state.members.exists { m ⇒
+          m.uniqueAddress == cluster.selfUniqueAddress && m.status == MemberStatus.Up
+        } should be(true))
+      }
+    }
+    enterBarrier(from.name + "-joined")
+  }
+
+  def startSharding(): Unit = {
+    ClusterSharding(system).start(
+      typeName = "Entity",
+      entityProps = Props[Entity],
+      settings = ClusterShardingSettings(system),
+      extractEntityId = extractEntityId,
+      extractShardId = extractShardId)
+  }
+
+  lazy val region = ClusterSharding(system).shardRegion("Entity")
+
+  private def fillAddress(a: Address): Address =
+    if (a.hasLocalScope) Cluster(system).selfAddress else a
+
+  private def assertCurrentRegions(expected: Set[Address]): Unit = {
+    awaitAssert({
+      val p = TestProbe()
+      region.tell(GetCurrentRegions, p.ref)
+      p.expectMsg(CurrentRegions(expected))
+    }, 10.seconds)
+  }
+
+  s"Cluster sharding in multi data center cluster" must {
+    "join cluster" in within(20.seconds) {
+      join(first, first)
+      join(second, first)
+      join(third, first)
+      join(fourth, first)
+
+      awaitAssert({
+        Cluster(system).state.members.size should ===(4)
+        Cluster(system).state.members.map(_.status) should ===(Set(MemberStatus.Up))
+      }, 10.seconds)
+
+      runOn(first, second) {
+        assertCurrentRegions(Set(first, second).map(r ⇒ node(r).address))
+      }
+      runOn(third, fourth) {
+        assertCurrentRegions(Set(third, fourth).map(r ⇒ node(r).address))
+      }
+
+      enterBarrier("after-1")
+    }
+
+    "initialize shards" in {
+      runOn(first) {
+        val locations = (for (n ← 1 to 10) yield {
+          val id = n.toString
+          region ! Ping(id)
+          id → expectMsgType[ActorRef]
+        }).toMap
+        val firstAddress = node(first).address
+        val secondAddress = node(second).address
+        val hosts = locations.values.map(ref ⇒ fillAddress(ref.path.address)).toSet
+        hosts should ===(Set(firstAddress, secondAddress))
+      }
+      runOn(third) {
+        val locations = (for (n ← 1 to 10) yield {
+          val id = n.toString
+          region ! Ping(id)
+          val ref1 = expectMsgType[ActorRef]
+          region ! Ping(id)
+          val ref2 = expectMsgType[ActorRef]
+          ref1 should ===(ref2)
+          id → ref1
+        }).toMap
+        val thirdAddress = node(third).address
+        val fourthAddress = node(fourth).address
+        val hosts = locations.values.map(ref ⇒ fillAddress(ref.path.address)).toSet
+        hosts should ===(Set(thirdAddress, fourthAddress))
+      }
+      enterBarrier("after-2")
+    }
+
+    "not mix entities in different data centers" in {
+      runOn(second) {
+        region ! GetCount("5")
+        expectMsg(1)
+      }
+      runOn(fourth) {
+        region ! GetCount("5")
+        expectMsg(2)
+      }
+      enterBarrier("after-3")
+    }
+
+    "allow proxy within same data center" in {
+      runOn(second) {
+        val proxy = ClusterSharding(system).startProxy(
+          typeName = "Entity",
+          role = None,
+          dataCenter = None, // by default use own DC
+          extractEntityId = extractEntityId,
+          extractShardId = extractShardId)
+
+        proxy ! GetCount("5")
+        expectMsg(1)
+      }
+      enterBarrier("after-4")
+    }
+
+    "allow proxy across different data centers" in {
+      runOn(second) {
+        val proxy = ClusterSharding(system).startProxy(
+          typeName = "Entity",
+          role = None,
+          dataCenter = Some("DC2"), // proxy to other DC
+          extractEntityId = extractEntityId,
+          extractShardId = extractShardId)
+
+        proxy ! GetCount("5")
+        expectMsg(2)
+      }
+      enterBarrier("after-5")
+    }
+
+  }
+}
+

akka-cluster-tools/src/main/scala/akka/cluster/singleton/ClusterSingletonManager.scala

@@ -33,6 +33,7 @@ import akka.actor.CoordinatedShutdown
 import akka.annotation.DoNotInherit
 import akka.pattern.ask
 import akka.util.Timeout
+import akka.cluster.ClusterSettings
 
 object ClusterSingletonManagerSettings {
 
@@ -259,10 +260,10 @@ object ClusterSingletonManager {
       }
       override def postStop(): Unit = cluster.unsubscribe(self)
 
-      def matchingRole(member: Member): Boolean = role match {
-        case None    ⇒ true
-        case Some(r) ⇒ member.hasRole(r)
-      }
+      private val selfDc = ClusterSettings.DcRolePrefix + cluster.settings.SelfDataCenter
+
+      def matchingRole(member: Member): Boolean =
+        member.hasRole(selfDc) && role.forall(member.hasRole)
 
       def trackChange(block: () ⇒ Unit): Unit = {
         val before = membersByAge.headOption

akka-cluster-tools/src/main/scala/akka/cluster/singleton/ClusterSingletonProxy.scala

@@ -19,6 +19,8 @@ import com.typesafe.config.Config
 import akka.actor.NoSerializationVerificationNeeded
 import akka.event.Logging
 import akka.util.MessageBuffer
+import akka.cluster.ClusterSettings
+import akka.cluster.ClusterSettings.DataCenter
 
 object ClusterSingletonProxySettings {
 
@@ -63,6 +65,7 @@ object ClusterSingletonProxySettings {
 /**
  * @param singletonName The actor name of the singleton actor that is started by the [[ClusterSingletonManager]].
  * @param role The role of the cluster nodes where the singleton can be deployed. If None, then any node will do.
+ * @param dataCenter The data center of the cluster nodes where the singleton is running. If None then the same data center as current node.
  * @param singletonIdentificationInterval Interval at which the proxy will try to resolve the singleton instance.
  * @param bufferSize If the location of the singleton is unknown the proxy will buffer this number of messages
  *   and deliver them when the singleton is identified. When the buffer is full old messages will be dropped
@@ -72,9 +75,18 @@ object ClusterSingletonProxySettings {
 final class ClusterSingletonProxySettings(
   val singletonName:                   String,
   val role:                            Option[String],
+  val dataCenter:                      Option[DataCenter],
   val singletonIdentificationInterval: FiniteDuration,
   val bufferSize:                      Int) extends NoSerializationVerificationNeeded {
 
+  // for backwards compatibility
+  def this(
+    singletonName:                   String,
+    role:                            Option[String],
+    singletonIdentificationInterval: FiniteDuration,
+    bufferSize:                      Int) =
+    this(singletonName, role, None, singletonIdentificationInterval, bufferSize)
+
   require(bufferSize >= 0 && bufferSize <= 10000, "bufferSize must be >= 0 and <= 10000")
 
   def withSingletonName(name: String): ClusterSingletonProxySettings = copy(singletonName = name)
@@ -83,6 +95,8 @@ final class ClusterSingletonProxySettings(
 
   def withRole(role: Option[String]): ClusterSingletonProxySettings = copy(role = role)
 
+  def withDataCenter(dataCenter: DataCenter): ClusterSingletonProxySettings = copy(dataCenter = Some(dataCenter))
+
   def withSingletonIdentificationInterval(singletonIdentificationInterval: FiniteDuration): ClusterSingletonProxySettings =
     copy(singletonIdentificationInterval = singletonIdentificationInterval)
 
@@ -92,9 +106,10 @@ final class ClusterSingletonProxySettings(
   private def copy(
     singletonName:                   String             = singletonName,
     role:                            Option[String]     = role,
+    dataCenter:                      Option[DataCenter] = dataCenter,
     singletonIdentificationInterval: FiniteDuration     = singletonIdentificationInterval,
     bufferSize:                      Int                = bufferSize): ClusterSingletonProxySettings =
-    new ClusterSingletonProxySettings(singletonName, role, singletonIdentificationInterval, bufferSize)
+    new ClusterSingletonProxySettings(singletonName, role, dataCenter, singletonIdentificationInterval, bufferSize)
 }
 
 object ClusterSingletonProxy {
@@ -162,11 +177,14 @@ final class ClusterSingletonProxy(singletonManagerPath: String, settings: Cluste
     identifyTimer = None
   }
 
-  def matchingRole(member: Member): Boolean = role match {
-    case None    ⇒ true
-    case Some(r) ⇒ member.hasRole(r)
+  private val targetDcRole = settings.dataCenter match {
+    case Some(t) ⇒ ClusterSettings.DcRolePrefix + t
+    case None    ⇒ ClusterSettings.DcRolePrefix + cluster.settings.SelfDataCenter
   }
 
+  def matchingRole(member: Member): Boolean =
+    member.hasRole(targetDcRole) && role.forall(member.hasRole)
+
   def handleInitial(state: CurrentClusterState): Unit = {
     trackChange {
       () ⇒
@@ -204,7 +222,8 @@ final class ClusterSingletonProxy(singletonManagerPath: String, settings: Cluste
   def add(m: Member): Unit = {
     if (matchingRole(m))
       trackChange { () ⇒
-        membersByAge -= m // replace
+        // replace, it's possible that the upNumber is changed
+        membersByAge = membersByAge.filterNot(_.uniqueAddress == m.uniqueAddress)
         membersByAge += m
       }
   }
@@ -215,8 +234,9 @@ final class ClusterSingletonProxy(singletonManagerPath: String, settings: Cluste
    */
   def remove(m: Member): Unit = {
     if (matchingRole(m))
-      trackChange {
-        () ⇒ membersByAge -= m
+      trackChange { () ⇒
+        // filter, it's possible that the upNumber is changed
+        membersByAge = membersByAge.filterNot(_.uniqueAddress == m.uniqueAddress)
       }
   }
 

akka-cluster-tools/src/multi-jvm/scala/akka/cluster/singleton/ClusterSingletonManagerSpec.scala

@@ -228,12 +228,26 @@ class ClusterSingletonManagerSpec extends MultiNodeSpec(ClusterSingletonManagerS
 
   def createSingletonProxy(): ActorRef = {
     //#create-singleton-proxy
-    system.actorOf(
+    val proxy = system.actorOf(
       ClusterSingletonProxy.props(
         singletonManagerPath = "/user/consumer",
         settings = ClusterSingletonProxySettings(system).withRole("worker")),
       name = "consumerProxy")
     //#create-singleton-proxy
+    proxy
+  }
+
+  def createSingletonProxyDc(): ActorRef = {
+    //#create-singleton-proxy-dc
+    val proxyDcB = system.actorOf(
+      ClusterSingletonProxy.props(
+        singletonManagerPath = "/user/consumer",
+        settings = ClusterSingletonProxySettings(system)
+          .withRole("worker")
+          .withDataCenter("B")),
+      name = "consumerProxyDcB")
+    //#create-singleton-proxy-dc
+    proxyDcB
   }
 
   def verifyProxyMsg(oldest: RoleName, proxyNode: RoleName, msg: Int): Unit = {

akka-cluster-tools/src/multi-jvm/scala/akka/cluster/singleton/MultiDcSingletonManagerSpec.scala

@@ -0,0 +1,128 @@
+/**
+ * Copyright (C) 2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster.singleton
+
+import scala.concurrent.duration._
+
+import com.typesafe.config.ConfigFactory
+
+import akka.actor.{ Actor, ActorLogging, Address, PoisonPill, Props }
+import akka.cluster.Cluster
+
+import akka.testkit.ImplicitSender
+import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec, STMultiNodeSpec }
+import akka.cluster.ClusterSettings
+
+object MultiDcSingletonManagerSpec extends MultiNodeConfig {
+  val controller = role("controller")
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+
+  commonConfig(ConfigFactory.parseString("""
+    akka.loglevel = INFO
+    akka.actor.provider = "cluster"
+    akka.actor.serialize-creators = off
+    akka.remote.log-remote-lifecycle-events = off"""))
+
+  nodeConfig(controller) {
+    ConfigFactory.parseString("""
+      akka.cluster.multi-data-center.self-data-center = one
+      akka.cluster.roles = []""")
+  }
+
+  nodeConfig(first) {
+    ConfigFactory.parseString("""
+      akka.cluster.multi-data-center.self-data-center = one
+      akka.cluster.roles = [ worker ]""")
+  }
+  nodeConfig(second, third) {
+    ConfigFactory.parseString("""
+      akka.cluster.multi-data-center.self-data-center = two
+      akka.cluster.roles = [ worker ]""")
+  }
+}
+
+class MultiDcSingletonManagerMultiJvmNode1 extends MultiDcSingletonManagerSpec
+class MultiDcSingletonManagerMultiJvmNode2 extends MultiDcSingletonManagerSpec
+class MultiDcSingletonManagerMultiJvmNode3 extends MultiDcSingletonManagerSpec
+class MultiDcSingletonManagerMultiJvmNode4 extends MultiDcSingletonManagerSpec
+
+class MultiDcSingleton extends Actor with ActorLogging {
+  import MultiDcSingleton._
+
+  val cluster = Cluster(context.system)
+
+  override def receive: Receive = {
+    case Ping ⇒
+      sender() ! Pong(cluster.settings.SelfDataCenter, cluster.selfAddress, cluster.selfRoles)
+  }
+}
+object MultiDcSingleton {
+  case object Ping
+  case class Pong(fromDc: String, fromAddress: Address, roles: Set[String])
+}
+
+abstract class MultiDcSingletonManagerSpec extends MultiNodeSpec(MultiDcSingletonManagerSpec) with STMultiNodeSpec with ImplicitSender {
+  import MultiDcSingletonManagerSpec._
+
+  override def initialParticipants = roles.size
+
+  val cluster = Cluster(system)
+  cluster.join(node(controller).address)
+  enterBarrier("nodes-joined")
+
+  val worker = "worker"
+
+  "A SingletonManager in a multi data center cluster" must {
+    "start a singleton instance for each data center" in {
+
+      runOn(first, second, third) {
+        system.actorOf(
+          ClusterSingletonManager.props(
+            Props[MultiDcSingleton](),
+            PoisonPill,
+            ClusterSingletonManagerSettings(system).withRole(worker)),
+          "singletonManager")
+      }
+
+      val proxy = system.actorOf(ClusterSingletonProxy.props(
+        "/user/singletonManager",
+        ClusterSingletonProxySettings(system).withRole(worker)))
+
+      enterBarrier("managers-started")
+
+      proxy ! MultiDcSingleton.Ping
+      val pong = expectMsgType[MultiDcSingleton.Pong](10.seconds)
+
+      enterBarrier("pongs-received")
+
+      pong.fromDc should equal(Cluster(system).settings.SelfDataCenter)
+      pong.roles should contain(worker)
+      runOn(controller, first) {
+        pong.roles should contain(ClusterSettings.DcRolePrefix + "one")
+      }
+      runOn(second, third) {
+        pong.roles should contain(ClusterSettings.DcRolePrefix + "two")
+      }
+
+      enterBarrier("after-1")
+    }
+
+    "be able to use proxy across different data centers" in {
+      runOn(third) {
+        val proxy = system.actorOf(ClusterSingletonProxy.props(
+          "/user/singletonManager",
+          ClusterSingletonProxySettings(system).withRole(worker).withDataCenter("one")))
+        proxy ! MultiDcSingleton.Ping
+        val pong = expectMsgType[MultiDcSingleton.Pong](10.seconds)
+        pong.fromDc should ===("one")
+        pong.roles should contain(worker)
+        pong.roles should contain(ClusterSettings.DcRolePrefix + "one")
+      }
+      enterBarrier("after-1")
+    }
+
+  }
+}

akka-cluster-tools/src/test/java/akka/cluster/singleton/ClusterSingletonManagerTest.java

@@ -5,6 +5,10 @@
 package akka.cluster.singleton;
 
 import akka.actor.ActorSystem;
+
+import java.util.HashMap;
+import java.util.Map;
+
 import akka.actor.ActorRef;
 import akka.actor.Props;
 
@@ -32,8 +36,17 @@ public class ClusterSingletonManagerTest {
     ClusterSingletonProxySettings proxySettings =
         ClusterSingletonProxySettings.create(system).withRole("worker");
 
+    ActorRef proxy =
       system.actorOf(ClusterSingletonProxy.props("/user/consumer", proxySettings),
         "consumerProxy");
     //#create-singleton-proxy
+
+    //#create-singleton-proxy-dc
+    ActorRef proxyDcB =
+      system.actorOf(ClusterSingletonProxy.props("/user/consumer",
+        ClusterSingletonProxySettings.create(system)
+          .withRole("worker")
+          .withDataCenter("B")), "consumerProxyDcB");
+    //#create-singleton-proxy-dc
   }
 }

akka-cluster/src/main/mima-filters/2.5.4.backwards.excludes

@@ -5,3 +5,21 @@ ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.
 ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#ClusterRouterPoolSettingsOrBuilder.getUseRolesList")
 ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.routing.ClusterRouterSettingsBase.useRole")
 ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.routing.ClusterRouterSettingsBase.useRoles")
+
+# #23228 single leader per cluster data center
+ProblemFilters.exclude[Problem]("akka.cluster.Gossip*")
+ProblemFilters.exclude[Problem]("akka.cluster.ClusterCoreDaemon*")
+ProblemFilters.exclude[Problem]("akka.cluster.ClusterDomainEventPublisher*")
+ProblemFilters.exclude[Problem]("akka.cluster.InternalClusterAction*")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ClusterEvent.diffReachable")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ClusterEvent.diffLeader")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ClusterEvent.diffRolesLeader")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ClusterEvent.diffSeen")
+ProblemFilters.exclude[IncompatibleMethTypeProblem]("akka.cluster.ClusterEvent.diffReachability")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ClusterEvent.diffUnreachable")
+ProblemFilters.exclude[IncompatibleMethTypeProblem]("akka.cluster.ClusterEvent.diffMemberEvents")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#GossipOrBuilder.getTombstonesCount")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#GossipOrBuilder.getTombstones")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#GossipOrBuilder.getTombstonesList")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#GossipOrBuilder.getTombstonesOrBuilderList")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("akka.cluster.protobuf.msg.ClusterMessages#GossipOrBuilder.getTombstonesOrBuilder")

akka-cluster/src/main/protobuf/ClusterMessages.proto

@@ -101,6 +101,7 @@ message Gossip {
   repeated Member members = 4;
   required GossipOverview overview = 5;
   required VectorClock version = 6;
+  repeated Tombstone tombstones = 7;
 }
 
 /**
@@ -127,6 +128,11 @@ message SubjectReachability {
   required int64 version = 4;
 }
 
+message Tombstone {
+  required int32 addressIndex = 1;
+  required int64 timestamp = 2;
+}
+
 /**
  * Reachability status
  */

akka-cluster/src/main/resources/reference.conf

@@ -71,6 +71,9 @@ akka {
     # The roles are part of the membership information and can be used by
     # routers or other services to distribute work to certain member types,
     # e.g. front-end and back-end nodes.
+    # Roles are not allowed to start with "dc-" as that is reserved for the
+    # special role assigned from the data-center a node belongs to (see the
+    # multi-data-center section below)
     roles = []
     
     # Run the coordinated shutdown from phase 'cluster-shutdown' when the cluster
@@ -142,6 +145,11 @@ akka {
     # greater than this value.
     reduce-gossip-different-view-probability = 400
 
+    # When a node is removed the removal is marked with a tombstone
+    # which is kept at least this long, after which it is pruned, if there is a partition
+    # longer than this it could lead to removed nodes being re-added to the cluster
+    prune-gossip-tombstones-after = 24h
+
     # Settings for the Phi accrual failure detector (http://www.jaist.ac.jp/~defago/files/pdf/IS_RR_2004_010.pdf
     # [Hayashibara et al]) used by the cluster subsystem to detect unreachable
     # members.
@@ -194,6 +202,52 @@ akka {
 
     }
 
+    # Configures mult-dc specific heartbeating and other mechanisms,
+    # many of them have a direct counter-part in "one datacenter mode",
+    # in which case these settings would not be used at all - they only apply,
+    # if your cluster nodes are configured with at-least 2 different `akka.cluster.data-center` values.
+    multi-data-center {
+
+      # Defines which data center this node belongs to. It is typically used to make islands of the
+      # cluster that are colocated. This can be used to make the cluster aware that it is running
+      # across multiple availability zones or regions. It can also be used for other logical
+      # grouping of nodes.
+      self-data-center = "default"
+
+
+      # Try to limit the number of connections between data centers. Used for gossip and heartbeating.
+      # This will not limit connections created for the messaging of the application.
+      # If the cluster does not span multiple data centers, this value has no effect.
+      cross-data-center-connections = 5
+
+      # The n oldest nodes in a data center will choose to gossip to another data center with
+      # this probability. Must be a value between 0.0 and 1.0 where 0.0 means never, 1.0 means always.
+      cross-data-center-gossip-probability = 0.2
+
+      failure-detector {
+        # FQCN of the failure detector implementation.
+        # It must implement akka.remote.FailureDetector and have
+        # a public constructor with a com.typesafe.config.Config and
+        # akka.actor.EventStream parameter.
+        implementation-class = "akka.remote.DeadlineFailureDetector"
+  
+        # Number of potentially lost/delayed heartbeats that will be
+        # accepted before considering it to be an anomaly.
+        # This margin is important to be able to survive sudden, occasional,
+        # pauses in heartbeat arrivals, due to for example garbage collect or
+        # network drop.
+        acceptable-heartbeat-pause = 10 s
+        
+        # How often keep-alive heartbeat messages should be sent to each connection.
+        heartbeat-interval = 3 s
+  
+        # After the heartbeat request has been sent the first failure detection
+        # will start after this period, even though no heartbeat message has
+        # been received.
+        expected-response-after = 1 s
+      }
+    }
+
     # If the tick-duration of the default scheduler is longer than the
     # tick-duration configured here a dedicated scheduler will be used for
     # periodic tasks of the cluster, otherwise the default scheduler is used.
@@ -206,6 +260,9 @@ akka {
     debug {
       # log heartbeat events (very verbose, useful mostly when debugging heartbeating issues)
       verbose-heartbeat-logging = off
+
+      # log verbose details about gossip
+      verbose-gossip-logging = off
     }
 
   }

akka-cluster/src/main/scala/akka/cluster/AutoDown.scala

@@ -101,7 +101,7 @@ private[cluster] abstract class AutoDownBase(autoDownUnreachableAfter: FiniteDur
 
   import context.dispatcher
 
-  val skipMemberStatus = Gossip.convergenceSkipUnreachableWithMemberStatus
+  val skipMemberStatus = MembershipState.convergenceSkipUnreachableWithMemberStatus
 
   var scheduledUnreachable: Map[UniqueAddress, Cancellable] = Map.empty
   var pendingUnreachable: Set[UniqueAddress] = Set.empty

akka-cluster/src/main/scala/akka/cluster/Cluster.scala

@@ -10,6 +10,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 
 import akka.ConfigurationException
 import akka.actor._
+import akka.cluster.ClusterSettings.DataCenter
 import akka.dispatch.MonitorableThreadFactory
 import akka.event.{ Logging, LoggingAdapter }
 import akka.japi.Util
@@ -77,6 +78,9 @@ class Cluster(val system: ExtendedActorSystem) extends Extension {
    */
   def selfAddress: Address = selfUniqueAddress.address
 
+  /** Data center to which this node belongs to (defaults to "default" if not configured explicitly) */
+  def selfDataCenter: DataCenter = settings.SelfDataCenter
+
   /**
    * roles that this member has
    */
@@ -96,10 +100,19 @@ class Cluster(val system: ExtendedActorSystem) extends Extension {
   logInfo("Starting up...")
 
   val failureDetector: FailureDetectorRegistry[Address] = {
-    def createFailureDetector(): FailureDetector =
+    val createFailureDetector = () ⇒
       FailureDetectorLoader.load(settings.FailureDetectorImplementationClass, settings.FailureDetectorConfig, system)
 
-    new DefaultFailureDetectorRegistry(() ⇒ createFailureDetector())
+    new DefaultFailureDetectorRegistry(createFailureDetector)
+  }
+
+  val crossDcFailureDetector: FailureDetectorRegistry[Address] = {
+    val createFailureDetector = () ⇒
+      FailureDetectorLoader.load(
+        settings.MultiDataCenter.CrossDcFailureDetectorSettings.ImplementationClass,
+        settings.MultiDataCenter.CrossDcFailureDetectorSettings.config, system)
+
+    new DefaultFailureDetectorRegistry(createFailureDetector)
   }
 
   // needs to be lazy to allow downing provider impls to access Cluster (if not we get deadlock)
@@ -411,7 +424,7 @@ class Cluster(val system: ExtendedActorSystem) extends Extension {
 
   private def closeScheduler(): Unit = scheduler match {
     case x: Closeable ⇒ x.close()
-    case _            ⇒
+    case _            ⇒ // ignore, this is fine
   }
 
   /**
@@ -420,13 +433,32 @@ class Cluster(val system: ExtendedActorSystem) extends Extension {
   private[cluster] object InfoLogger {
 
     def logInfo(message: String): Unit =
-      if (LogInfo) log.info("Cluster Node [{}] - {}", selfAddress, message)
+      if (LogInfo)
+        if (settings.SelfDataCenter == ClusterSettings.DefaultDataCenter)
+          log.info("Cluster Node [{}] - {}", selfAddress, message)
+        else
+          log.info("Cluster Node [{}] dc [{}] - {}", selfAddress, settings.SelfDataCenter, message)
 
     def logInfo(template: String, arg1: Any): Unit =
-      if (LogInfo) log.info("Cluster Node [{}] - " + template, selfAddress, arg1)
+      if (LogInfo)
+        if (settings.SelfDataCenter == ClusterSettings.DefaultDataCenter)
+          log.info("Cluster Node [{}] - " + template, selfAddress, arg1)
+        else
+          log.info("Cluster Node [{}] dc [{}] - " + template, selfAddress, settings.SelfDataCenter, arg1)
 
     def logInfo(template: String, arg1: Any, arg2: Any): Unit =
-      if (LogInfo) log.info("Cluster Node [{}] - " + template, selfAddress, arg1, arg2)
+      if (LogInfo)
+        if (settings.SelfDataCenter == ClusterSettings.DefaultDataCenter)
+          log.info("Cluster Node [{}] - " + template, selfAddress, arg1, arg2)
+        else
+          log.info("Cluster Node [{}] dc [{}] - " + template, selfAddress, settings.SelfDataCenter, arg1, arg2)
+
+    def logInfo(template: String, arg1: Any, arg2: Any, arg3: Any): Unit =
+      if (LogInfo)
+        if (settings.SelfDataCenter == ClusterSettings.DefaultDataCenter)
+          log.info("Cluster Node [{}] - " + template, selfAddress, arg1, arg2, arg3)
+        else
+          log.info("Cluster Node [{}] dc [" + settings.SelfDataCenter + "] - " + template, selfAddress, arg1, arg2, arg3)
   }
 
 }

akka-cluster/src/main/scala/akka/cluster/ClusterDaemon.scala

@@ -3,25 +3,23 @@
  */
 package akka.cluster
 
-import language.existentials
-import scala.collection.immutable
-import scala.concurrent.duration._
-import java.util.concurrent.ThreadLocalRandom
-import scala.util.control.NonFatal
 import akka.actor._
+import akka.annotation.InternalApi
 import akka.actor.SupervisorStrategy.Stop
 import akka.cluster.MemberStatus._
 import akka.cluster.ClusterEvent._
-import akka.dispatch.{ UnboundedMessageQueueSemantics, RequiresMessageQueue }
-import scala.collection.breakOut
-import akka.remote.QuarantinedEvent
-import java.util.ArrayList
-import java.util.Collections
-import akka.pattern.ask
-import akka.util.Timeout
+import akka.dispatch.{ RequiresMessageQueue, UnboundedMessageQueueSemantics }
 import akka.Done
+import akka.pattern.ask
+import akka.remote.QuarantinedEvent
+import akka.util.Timeout
+
+import scala.collection.immutable
+import scala.concurrent.duration._
 import scala.concurrent.Future
 import scala.concurrent.Promise
+import scala.util.control.NonFatal
+import language.existentials
 
 /**
  * Base trait for all cluster messages. All ClusterMessage's are serializable.
@@ -152,7 +150,7 @@ private[cluster] object InternalClusterAction {
   final case class SendCurrentClusterState(receiver: ActorRef) extends SubscriptionMessage
 
   sealed trait PublishMessage
-  final case class PublishChanges(newGossip: Gossip) extends PublishMessage
+  final case class PublishChanges(state: MembershipState) extends PublishMessage
   final case class PublishEvent(event: ClusterDomainEvent) extends PublishMessage
 
   final case object ExitingCompleted
@@ -229,7 +227,6 @@ private[cluster] final class ClusterDaemon(settings: ClusterSettings) extends Ac
  */
 private[cluster] final class ClusterCoreSupervisor extends Actor with ActorLogging
   with RequiresMessageQueue[UnboundedMessageQueueSemantics] {
-  import InternalClusterAction._
 
   // Important - don't use Cluster(context.system) in constructor because that would
   // cause deadlock. The Cluster extension is currently being created and is waiting
@@ -266,26 +263,46 @@ private[cluster] final class ClusterCoreSupervisor extends Actor with ActorLoggi
 /**
  * INTERNAL API.
  */
-private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with ActorLogging
-  with RequiresMessageQueue[UnboundedMessageQueueSemantics] {
-  import InternalClusterAction._
-
-  val cluster = Cluster(context.system)
-  import cluster.{ selfAddress, selfRoles, scheduler, failureDetector }
-  import cluster.settings._
-  import cluster.InfoLogger._
-
-  protected def selfUniqueAddress = cluster.selfUniqueAddress
-
+@InternalApi
+private[cluster] object ClusterCoreDaemon {
   val NumberOfGossipsBeforeShutdownWhenLeaderExits = 5
   val MaxGossipsBeforeShuttingDownMyself = 5
 
-  def vclockName(node: UniqueAddress): String = s"${node.address}-${node.longUid}"
-  val vclockNode = VectorClock.Node(vclockName(selfUniqueAddress))
+}
+
+/**
+ * INTERNAL API.
+ */
+@InternalApi
+private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with ActorLogging
+  with RequiresMessageQueue[UnboundedMessageQueueSemantics] {
+  import InternalClusterAction._
+  import ClusterCoreDaemon._
+  import MembershipState._
+
+  val cluster = Cluster(context.system)
+  import cluster.{ selfAddress, selfRoles, scheduler, failureDetector, crossDcFailureDetector }
+  import cluster.settings._
+  import cluster.InfoLogger._
+
+  val selfDc = cluster.selfDataCenter
+
+  protected def selfUniqueAddress = cluster.selfUniqueAddress
+
+  val vclockNode = VectorClock.Node(Gossip.vclockName(selfUniqueAddress))
+  val gossipTargetSelector = new GossipTargetSelector(
+    ReduceGossipDifferentViewProbability,
+    cluster.settings.MultiDataCenter.CrossDcGossipProbability)
 
   // note that self is not initially member,
   // and the Gossip is not versioned for this 'Node' yet
-  var latestGossip: Gossip = Gossip.empty
+  var membershipState = MembershipState(
+    Gossip.empty,
+    cluster.selfUniqueAddress,
+    cluster.settings.SelfDataCenter,
+    cluster.settings.MultiDataCenter.CrossDcConnections)
+
+  def latestGossip: Gossip = membershipState.latestGossip
 
   val statsEnabled = PublishStatsInterval.isFinite
   var gossipStats = GossipStats()
@@ -411,8 +428,12 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
   def becomeInitialized(): Unit = {
     // start heartbeatSender here, and not in constructor to make sure that
     // heartbeating doesn't start before Welcome is received
-    context.actorOf(Props[ClusterHeartbeatSender].
-      withDispatcher(UseDispatcher), name = "heartbeatSender")
+    val internalHeartbeatSenderProps = Props(new ClusterHeartbeatSender()).withDispatcher(UseDispatcher)
+    context.actorOf(internalHeartbeatSenderProps, name = "heartbeatSender")
+
+    val externalHeartbeatProps = Props(new CrossDcHeartbeatSender()).withDispatcher(UseDispatcher)
+    context.actorOf(externalHeartbeatProps, name = "crossDcHeartbeatSender")
+
     // make sure that join process is stopped
     stopSeedNodeProcess()
     context.become(initialized)
@@ -462,7 +483,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
 
   def initJoin(): Unit = {
     val selfStatus = latestGossip.member(selfUniqueAddress).status
-    if (Gossip.removeUnreachableWithMemberStatus.contains(selfStatus)) {
+    if (removeUnreachableWithMemberStatus.contains(selfStatus)) {
       // prevents a Down and Exiting node from being used for joining
       logInfo("Sending InitJoinNack message from node [{}] to [{}]", selfAddress, sender())
       sender() ! InitJoinNack(selfAddress)
@@ -544,28 +565,28 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    * Received `Join` message and replies with `Welcome` message, containing
    * current gossip state, including the new joining member.
    */
-  def joining(node: UniqueAddress, roles: Set[String]): Unit = {
+  def joining(joiningNode: UniqueAddress, roles: Set[String]): Unit = {
     val selfStatus = latestGossip.member(selfUniqueAddress).status
-    if (node.address.protocol != selfAddress.protocol)
+    if (joiningNode.address.protocol != selfAddress.protocol)
       log.warning(
         "Member with wrong protocol tried to join, but was ignored, expected [{}] but was [{}]",
-        selfAddress.protocol, node.address.protocol)
-    else if (node.address.system != selfAddress.system)
+        selfAddress.protocol, joiningNode.address.protocol)
+    else if (joiningNode.address.system != selfAddress.system)
       log.warning(
         "Member with wrong ActorSystem name tried to join, but was ignored, expected [{}] but was [{}]",
-        selfAddress.system, node.address.system)
-    else if (Gossip.removeUnreachableWithMemberStatus.contains(selfStatus))
-      logInfo("Trying to join [{}] to [{}] member, ignoring. Use a member that is Up instead.", node, selfStatus)
+        selfAddress.system, joiningNode.address.system)
+    else if (removeUnreachableWithMemberStatus.contains(selfStatus))
+      logInfo("Trying to join [{}] to [{}] member, ignoring. Use a member that is Up instead.", joiningNode, selfStatus)
     else {
       val localMembers = latestGossip.members
 
       // check by address without uid to make sure that node with same host:port is not allowed
       // to join until previous node with that host:port has been removed from the cluster
-      localMembers.find(_.address == node.address) match {
-        case Some(m) if m.uniqueAddress == node ⇒
+      localMembers.find(_.address == joiningNode.address) match {
+        case Some(m) if m.uniqueAddress == joiningNode ⇒
           // node retried join attempt, probably due to lost Welcome message
           logInfo("Existing member [{}] is joining again.", m)
-          if (node != selfUniqueAddress)
+          if (joiningNode != selfUniqueAddress)
             sender() ! Welcome(selfUniqueAddress, latestGossip)
         case Some(m) ⇒
           // node restarted, same host:port as existing member, but with different uid
@@ -584,23 +605,24 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
           }
         case None ⇒
           // remove the node from the failure detector
-          failureDetector.remove(node.address)
+          failureDetector.remove(joiningNode.address)
+          crossDcFailureDetector.remove(joiningNode.address)
 
           // add joining node as Joining
           // add self in case someone else joins before self has joined (Set discards duplicates)
-          val newMembers = localMembers + Member(node, roles) + Member(selfUniqueAddress, cluster.selfRoles)
+          val newMembers = localMembers + Member(joiningNode, roles) + Member(selfUniqueAddress, cluster.selfRoles)
           val newGossip = latestGossip copy (members = newMembers)
 
           updateLatestGossip(newGossip)
 
-          logInfo("Node [{}] is JOINING, roles [{}]", node.address, roles.mkString(", "))
-          if (node == selfUniqueAddress) {
+          logInfo("Node [{}] is JOINING, roles [{}]", joiningNode.address, roles.mkString(", "))
+          if (joiningNode == selfUniqueAddress) {
             if (localMembers.isEmpty)
               leaderActions() // important for deterministic oldest when bootstrapping
           } else
             sender() ! Welcome(selfUniqueAddress, latestGossip)
 
-          publish(latestGossip)
+          publishMembershipState()
       }
     }
   }
@@ -613,10 +635,10 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     if (joinWith != from.address)
       logInfo("Ignoring welcome from [{}] when trying to join with [{}]", from.address, joinWith)
     else {
+      membershipState = membershipState.copy(latestGossip = gossip).seen()
       logInfo("Welcome from [{}]", from.address)
-      latestGossip = gossip seen selfUniqueAddress
       assertLatestGossip()
-      publish(latestGossip)
+      publishMembershipState()
       if (from != selfUniqueAddress)
         gossipTo(from, sender())
       becomeInitialized()
@@ -637,7 +659,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
       updateLatestGossip(newGossip)
 
       logInfo("Marked address [{}] as [{}]", address, Leaving)
-      publish(latestGossip)
+      publishMembershipState()
       // immediate gossip to speed up the leaving process
       gossip()
     }
@@ -648,9 +670,9 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     // ExitingCompleted sent via CoordinatedShutdown to continue the leaving process.
     exitingTasksInProgress = false
     // mark as seen
-    latestGossip = latestGossip seen selfUniqueAddress
+    membershipState = membershipState.seen()
     assertLatestGossip()
-    publish(latestGossip)
+    publishMembershipState()
 
     // Let others know (best effort) before shutdown. Otherwise they will not see
     // convergence of the Exiting state until they have detected this node as
@@ -663,11 +685,12 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     gossipRandomN(NumberOfGossipsBeforeShutdownWhenLeaderExits)
 
     // send ExitingConfirmed to two potential leaders
-    val membersWithoutSelf = latestGossip.members.filterNot(_.uniqueAddress == selfUniqueAddress)
-    latestGossip.leaderOf(membersWithoutSelf, selfUniqueAddress) match {
+    val membersExceptSelf = latestGossip.members.filter(_.uniqueAddress != selfUniqueAddress)
+
+    membershipState.leaderOf(membersExceptSelf) match {
       case Some(node1) ⇒
         clusterCore(node1.address) ! ExitingConfirmed(selfUniqueAddress)
-        latestGossip.leaderOf(membersWithoutSelf.filterNot(_.uniqueAddress == node1), selfUniqueAddress) match {
+        membershipState.leaderOf(membersExceptSelf.filterNot(_.uniqueAddress == node1)) match {
           case Some(node2) ⇒
             clusterCore(node2.address) ! ExitingConfirmed(selfUniqueAddress)
           case None ⇒ // no more potential leader
@@ -704,29 +727,19 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
   def downing(address: Address): Unit = {
     val localGossip = latestGossip
     val localMembers = localGossip.members
-    val localOverview = localGossip.overview
-    val localSeen = localOverview.seen
-    val localReachability = localOverview.reachability
+    val localReachability = membershipState.dcReachability
 
     // check if the node to DOWN is in the `members` set
     localMembers.find(_.address == address) match {
-      case Some(m) if (m.status != Down) ⇒
+      case Some(m) if m.status != Down ⇒
         if (localReachability.isReachable(m.uniqueAddress))
           logInfo("Marking node [{}] as [{}]", m.address, Down)
         else
           logInfo("Marking unreachable node [{}] as [{}]", m.address, Down)
 
-        // replace member (changed status)
-        val newMembers = localMembers - m + m.copy(status = Down)
-        // remove nodes marked as DOWN from the `seen` table
-        val newSeen = localSeen - m.uniqueAddress
-
-        // update gossip overview
-        val newOverview = localOverview copy (seen = newSeen)
-        val newGossip = localGossip copy (members = newMembers, overview = newOverview) // update gossip
+        val newGossip = localGossip.markAsDown(m)
         updateLatestGossip(newGossip)
-
-        publish(latestGossip)
+        publishMembershipState()
       case Some(_) ⇒ // already down
       case None ⇒
         logInfo("Ignoring down of unknown node [{}]", address)
@@ -744,14 +757,14 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
       log.warning(
         "Cluster Node [{}] - Marking node as TERMINATED [{}], due to quarantine. Node roles [{}]",
         selfAddress, node.address, selfRoles.mkString(","))
-      publish(latestGossip)
+      publishMembershipState()
       downing(node.address)
     }
   }
 
   def receiveGossipStatus(status: GossipStatus): Unit = {
     val from = status.from
-    if (!latestGossip.overview.reachability.isReachable(selfUniqueAddress, from))
+    if (!latestGossip.isReachable(selfUniqueAddress, from))
       logInfo("Ignoring received gossip status from unreachable [{}] ", from)
     else if (latestGossip.members.forall(_.uniqueAddress != from))
       log.debug("Cluster Node [{}] - Ignoring received gossip status from unknown [{}]", selfAddress, from)
@@ -778,6 +791,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    * Receive new gossip.
    */
   def receiveGossip(envelope: GossipEnvelope): ReceiveGossipType = {
+
     val from = envelope.from
     val remoteGossip = envelope.gossip
     val localGossip = latestGossip
@@ -788,7 +802,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     } else if (envelope.to != selfUniqueAddress) {
       logInfo("Ignoring received gossip intended for someone else, from [{}] to [{}]", from.address, envelope.to)
       Ignored
-    } else if (!localGossip.overview.reachability.isReachable(selfUniqueAddress, from)) {
+    } else if (!localGossip.isReachable(selfUniqueAddress, from)) {
       logInfo("Ignoring received gossip from unreachable [{}] ", from)
       Ignored
     } else if (localGossip.members.forall(_.uniqueAddress != from)) {
@@ -819,16 +833,16 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
           // Perform the same pruning (clear of VectorClock) as the leader did when removing a member.
           // Removal of member itself is handled in merge (pickHighestPriority)
           val prunedLocalGossip = localGossip.members.foldLeft(localGossip) { (g, m) ⇒
-            if (Gossip.removeUnreachableWithMemberStatus(m.status) && !remoteGossip.members.contains(m)) {
+            if (removeUnreachableWithMemberStatus(m.status) && !remoteGossip.members.contains(m)) {
               log.debug("Cluster Node [{}] - Pruned conflicting local gossip: {}", selfAddress, m)
-              g.prune(VectorClock.Node(vclockName(m.uniqueAddress)))
+              g.prune(VectorClock.Node(Gossip.vclockName(m.uniqueAddress)))
             } else
               g
           }
           val prunedRemoteGossip = remoteGossip.members.foldLeft(remoteGossip) { (g, m) ⇒
-            if (Gossip.removeUnreachableWithMemberStatus(m.status) && !localGossip.members.contains(m)) {
+            if (removeUnreachableWithMemberStatus(m.status) && !localGossip.members.contains(m)) {
               log.debug("Cluster Node [{}] - Pruned conflicting remote gossip: {}", selfAddress, m)
-              g.prune(VectorClock.Node(vclockName(m.uniqueAddress)))
+              g.prune(VectorClock.Node(Gossip.vclockName(m.uniqueAddress)))
             } else
               g
           }
@@ -839,20 +853,23 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
       // Don't mark gossip state as seen while exiting is in progress, e.g.
       // shutting down singleton actors. This delays removal of the member until
       // the exiting tasks have been completed.
-      if (exitingTasksInProgress)
-        latestGossip = winningGossip
-      else
-        latestGossip = winningGossip seen selfUniqueAddress
+      membershipState = membershipState.copy(latestGossip =
+        if (exitingTasksInProgress) winningGossip
+        else winningGossip seen selfUniqueAddress)
       assertLatestGossip()
 
       // for all new joining nodes we remove them from the failure detector
       latestGossip.members foreach {
-        node ⇒ if (node.status == Joining && !localGossip.members(node)) failureDetector.remove(node.address)
+        node ⇒
+          if (node.status == Joining && !localGossip.members(node)) {
+            failureDetector.remove(node.address)
+            crossDcFailureDetector.remove(node.address)
+          }
       }
 
       log.debug("Cluster Node [{}] - Receiving gossip from [{}]", selfAddress, from)
 
-      if (comparison == VectorClock.Concurrent) {
+      if (comparison == VectorClock.Concurrent && cluster.settings.Debug.VerboseGossipLogging) {
         log.debug(
           """Couldn't establish a causal relationship between "remote" gossip and "local" gossip - Remote[{}] - Local[{}] - merged them into [{}]""",
           remoteGossip, localGossip, winningGossip)
@@ -868,7 +885,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
         }
       }
 
-      publish(latestGossip)
+      publishMembershipState()
 
       val selfStatus = latestGossip.member(selfUniqueAddress).status
       if (selfStatus == Exiting && !exitingTasksInProgress) {
@@ -908,86 +925,23 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    */
   def gossipRandomN(n: Int): Unit = {
     if (!isSingletonCluster && n > 0) {
-      val localGossip = latestGossip
-      // using ArrayList to be able to shuffle
-      val possibleTargets = new ArrayList[UniqueAddress](localGossip.members.size)
-      localGossip.members.foreach { m ⇒
-        if (validNodeForGossip(m.uniqueAddress))
-          possibleTargets.add(m.uniqueAddress)
-      }
-      val randomTargets =
-        if (possibleTargets.size <= n)
-          possibleTargets
-        else {
-          Collections.shuffle(possibleTargets, ThreadLocalRandom.current())
-          possibleTargets.subList(0, n)
-        }
-
-      val iter = randomTargets.iterator
-      while (iter.hasNext)
-        gossipTo(iter.next())
+      gossipTargetSelector.randomNodesForFullGossip(membershipState, n).foreach(gossipTo)
     }
   }
 
   /**
    * Initiates a new round of gossip.
    */
-  def gossip(): Unit = {
-
+  def gossip(): Unit =
     if (!isSingletonCluster) {
-      val localGossip = latestGossip
-
-      val preferredGossipTargets: Vector[UniqueAddress] =
-        if (ThreadLocalRandom.current.nextDouble() < adjustedGossipDifferentViewProbability) {
-          // If it's time to try to gossip to some nodes with a different view
-          // gossip to a random alive member with preference to a member with older gossip version
-          localGossip.members.collect {
-            case m if !localGossip.seenByNode(m.uniqueAddress) && validNodeForGossip(m.uniqueAddress) ⇒
-              m.uniqueAddress
-          }(breakOut)
-        } else Vector.empty
-
-      if (preferredGossipTargets.nonEmpty) {
-        val peer = selectRandomNode(preferredGossipTargets)
-        // send full gossip because it has different view
-        peer foreach gossipTo
-      } else {
-        // Fall back to localGossip; important to not accidentally use `map` of the SortedSet, since the original order is not preserved)
-        val peer = selectRandomNode(localGossip.members.toIndexedSeq.collect {
-          case m if validNodeForGossip(m.uniqueAddress) ⇒ m.uniqueAddress
-        })
-        peer foreach { node ⇒
-          if (localGossip.seenByNode(node)) gossipStatusTo(node)
-          else gossipTo(node)
-        }
-      }
-    }
-  }
-
-  /**
-   * For large clusters we should avoid shooting down individual
-   * nodes. Therefore the probability is reduced for large clusters.
-   */
-  def adjustedGossipDifferentViewProbability: Double = {
-    val size = latestGossip.members.size
-    val low = ReduceGossipDifferentViewProbability
-    val high = low * 3
-    // start reduction when cluster is larger than configured ReduceGossipDifferentViewProbability
-    if (size <= low)
-      GossipDifferentViewProbability
-    else {
-      // don't go lower than 1/10 of the configured GossipDifferentViewProbability
-      val minP = GossipDifferentViewProbability / 10
-      if (size >= high)
-        minP
-      else {
-        // linear reduction of the probability with increasing number of nodes
-        // from ReduceGossipDifferentViewProbability at ReduceGossipDifferentViewProbability nodes
-        // to ReduceGossipDifferentViewProbability / 10 at ReduceGossipDifferentViewProbability * 3 nodes
-        // i.e. default from 0.8 at 400 nodes, to 0.08 at 1600 nodes
-        val k = (minP - GossipDifferentViewProbability) / (high - low)
-        GossipDifferentViewProbability + (size - low) * k
-      }
+      gossipTargetSelector.gossipTarget(membershipState) match {
+        case Some(peer) ⇒
+          if (!membershipState.isInSameDc(peer) || latestGossip.seenByNode(peer))
+            // avoid transferring the full state if possible
+            gossipStatusTo(peer)
+          else
+            gossipTo(peer)
+        case None ⇒ // nothing to see here
       }
     }
 
@@ -995,11 +949,11 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    * Runs periodic leader actions, such as member status transitions, assigning partitions etc.
    */
   def leaderActions(): Unit = {
-    if (latestGossip.isLeader(selfUniqueAddress, selfUniqueAddress)) {
-      // only run the leader actions if we are the LEADER
+    if (membershipState.isLeader(selfUniqueAddress)) {
+      // only run the leader actions if we are the LEADER of the data center
       val firstNotice = 20
       val periodicNotice = 60
-      if (latestGossip.convergence(selfUniqueAddress, exitingConfirmed)) {
+      if (membershipState.convergence(exitingConfirmed)) {
         if (leaderActionCounter >= firstNotice)
           logInfo("Leader can perform its duties again")
         leaderActionCounter = 0
@@ -1012,9 +966,11 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
         if (leaderActionCounter == firstNotice || leaderActionCounter % periodicNotice == 0)
           logInfo(
             "Leader can currently not perform its duties, reachability status: [{}], member status: [{}]",
-            latestGossip.reachabilityExcludingDownedObservers,
-            latestGossip.members.map(m ⇒
-              s"${m.address} ${m.status} seen=${latestGossip.seenByNode(m.uniqueAddress)}").mkString(", "))
+            membershipState.dcReachabilityExcludingDownedObservers,
+            latestGossip.members.collect {
+              case m if m.dataCenter == selfDc ⇒
+                s"${m.address} ${m.status} seen=${latestGossip.seenByNode(m.uniqueAddress)}"
+            }.mkString(", "))
       }
     }
     cleanupExitingConfirmed()
@@ -1025,8 +981,8 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     if (latestGossip.member(selfUniqueAddress).status == Down) {
       // When all reachable have seen the state this member will shutdown itself when it has
       // status Down. The down commands should spread before we shutdown.
-      val unreachable = latestGossip.overview.reachability.allUnreachableOrTerminated
-      val downed = latestGossip.members.collect { case m if m.status == Down ⇒ m.uniqueAddress }
+      val unreachable = membershipState.dcReachability.allUnreachableOrTerminated
+      val downed = membershipState.dcMembers.collect { case m if m.status == Down ⇒ m.uniqueAddress }
       if (downed.forall(node ⇒ unreachable(node) || latestGossip.seenByNode(node))) {
         // the reason for not shutting down immediately is to give the gossip a chance to spread
         // the downing information to other downed nodes, so that they can shutdown themselves
@@ -1059,66 +1015,53 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    * 9. Update the state with the new gossip
    */
   def leaderActionsOnConvergence(): Unit = {
-    val localGossip = latestGossip
-    val localMembers = localGossip.members
-    val localOverview = localGossip.overview
-    val localSeen = localOverview.seen
 
+    val removedUnreachable = for {
+      node ← membershipState.dcReachability.allUnreachableOrTerminated
+      m = latestGossip.member(node)
+      if m.dataCenter == selfDc && removeUnreachableWithMemberStatus(m.status)
+    } yield m
+
+    val removedExitingConfirmed = exitingConfirmed.filter { n ⇒
+      val member = latestGossip.member(n)
+      member.dataCenter == selfDc && member.status == Exiting
+    }
+
+    val changedMembers = {
       val enoughMembers: Boolean = isMinNrOfMembersFulfilled
       def isJoiningToUp(m: Member): Boolean = (m.status == Joining || m.status == WeaklyUp) && enoughMembers
 
-    val removedUnreachable = for {
-      node ← localOverview.reachability.allUnreachableOrTerminated
-      m = localGossip.member(node)
-      if Gossip.removeUnreachableWithMemberStatus(m.status)
-    } yield m
-
-    val removedExitingConfirmed = exitingConfirmed.filter(n ⇒ localGossip.member(n).status == Exiting)
-
-    val changedMembers = localMembers collect {
+      latestGossip.members collect {
         var upNumber = 0
 
         {
-        case m if isJoiningToUp(m) ⇒
+          case m if m.dataCenter == selfDc && isJoiningToUp(m) ⇒
             // Move JOINING => UP (once all nodes have seen that this node is JOINING, i.e. we have a convergence)
             // and minimum number of nodes have joined the cluster
             if (upNumber == 0) {
               // It is alright to use same upNumber as already used by a removed member, since the upNumber
               // is only used for comparing age of current cluster members (Member.isOlderThan)
-            val youngest = localGossip.youngestMember
+              val youngest = latestGossip.youngestMember
               upNumber = 1 + (if (youngest.upNumber == Int.MaxValue) 0 else youngest.upNumber)
             } else {
               upNumber += 1
             }
             m.copyUp(upNumber)
 
-        case m if m.status == Leaving ⇒
+          case m if m.dataCenter == selfDc && m.status == Leaving ⇒
             // Move LEAVING => EXITING (once we have a convergence on LEAVING)
             m copy (status = Exiting)
         }
       }
+    }
 
+    val updatedGossip: Gossip =
       if (removedUnreachable.nonEmpty || removedExitingConfirmed.nonEmpty || changedMembers.nonEmpty) {
-      // handle changes
 
         // replace changed members
-      val newMembers = changedMembers.union(localMembers).diff(removedUnreachable)
-        .filterNot(m ⇒ removedExitingConfirmed(m.uniqueAddress))
-
-      // removing REMOVED nodes from the `seen` table
         val removed = removedUnreachable.map(_.uniqueAddress).union(removedExitingConfirmed)
-      val newSeen = localSeen diff removed
-      // removing REMOVED nodes from the `reachability` table
-      val newReachability = localOverview.reachability.remove(removed)
-      val newOverview = localOverview copy (seen = newSeen, reachability = newReachability)
-      // Clear the VectorClock when member is removed. The change made by the leader is stamped
-      // and will propagate as is if there are no other changes on other nodes.
-      // If other concurrent changes on other nodes (e.g. join) the pruning is also
-      // taken care of when receiving gossips.
-      val newVersion = removed.foldLeft(localGossip.version) { (v, node) ⇒
-        v.prune(VectorClock.Node(vclockName(node)))
-      }
-      val newGossip = localGossip copy (members = newMembers, overview = newOverview, version = newVersion)
+        val newGossip =
+          latestGossip.update(changedMembers).removeAll(removed, System.currentTimeMillis())
 
         if (!exitingTasksInProgress && newGossip.member(selfUniqueAddress).status == Exiting) {
           // Leader is moving itself from Leaving to Exiting.
@@ -1130,15 +1073,11 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
           coordShutdown.run()
         }
 
-      updateLatestGossip(newGossip)
         exitingConfirmed = exitingConfirmed.filterNot(removedExitingConfirmed)
 
-      // log status changes
         changedMembers foreach { m ⇒
           logInfo("Leader is moving node [{}] to [{}]", m.address, m.status)
         }
-
-      // log the removal of the unreachable nodes
         removedUnreachable foreach { m ⇒
           val status = if (m.status == Exiting) "exiting" else "unreachable"
           logInfo("Leader is removing {} node [{}]", status, m.address)
@@ -1147,7 +1086,14 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
           logInfo("Leader is removing confirmed Exiting node [{}]", n.address)
         }
 
-      publish(latestGossip)
+        newGossip
+      } else
+        latestGossip
+
+    val pruned = updatedGossip.pruneTombstones(System.currentTimeMillis() - PruneGossipTombstonesAfter.toMillis)
+    if (pruned ne latestGossip) {
+      updateLatestGossip(pruned)
+      publishMembershipState()
     }
   }
 
@@ -1157,7 +1103,10 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
 
     val enoughMembers: Boolean = isMinNrOfMembersFulfilled
     def isJoiningToWeaklyUp(m: Member): Boolean =
-      m.status == Joining && enoughMembers && latestGossip.reachabilityExcludingDownedObservers.isReachable(m.uniqueAddress)
+      m.dataCenter == selfDc &&
+        m.status == Joining &&
+        enoughMembers &&
+        membershipState.dcReachabilityExcludingDownedObservers.isReachable(m.uniqueAddress)
     val changedMembers = localMembers.collect {
       case m if isJoiningToWeaklyUp(m) ⇒ m.copy(status = WeaklyUp)
     }
@@ -1173,7 +1122,7 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
         logInfo("Leader is moving node [{}] to [{}]", m.address, m.status)
       }
 
-      publish(latestGossip)
+      publishMembershipState()
     }
 
   }
@@ -1189,24 +1138,29 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
       val localOverview = localGossip.overview
       val localMembers = localGossip.members
 
+      def isAvailable(member: Member): Boolean = {
+        if (member.dataCenter == SelfDataCenter) failureDetector.isAvailable(member.address)
+        else crossDcFailureDetector.isAvailable(member.address)
+      }
+
       val newlyDetectedUnreachableMembers = localMembers filterNot { member ⇒
         member.uniqueAddress == selfUniqueAddress ||
           localOverview.reachability.status(selfUniqueAddress, member.uniqueAddress) == Reachability.Unreachable ||
           localOverview.reachability.status(selfUniqueAddress, member.uniqueAddress) == Reachability.Terminated ||
-          failureDetector.isAvailable(member.address)
+          isAvailable(member)
       }
 
       val newlyDetectedReachableMembers = localOverview.reachability.allUnreachableFrom(selfUniqueAddress) collect {
-        case node if node != selfUniqueAddress && failureDetector.isAvailable(node.address) ⇒
+        case node if node != selfUniqueAddress && isAvailable(localGossip.member(node)) ⇒
           localGossip.member(node)
       }
 
       if (newlyDetectedUnreachableMembers.nonEmpty || newlyDetectedReachableMembers.nonEmpty) {
 
-        val newReachability1 = (localOverview.reachability /: newlyDetectedUnreachableMembers) {
+        val newReachability1 = newlyDetectedUnreachableMembers.foldLeft(localOverview.reachability) {
           (reachability, m) ⇒ reachability.unreachable(selfUniqueAddress, m.uniqueAddress)
         }
-        val newReachability2 = (newReachability1 /: newlyDetectedReachableMembers) {
+        val newReachability2 = newlyDetectedReachableMembers.foldLeft(newReachability1) {
           (reachability, m) ⇒ reachability.reachable(selfUniqueAddress, m.uniqueAddress)
         }
 
@@ -1226,16 +1180,12 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
           if (newlyDetectedReachableMembers.nonEmpty)
             logInfo("Marking node(s) as REACHABLE [{}]. Node roles [{}]", newlyDetectedReachableMembers.mkString(", "), selfRoles.mkString(","))
 
-          publish(latestGossip)
+          publishMembershipState()
         }
       }
     }
   }
 
-  def selectRandomNode(nodes: IndexedSeq[UniqueAddress]): Option[UniqueAddress] =
-    if (nodes.isEmpty) None
-    else Some(nodes(ThreadLocalRandom.current nextInt nodes.size))
-
   def isSingletonCluster: Boolean = latestGossip.isSingletonCluster
 
   // needed for tests
@@ -1249,40 +1199,38 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
    * Gossips latest gossip to a node.
    */
   def gossipTo(node: UniqueAddress): Unit =
-    if (validNodeForGossip(node))
+    if (membershipState.validNodeForGossip(node))
       clusterCore(node.address) ! GossipEnvelope(selfUniqueAddress, node, latestGossip)
 
   def gossipTo(node: UniqueAddress, destination: ActorRef): Unit =
-    if (validNodeForGossip(node))
+    if (membershipState.validNodeForGossip(node))
       destination ! GossipEnvelope(selfUniqueAddress, node, latestGossip)
 
   def gossipStatusTo(node: UniqueAddress, destination: ActorRef): Unit =
-    if (validNodeForGossip(node))
+    if (membershipState.validNodeForGossip(node))
       destination ! GossipStatus(selfUniqueAddress, latestGossip.version)
 
   def gossipStatusTo(node: UniqueAddress): Unit =
-    if (validNodeForGossip(node))
+    if (membershipState.validNodeForGossip(node))
       clusterCore(node.address) ! GossipStatus(selfUniqueAddress, latestGossip.version)
 
-  def validNodeForGossip(node: UniqueAddress): Boolean =
-    (node != selfUniqueAddress && latestGossip.hasMember(node) &&
-      latestGossip.reachabilityExcludingDownedObservers.isReachable(node))
-
-  def updateLatestGossip(newGossip: Gossip): Unit = {
+  def updateLatestGossip(gossip: Gossip): Unit = {
     // Updating the vclock version for the changes
-    val versionedGossip = newGossip :+ vclockNode
+    val versionedGossip = gossip :+ vclockNode
 
     // Don't mark gossip state as seen while exiting is in progress, e.g.
     // shutting down singleton actors. This delays removal of the member until
     // the exiting tasks have been completed.
+    val newGossip =
       if (exitingTasksInProgress)
-      latestGossip = versionedGossip.clearSeen()
+        versionedGossip.clearSeen()
       else {
         // Nobody else has seen this gossip but us
         val seenVersionedGossip = versionedGossip onlySeen (selfUniqueAddress)
         // Update the state with the new gossip
-      latestGossip = seenVersionedGossip
+        seenVersionedGossip
       }
+    membershipState = membershipState.copy(newGossip)
     assertLatestGossip()
   }
 
@@ -1290,8 +1238,11 @@ private[cluster] class ClusterCoreDaemon(publisher: ActorRef) extends Actor with
     if (Cluster.isAssertInvariantsEnabled && latestGossip.version.versions.size > latestGossip.members.size)
       throw new IllegalStateException(s"Too many vector clock entries in gossip state ${latestGossip}")
 
-  def publish(newGossip: Gossip): Unit = {
-    publisher ! PublishChanges(newGossip)
+  def publishMembershipState(): Unit = {
+    if (cluster.settings.Debug.VerboseGossipLogging)
+      log.debug("Cluster Node [{}] dc [{}] - New gossip published [{}]", selfAddress, cluster.settings.SelfDataCenter, membershipState.latestGossip)
+
+    publisher ! PublishChanges(membershipState)
     if (PublishStatsInterval == Duration.Zero) publishInternalStats()
   }
 

akka-cluster/src/main/scala/akka/cluster/ClusterEvent.scala

@@ -5,13 +5,18 @@ package akka.cluster
 
 import language.postfixOps
 import scala.collection.immutable
-import scala.collection.immutable.VectorBuilder
+import scala.collection.immutable.{ SortedSet, VectorBuilder }
 import akka.actor.{ Actor, ActorLogging, ActorRef, Address }
+import akka.cluster.ClusterSettings.DataCenter
 import akka.cluster.ClusterEvent._
 import akka.cluster.MemberStatus._
 import akka.event.EventStream
-import akka.dispatch.{ UnboundedMessageQueueSemantics, RequiresMessageQueue }
+import akka.dispatch.{ RequiresMessageQueue, UnboundedMessageQueueSemantics }
 import akka.actor.DeadLetterSuppression
+import akka.annotation.InternalApi
+
+import scala.collection.breakOut
+import scala.runtime.AbstractFunction5
 
 /**
  * Domain events published to the event bus.
@@ -51,15 +56,51 @@ object ClusterEvent {
    */
   sealed trait ClusterDomainEvent extends DeadLetterSuppression
 
-  /**
-   * Current snapshot state of the cluster. Sent to new subscriber.
-   */
-  final case class CurrentClusterState(
+  // for binary compatibility (used to be a case class)
+  object CurrentClusterState extends AbstractFunction5[immutable.SortedSet[Member], Set[Member], Set[Address], Option[Address], Map[String, Option[Address]], CurrentClusterState] {
+
+    def apply(
       members:       immutable.SortedSet[Member]  = immutable.SortedSet.empty,
       unreachable:   Set[Member]                  = Set.empty,
       seenBy:        Set[Address]                 = Set.empty,
       leader:        Option[Address]              = None,
-    roleLeaderMap: Map[String, Option[Address]] = Map.empty) {
+      roleLeaderMap: Map[String, Option[Address]] = Map.empty): CurrentClusterState =
+      new CurrentClusterState(members, unreachable, seenBy, leader, roleLeaderMap)
+
+    def unapply(cs: CurrentClusterState): Option[(immutable.SortedSet[Member], Set[Member], Set[Address], Option[Address], Map[String, Option[Address]])] =
+      Some((
+        cs.members,
+        cs.unreachable,
+        cs.seenBy,
+        cs.leader,
+        cs.roleLeaderMap))
+
+  }
+
+  /**
+   * Current snapshot state of the cluster. Sent to new subscriber.
+   *
+   * @param leader leader of the data center of this node
+   */
+  @SerialVersionUID(2)
+  final class CurrentClusterState(
+    val members:                immutable.SortedSet[Member],
+    val unreachable:            Set[Member],
+    val seenBy:                 Set[Address],
+    val leader:                 Option[Address],
+    val roleLeaderMap:          Map[String, Option[Address]],
+    val unreachableDataCenters: Set[DataCenter])
+    extends Product5[immutable.SortedSet[Member], Set[Member], Set[Address], Option[Address], Map[String, Option[Address]]]
+    with Serializable {
+
+    // for binary compatibility
+    def this(
+      members:       immutable.SortedSet[Member]  = immutable.SortedSet.empty,
+      unreachable:   Set[Member]                  = Set.empty,
+      seenBy:        Set[Address]                 = Set.empty,
+      leader:        Option[Address]              = None,
+      roleLeaderMap: Map[String, Option[Address]] = Map.empty) =
+      this(members, unreachable, seenBy, leader, roleLeaderMap, Set.empty)
 
     /**
      * Java API: get current member list.
@@ -75,6 +116,12 @@ object ClusterEvent {
     def getUnreachable: java.util.Set[Member] =
       scala.collection.JavaConverters.setAsJavaSetConverter(unreachable).asJava
 
+    /**
+     * Java API: All data centers in the cluster
+     */
+    def getUnreachableDataCenters: java.util.Set[String] =
+      scala.collection.JavaConverters.setAsJavaSetConverter(unreachableDataCenters).asJava
+
     /**
      * Java API: get current “seen-by” set.
      */
@@ -82,10 +129,21 @@ object ClusterEvent {
       scala.collection.JavaConverters.setAsJavaSetConverter(seenBy).asJava
 
     /**
-     * Java API: get address of current leader, or null if none
+     * Java API: get address of current data center leader, or null if none
      */
     def getLeader: Address = leader orNull
 
+    /**
+     * get address of current leader, if any, within the data center that has the given role
+     */
+    def roleLeader(role: String): Option[Address] = roleLeaderMap.getOrElse(role, None)
+
+    /**
+     * Java API: get address of current leader, if any, within the data center that has the given role
+     * or null if no such node exists
+     */
+    def getRoleLeader(role: String): Address = roleLeaderMap.get(role).flatten.orNull
+
     /**
      * All node roles in the cluster
      */
@@ -98,15 +156,57 @@ object ClusterEvent {
       scala.collection.JavaConverters.setAsJavaSetConverter(allRoles).asJava
 
     /**
-     * get address of current leader, if any, within the role set
+     * All data centers in the cluster
      */
-    def roleLeader(role: String): Option[Address] = roleLeaderMap.getOrElse(role, None)
+    def allDataCenters: Set[String] = members.map(_.dataCenter)(breakOut)
 
     /**
-     * Java API: get address of current leader within the role set,
-     * or null if no node with that role
+     * Java API: All data centers in the cluster
      */
-    def getRoleLeader(role: String): Address = roleLeaderMap.get(role).flatten.orNull
+    def getAllDataCenters: java.util.Set[String] =
+      scala.collection.JavaConverters.setAsJavaSetConverter(allDataCenters).asJava
+
+    /**
+     * Replace the set of unreachable datacenters with the given set
+     */
+    def withUnreachableDataCenters(unreachableDataCenters: Set[DataCenter]): CurrentClusterState =
+      new CurrentClusterState(members, unreachable, seenBy, leader, roleLeaderMap, unreachableDataCenters)
+
+    // for binary compatibility (used to be a case class)
+    def copy(
+      members:       immutable.SortedSet[Member]  = this.members,
+      unreachable:   Set[Member]                  = this.unreachable,
+      seenBy:        Set[Address]                 = this.seenBy,
+      leader:        Option[Address]              = this.leader,
+      roleLeaderMap: Map[String, Option[Address]] = this.roleLeaderMap) =
+      new CurrentClusterState(members, unreachable, seenBy, leader, roleLeaderMap, unreachableDataCenters)
+
+    override def equals(other: Any): Boolean = other match {
+      case that: CurrentClusterState ⇒
+        (this eq that) || (
+          members == that.members &&
+          unreachable == that.unreachable &&
+          seenBy == that.seenBy &&
+          leader == that.leader &&
+          roleLeaderMap == that.roleLeaderMap)
+      case _ ⇒ false
+    }
+
+    override def hashCode(): Int = {
+      val state = Seq(members, unreachable, seenBy, leader, roleLeaderMap)
+      state.map(_.hashCode()).foldLeft(0)((a, b) ⇒ 31 * a + b)
+    }
+
+    // Product5
+    override def productPrefix = "CurrentClusterState"
+    def _1: SortedSet[Member] = members
+    def _2: Set[Member] = unreachable
+    def _3: Set[Address] = seenBy
+    def _4: Option[Address] = leader
+    def _5: Map[String, Option[Address]] = roleLeaderMap
+    def canEqual(that: Any): Boolean = that.isInstanceOf[CurrentClusterState]
+
+    override def toString = s"CurrentClusterState($members, $unreachable, $seenBy, $leader, $roleLeaderMap)"
   }
 
   /**
@@ -171,7 +271,7 @@ object ClusterEvent {
   }
 
   /**
-   * Leader of the cluster members changed. Published when the state change
+   * Leader of the cluster data center of this node changed. Published when the state change
    * is first seen on a node.
    */
   final case class LeaderChanged(leader: Option[Address]) extends ClusterDomainEvent {
@@ -183,7 +283,8 @@ object ClusterEvent {
   }
 
   /**
-   * First member (leader) of the members within a role set changed.
+   * First member (leader) of the members within a role set (in the same data center as this node,
+   * if data centers are used) changed.
    * Published when the state change is first seen on a node.
    */
   final case class RoleLeaderChanged(role: String, leader: Option[Address]) extends ClusterDomainEvent {
@@ -225,6 +326,22 @@ object ClusterEvent {
    */
   final case class ReachableMember(member: Member) extends ReachabilityEvent
 
+  /**
+   * Marker interface to facilitate subscription of
+   * both [[UnreachableDataCenter]] and [[ReachableDataCenter]].
+   */
+  sealed trait DataCenterReachabilityEvent extends ClusterDomainEvent
+
+  /**
+   * A data center is considered as unreachable when any members from the data center are unreachable
+   */
+  final case class UnreachableDataCenter(dataCenter: DataCenter) extends DataCenterReachabilityEvent
+
+  /**
+   * A data center is considered reachable when all members from the data center are reachable
+   */
+  final case class ReachableDataCenter(dataCenter: DataCenter) extends DataCenterReachabilityEvent
+
   /**
    * INTERNAL API
    * The nodes that have seen current version of the Gossip.
@@ -246,35 +363,76 @@ object ClusterEvent {
   /**
    * INTERNAL API
    */
-  private[cluster] def diffUnreachable(oldGossip: Gossip, newGossip: Gossip, selfUniqueAddress: UniqueAddress): immutable.Seq[UnreachableMember] =
-    if (newGossip eq oldGossip) Nil
+  private[cluster] def diffUnreachable(oldState: MembershipState, newState: MembershipState): immutable.Seq[UnreachableMember] =
+    if (newState eq oldState) Nil
     else {
-      val oldUnreachableNodes = oldGossip.overview.reachability.allUnreachableOrTerminated
-      (newGossip.overview.reachability.allUnreachableOrTerminated.collect {
-        case node if !oldUnreachableNodes.contains(node) && node != selfUniqueAddress ⇒
+      val newGossip = newState.latestGossip
+      val oldUnreachableNodes = oldState.dcReachabilityNoOutsideNodes.allUnreachableOrTerminated
+      newState.dcReachabilityNoOutsideNodes.allUnreachableOrTerminated.collect {
+        case node if !oldUnreachableNodes.contains(node) && node != newState.selfUniqueAddress ⇒
           UnreachableMember(newGossip.member(node))
-      })(collection.breakOut)
+      }(collection.breakOut)
     }
 
   /**
    * INTERNAL API
    */
-  private[cluster] def diffReachable(oldGossip: Gossip, newGossip: Gossip, selfUniqueAddress: UniqueAddress): immutable.Seq[ReachableMember] =
-    if (newGossip eq oldGossip) Nil
+  private[cluster] def diffReachable(oldState: MembershipState, newState: MembershipState): immutable.Seq[ReachableMember] =
+    if (newState eq oldState) Nil
     else {
-      (oldGossip.overview.reachability.allUnreachable.collect {
-        case node if newGossip.hasMember(node) && newGossip.overview.reachability.isReachable(node) && node != selfUniqueAddress ⇒
+      val newGossip = newState.latestGossip
+      oldState.dcReachabilityNoOutsideNodes.allUnreachable.collect {
+        case node if newGossip.hasMember(node) && newState.dcReachabilityNoOutsideNodes.isReachable(node) && node != newState.selfUniqueAddress ⇒
           ReachableMember(newGossip.member(node))
-      })(collection.breakOut)
+      }(collection.breakOut)
+    }
 
+  /**
+   * Internal API
+   */
+  private[cluster] def isReachable(state: MembershipState, oldUnreachableNodes: Set[UniqueAddress])(otherDc: DataCenter): Boolean = {
+    val unrelatedDcNodes = state.latestGossip.members.collect {
+      case m if m.dataCenter != otherDc && m.dataCenter != state.selfDc ⇒ m.uniqueAddress
+    }
+
+    val reachabilityForOtherDc = state.dcReachabilityWithoutObservationsWithin.remove(unrelatedDcNodes)
+    reachabilityForOtherDc.allUnreachable.filterNot(oldUnreachableNodes).isEmpty
+  }
+
+  /**
+   * INTERNAL API
+   */
+  private[cluster] def diffUnreachableDataCenter(oldState: MembershipState, newState: MembershipState): immutable.Seq[UnreachableDataCenter] = {
+    if (newState eq oldState) Nil
+    else {
+      val otherDcs = (oldState.latestGossip.allDataCenters union newState.latestGossip.allDataCenters) - newState.selfDc
+      otherDcs.filterNot(isReachable(newState, oldState.dcReachability.allUnreachableOrTerminated)).map(UnreachableDataCenter)(collection.breakOut)
+    }
+  }
+
+  /**
+   * INTERNAL API
+   */
+  private[cluster] def diffReachableDataCenter(oldState: MembershipState, newState: MembershipState): immutable.Seq[ReachableDataCenter] = {
+    if (newState eq oldState) Nil
+    else {
+      val otherDcs = (oldState.latestGossip.allDataCenters union newState.latestGossip.allDataCenters) - newState.selfDc
+
+      val oldUnreachableDcs = otherDcs.filterNot(isReachable(oldState, Set()))
+      val currentUnreachableDcs = otherDcs.filterNot(isReachable(newState, Set()))
+
+      (oldUnreachableDcs diff currentUnreachableDcs).map(ReachableDataCenter)(collection.breakOut)
+    }
   }
 
   /**
    * INTERNAL API.
    */
-  private[cluster] def diffMemberEvents(oldGossip: Gossip, newGossip: Gossip): immutable.Seq[MemberEvent] =
-    if (newGossip eq oldGossip) Nil
+  private[cluster] def diffMemberEvents(oldState: MembershipState, newState: MembershipState): immutable.Seq[MemberEvent] =
+    if (newState eq oldState) Nil
     else {
+      val oldGossip = oldState.latestGossip
+      val newGossip = newState.latestGossip
       val newMembers = newGossip.members diff oldGossip.members
       val membersGroupedByAddress = List(newGossip.members, oldGossip.members).flatten.groupBy(_.uniqueAddress)
       val changedMembers = membersGroupedByAddress collect {
@@ -299,32 +457,35 @@ object ClusterEvent {
   /**
    * INTERNAL API
    */
-  private[cluster] def diffLeader(oldGossip: Gossip, newGossip: Gossip, selfUniqueAddress: UniqueAddress): immutable.Seq[LeaderChanged] = {
-    val newLeader = newGossip.leader(selfUniqueAddress)
-    if (newLeader != oldGossip.leader(selfUniqueAddress)) List(LeaderChanged(newLeader.map(_.address)))
+  @InternalApi
+  private[cluster] def diffLeader(oldState: MembershipState, newState: MembershipState): immutable.Seq[LeaderChanged] = {
+    val newLeader = newState.leader
+    if (newLeader != oldState.leader) List(LeaderChanged(newLeader.map(_.address)))
     else Nil
   }
 
   /**
    * INTERNAL API
    */
-  private[cluster] def diffRolesLeader(oldGossip: Gossip, newGossip: Gossip, selfUniqueAddress: UniqueAddress): Set[RoleLeaderChanged] = {
+  @InternalApi
+  private[cluster] def diffRolesLeader(oldState: MembershipState, newState: MembershipState): Set[RoleLeaderChanged] = {
     for {
-      role ← (oldGossip.allRoles union newGossip.allRoles)
-      newLeader = newGossip.roleLeader(role, selfUniqueAddress)
-      if newLeader != oldGossip.roleLeader(role, selfUniqueAddress)
+      role ← oldState.latestGossip.allRoles union newState.latestGossip.allRoles
+      newLeader = newState.roleLeader(role)
+      if newLeader != oldState.roleLeader(role)
     } yield RoleLeaderChanged(role, newLeader.map(_.address))
   }
 
   /**
    * INTERNAL API
    */
-  private[cluster] def diffSeen(oldGossip: Gossip, newGossip: Gossip, selfUniqueAddress: UniqueAddress): immutable.Seq[SeenChanged] =
-    if (newGossip eq oldGossip) Nil
+  @InternalApi
+  private[cluster] def diffSeen(oldState: MembershipState, newState: MembershipState): immutable.Seq[SeenChanged] =
+    if (oldState eq newState) Nil
     else {
-      val newConvergence = newGossip.convergence(selfUniqueAddress, Set.empty)
-      val newSeenBy = newGossip.seenBy
-      if (newConvergence != oldGossip.convergence(selfUniqueAddress, Set.empty) || newSeenBy != oldGossip.seenBy)
+      val newConvergence = newState.convergence(Set.empty)
+      val newSeenBy = newState.latestGossip.seenBy
+      if (newConvergence != oldState.convergence(Set.empty) || newSeenBy != oldState.latestGossip.seenBy)
         List(SeenChanged(newConvergence, newSeenBy.map(_.address)))
       else Nil
     }
@@ -332,9 +493,10 @@ object ClusterEvent {
   /**
    * INTERNAL API
    */
-  private[cluster] def diffReachability(oldGossip: Gossip, newGossip: Gossip): immutable.Seq[ReachabilityChanged] =
-    if (newGossip.overview.reachability eq oldGossip.overview.reachability) Nil
-    else List(ReachabilityChanged(newGossip.overview.reachability))
+  @InternalApi
+  private[cluster] def diffReachability(oldState: MembershipState, newState: MembershipState): immutable.Seq[ReachabilityChanged] =
+    if (newState.overview.reachability eq oldState.overview.reachability) Nil
+    else List(ReachabilityChanged(newState.overview.reachability))
 
 }
 
@@ -347,8 +509,15 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
   with RequiresMessageQueue[UnboundedMessageQueueSemantics] {
   import InternalClusterAction._
 
-  val selfUniqueAddress = Cluster(context.system).selfUniqueAddress
-  var latestGossip: Gossip = Gossip.empty
+  val cluster = Cluster(context.system)
+  val selfUniqueAddress = cluster.selfUniqueAddress
+  val emptyMembershipState = MembershipState(
+    Gossip.empty,
+    cluster.selfUniqueAddress,
+    cluster.settings.SelfDataCenter,
+    cluster.settings.MultiDataCenter.CrossDcConnections)
+  var membershipState: MembershipState = emptyMembershipState
+  def selfDc = cluster.settings.SelfDataCenter
 
   override def preRestart(reason: Throwable, message: Option[Any]) {
     // don't postStop when restarted, no children to stop
@@ -357,11 +526,11 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
   override def postStop(): Unit = {
     // publish the final removed state before shutting down
     publish(ClusterShuttingDown)
-    publishChanges(Gossip.empty)
+    publishChanges(emptyMembershipState)
   }
 
   def receive = {
-    case PublishChanges(newGossip)           ⇒ publishChanges(newGossip)
+    case PublishChanges(newState)            ⇒ publishChanges(newState)
     case currentStats: CurrentInternalStats  ⇒ publishInternalStats(currentStats)
     case SendCurrentClusterState(receiver)   ⇒ sendCurrentClusterState(receiver)
     case Subscribe(subscriber, initMode, to) ⇒ subscribe(subscriber, initMode, to)
@@ -376,16 +545,23 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
    * to mimic what you would have seen if you were listening to the events.
    */
   def sendCurrentClusterState(receiver: ActorRef): Unit = {
-    val unreachable: Set[Member] = latestGossip.overview.reachability.allUnreachableOrTerminated.collect {
-      case node if node != selfUniqueAddress ⇒ latestGossip.member(node)
+    val unreachable: Set[Member] =
+      membershipState.dcReachabilityNoOutsideNodes.allUnreachableOrTerminated.collect {
+        case node if node != selfUniqueAddress ⇒ membershipState.latestGossip.member(node)
       }
-    val state = CurrentClusterState(
-      members = latestGossip.members,
+
+    val unreachableDataCenters: Set[DataCenter] =
+      if (!membershipState.latestGossip.isMultiDc) Set.empty
+      else membershipState.latestGossip.allDataCenters.filterNot(isReachable(membershipState, Set.empty))
+
+    val state = new CurrentClusterState(
+      members = membershipState.latestGossip.members,
       unreachable = unreachable,
-      seenBy = latestGossip.seenBy.map(_.address),
-      leader = latestGossip.leader(selfUniqueAddress).map(_.address),
-      roleLeaderMap = latestGossip.allRoles.map(r ⇒ r → latestGossip.roleLeader(r, selfUniqueAddress)
-        .map(_.address))(collection.breakOut))
+      seenBy = membershipState.latestGossip.seenBy.map(_.address),
+      leader = membershipState.leader.map(_.address),
+      roleLeaderMap = membershipState.latestGossip.allRoles.map(r ⇒
+        r → membershipState.roleLeader(r).map(_.address))(collection.breakOut),
+      unreachableDataCenters)
     receiver ! state
   }
 
@@ -396,7 +572,7 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
           if (to.exists(_.isAssignableFrom(event.getClass)))
             subscriber ! event
         }
-        publishDiff(Gossip.empty, latestGossip, pub)
+        publishDiff(emptyMembershipState, membershipState, pub)
       case InitialStateAsSnapshot ⇒
         sendCurrentClusterState(subscriber)
     }
@@ -409,22 +585,27 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
     case Some(c) ⇒ eventStream.unsubscribe(subscriber, c)
   }
 
-  def publishChanges(newGossip: Gossip): Unit = {
-    val oldGossip = latestGossip
-    // keep the latestGossip to be sent to new subscribers
-    latestGossip = newGossip
-    publishDiff(oldGossip, newGossip, publish)
+  def publishChanges(newState: MembershipState): Unit = {
+    val oldState = membershipState
+    // keep the latest state to be sent to new subscribers
+    membershipState = newState
+    publishDiff(oldState, newState, publish)
   }
 
-  def publishDiff(oldGossip: Gossip, newGossip: Gossip, pub: AnyRef ⇒ Unit): Unit = {
-    diffMemberEvents(oldGossip, newGossip) foreach pub
-    diffUnreachable(oldGossip, newGossip, selfUniqueAddress) foreach pub
-    diffReachable(oldGossip, newGossip, selfUniqueAddress) foreach pub
-    diffLeader(oldGossip, newGossip, selfUniqueAddress) foreach pub
-    diffRolesLeader(oldGossip, newGossip, selfUniqueAddress) foreach pub
+  def publishDiff(oldState: MembershipState, newState: MembershipState, pub: AnyRef ⇒ Unit): Unit = {
+    def inSameDc(reachabilityEvent: ReachabilityEvent): Boolean =
+      reachabilityEvent.member.dataCenter == selfDc
+
+    diffMemberEvents(oldState, newState) foreach pub
+    diffUnreachable(oldState, newState) foreach pub
+    diffReachable(oldState, newState) foreach pub
+    diffUnreachableDataCenter(oldState, newState) foreach pub
+    diffReachableDataCenter(oldState, newState) foreach pub
+    diffLeader(oldState, newState) foreach pub
+    diffRolesLeader(oldState, newState) foreach pub
     // publish internal SeenState for testing purposes
-    diffSeen(oldGossip, newGossip, selfUniqueAddress) foreach pub
-    diffReachability(oldGossip, newGossip) foreach pub
+    diffSeen(oldState, newState) foreach pub
+    diffReachability(oldState, newState) foreach pub
   }
 
   def publishInternalStats(currentStats: CurrentInternalStats): Unit = publish(currentStats)
@@ -432,6 +613,6 @@ private[cluster] final class ClusterDomainEventPublisher extends Actor with Acto
   def publish(event: AnyRef): Unit = eventStream publish event
 
   def clearState(): Unit = {
-    latestGossip = Gossip.empty
+    membershipState = emptyMembershipState
   }
 }

akka-cluster/src/main/scala/akka/cluster/ClusterHeartbeat.scala

@@ -5,17 +5,18 @@ package akka.cluster
 
 import scala.annotation.tailrec
 import scala.collection.immutable
-import akka.actor.{ ActorLogging, ActorSelection, Address, Actor, RootActorPath }
+import akka.actor.{ Actor, ActorLogging, ActorPath, ActorRef, ActorSelection, Address, DeadLetterSuppression, RootActorPath }
 import akka.cluster.ClusterEvent._
 import akka.remote.FailureDetectorRegistry
 import akka.remote.HeartbeatMessage
-import akka.actor.DeadLetterSuppression
+import akka.annotation.InternalApi
 
 /**
  * INTERNAL API.
  *
  * Receives Heartbeat messages and replies.
  */
+@InternalApi
 private[cluster] final class ClusterHeartbeatReceiver extends Actor with ActorLogging {
   import ClusterHeartbeatSender._
 
@@ -29,6 +30,15 @@ private[cluster] final class ClusterHeartbeatReceiver extends Actor with ActorLo
 
 }
 
+/** INTERNAL API: Utilities to obtain ClusterHeartbeatReceiver paths */
+@InternalApi
+private[cluster] object ClusterHeartbeatReceiver {
+
+  def name: String = "heartbeatReceiver"
+  def path(address: Address): ActorPath =
+    RootActorPath(address) / "system" / "cluster" / name
+}
+
 /**
  * INTERNAL API
  */
@@ -65,12 +75,14 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
   import cluster.settings._
   import context.dispatcher
 
-  // the failureDetector is only updated by this actor, but read from other places
-  val failureDetector = Cluster(context.system).failureDetector
+  val filterInternalClusterMembers: Member ⇒ Boolean =
+    _.dataCenter == cluster.selfDataCenter
 
   val selfHeartbeat = Heartbeat(selfAddress)
 
-  var state = ClusterHeartbeatSenderState(
+  val failureDetector = cluster.failureDetector
+
+  var state: ClusterHeartbeatSenderState = ClusterHeartbeatSenderState(
     ring = HeartbeatNodeRing(selfUniqueAddress, Set(selfUniqueAddress), Set.empty, MonitoredByNrOfMembers),
     oldReceiversNowUnreachable = Set.empty[UniqueAddress],
     failureDetector)
@@ -94,7 +106,7 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
    * Looks up and returns the remote cluster heartbeat connection for the specific address.
    */
   def heartbeatReceiver(address: Address): ActorSelection =
-    context.actorSelection(RootActorPath(address) / "system" / "cluster" / "heartbeatReceiver")
+    context.actorSelection(ClusterHeartbeatReceiver.path(address))
 
   def receive = initializing
 
@@ -116,16 +128,21 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
   }
 
   def init(snapshot: CurrentClusterState): Unit = {
-    val nodes: Set[UniqueAddress] = snapshot.members.map(_.uniqueAddress)
-    val unreachable: Set[UniqueAddress] = snapshot.unreachable.map(_.uniqueAddress)
+    val nodes = snapshot.members.collect { case m if filterInternalClusterMembers(m) ⇒ m.uniqueAddress }
+    val unreachable = snapshot.unreachable.collect { case m if filterInternalClusterMembers(m) ⇒ m.uniqueAddress }
     state = state.init(nodes, unreachable)
   }
 
   def addMember(m: Member): Unit =
-    if (m.uniqueAddress != selfUniqueAddress && !state.contains(m.uniqueAddress))
+    if (m.uniqueAddress != selfUniqueAddress && // is not self 
+      !state.contains(m.uniqueAddress) && // not already added
+      filterInternalClusterMembers(m) // should be watching members from this DC (internal / external)
+      ) {
       state = state.addMember(m.uniqueAddress)
+    }
 
   def removeMember(m: Member): Unit =
+    if (filterInternalClusterMembers(m)) { // we only ever deal with internal cluster members here
       if (m.uniqueAddress == cluster.selfUniqueAddress) {
         // This cluster node will be shutdown, but stop this actor immediately
         // to avoid further updates
@@ -133,6 +150,7 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
       } else {
         state = state.removeMember(m.uniqueAddress)
       }
+    }
 
   def unreachableMember(m: Member): Unit =
     state = state.unreachableMember(m.uniqueAddress)
@@ -142,7 +160,7 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
 
   def heartbeat(): Unit = {
     state.activeReceivers foreach { to ⇒
-      if (cluster.failureDetector.isMonitoring(to.address)) {
+      if (failureDetector.isMonitoring(to.address)) {
         if (verboseHeartbeat) log.debug("Cluster Node [{}] - Heartbeat to [{}]", selfAddress, to.address)
       } else {
         if (verboseHeartbeat) log.debug("Cluster Node [{}] - First Heartbeat to [{}]", selfAddress, to.address)
@@ -152,7 +170,6 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
       }
       heartbeatReceiver(to.address) ! selfHeartbeat
     }
-
   }
 
   def heartbeatRsp(from: UniqueAddress): Unit = {
@@ -173,6 +190,7 @@ private[cluster] final class ClusterHeartbeatSender extends Actor with ActorLogg
  * State of [[ClusterHeartbeatSender]]. Encapsulated to facilitate unit testing.
  * It is immutable, but it updates the failureDetector.
  */
+@InternalApi
 private[cluster] final case class ClusterHeartbeatSenderState(
   ring:                       HeartbeatNodeRing,
   oldReceiversNowUnreachable: Set[UniqueAddress],
@@ -262,7 +280,7 @@ private[cluster] final case class HeartbeatNodeRing(
   /**
    * Receivers for `selfAddress`. Cached for subsequent access.
    */
-  lazy val myReceivers: immutable.Set[UniqueAddress] = receivers(selfAddress)
+  lazy val myReceivers: Set[UniqueAddress] = receivers(selfAddress)
 
   private val useAllAsReceivers = monitoredByNrOfMembers >= (nodeRing.size - 1)
 

akka-cluster/src/main/scala/akka/cluster/ClusterJmx.scala

@@ -154,7 +154,7 @@ private[akka] class ClusterJmx(cluster: Cluster, log: LoggingAdapter) {
         val members = clusterView.members.toSeq.sorted(Member.ordering).map { m ⇒
           s"""{
               |      "address": "${m.address}",
-              |      "roles": [${if (m.roles.isEmpty) "" else m.roles.map("\"" + _ + "\"").mkString("\n        ", ",\n        ", "\n      ")}],
+              |      "roles": [${if (m.roles.isEmpty) "" else m.roles.toList.sorted.map("\"" + _ + "\"").mkString("\n        ", ",\n        ", "\n      ")}],
               |      "status": "${m.status}"
               |    }""".stripMargin
         } mkString (",\n    ")

akka-cluster/src/main/scala/akka/cluster/ClusterReadView.scala

@@ -70,6 +70,12 @@ private[akka] class ClusterReadView(cluster: Cluster) extends Closeable {
             _state = _state.copy(roleLeaderMap = _state.roleLeaderMap + (role → leader))
           case stats: CurrentInternalStats ⇒ _latestStats = stats
           case ClusterShuttingDown         ⇒
+
+          case r: ReachableDataCenter ⇒
+            _state = _state.withUnreachableDataCenters(_state.unreachableDataCenters - r.dataCenter)
+          case r: UnreachableDataCenter ⇒
+            _state = _state.withUnreachableDataCenters(_state.unreachableDataCenters + r.dataCenter)
+
         }
         case s: CurrentClusterState ⇒ _state = s
       }
@@ -109,12 +115,12 @@ private[akka] class ClusterReadView(cluster: Cluster) extends Closeable {
   def status: MemberStatus = self.status
 
   /**
-   * Is this node the leader?
+   * Is this node the current data center leader
    */
   def isLeader: Boolean = leader.contains(selfAddress)
 
   /**
-   * Get the address of the current leader.
+   * Get the address of the current data center leader
    */
   def leader: Option[Address] = state.leader
 

akka-cluster/src/main/scala/akka/cluster/ClusterSettings.scala

@@ -10,14 +10,31 @@ import com.typesafe.config.ConfigObject
 import scala.concurrent.duration.Duration
 import akka.actor.Address
 import akka.actor.AddressFromURIString
+import akka.annotation.InternalApi
 import akka.dispatch.Dispatchers
-import akka.util.Helpers.{ Requiring, ConfigOps, toRootLowerCase }
+import akka.util.Helpers.{ ConfigOps, Requiring, toRootLowerCase }
 
 import scala.concurrent.duration.FiniteDuration
 import akka.japi.Util.immutableSeq
 
-final class ClusterSettings(val config: Config, val systemName: String) {
+object ClusterSettings {
+  type DataCenter = String
+  /**
+   * INTERNAL API.
+   */
+  @InternalApi
+  private[akka] val DcRolePrefix = "dc-"
 
+  /**
+   * INTERNAL API.
+   */
+  @InternalApi
+  private[akka] val DefaultDataCenter: DataCenter = "default"
+
+}
+
+final class ClusterSettings(val config: Config, val systemName: String) {
+  import ClusterSettings._
   private val cc = config.getConfig("akka.cluster")
 
   val LogInfo: Boolean = cc.getBoolean("log-info")
@@ -33,6 +50,28 @@ final class ClusterSettings(val config: Config, val systemName: String) {
     FailureDetectorConfig.getInt("monitored-by-nr-of-members")
   } requiring (_ > 0, "failure-detector.monitored-by-nr-of-members must be > 0")
 
+  final class CrossDcFailureDetectorSettings(val config: Config) {
+    val ImplementationClass: String = config.getString("implementation-class")
+    val HeartbeatInterval: FiniteDuration = {
+      config.getMillisDuration("heartbeat-interval")
+    } requiring (_ > Duration.Zero, "failure-detector.heartbeat-interval must be > 0")
+    val HeartbeatExpectedResponseAfter: FiniteDuration = {
+      config.getMillisDuration("expected-response-after")
+    } requiring (_ > Duration.Zero, "failure-detector.expected-response-after > 0")
+    def NrOfMonitoringActors: Int = MultiDataCenter.CrossDcConnections
+  }
+
+  object MultiDataCenter {
+    val CrossDcConnections: Int = cc.getInt("multi-data-center.cross-data-center-connections")
+      .requiring(_ > 0, "cross-data-center-connections must be > 0")
+
+    val CrossDcGossipProbability: Double = cc.getDouble("multi-data-center.cross-data-center-gossip-probability")
+      .requiring(d ⇒ d >= 0.0D && d <= 1.0D, "cross-data-center-gossip-probability must be >= 0.0 and <= 1.0")
+
+    val CrossDcFailureDetectorSettings: CrossDcFailureDetectorSettings =
+      new CrossDcFailureDetectorSettings(cc.getConfig("multi-data-center.failure-detector"))
+  }
+
   val SeedNodes: immutable.IndexedSeq[Address] =
     immutableSeq(cc.getStringList("seed-nodes")).map { case AddressFromURIString(address) ⇒ address }.toVector
   val SeedNodeTimeout: FiniteDuration = cc.getMillisDuration("seed-node-timeout")
@@ -58,6 +97,11 @@ final class ClusterSettings(val config: Config, val systemName: String) {
     }
   }
 
+  val PruneGossipTombstonesAfter: Duration = {
+    val key = "prune-gossip-tombstones-after"
+    cc.getMillisDuration(key) requiring (_ >= Duration.Zero, key + " >= 0s")
+  }
+
   // specific to the [[akka.cluster.DefaultDowningProvider]]
   val AutoDownUnreachableAfter: Duration = {
     val key = "auto-down-unreachable-after"
@@ -91,9 +135,18 @@ final class ClusterSettings(val config: Config, val systemName: String) {
   val QuarantineRemovedNodeAfter: FiniteDuration =
     cc.getMillisDuration("quarantine-removed-node-after") requiring (_ > Duration.Zero, "quarantine-removed-node-after must be > 0")
 
-  val AllowWeaklyUpMembers = cc.getBoolean("allow-weakly-up-members")
+  val AllowWeaklyUpMembers: Boolean = cc.getBoolean("allow-weakly-up-members")
+
+  val SelfDataCenter: DataCenter = cc.getString("multi-data-center.self-data-center")
+
+  val Roles: Set[String] = {
+    val configuredRoles = (immutableSeq(cc.getStringList("roles")).toSet) requiring (
+      _.forall(!_.startsWith(DcRolePrefix)),
+      s"Roles must not start with '${DcRolePrefix}' as that is reserved for the cluster self-data-center setting")
+
+    configuredRoles + s"$DcRolePrefix$SelfDataCenter"
+  }
 
-  val Roles: Set[String] = immutableSeq(cc.getStringList("roles")).toSet
   val MinNrOfMembers: Int = {
     cc.getInt("min-nr-of-members")
   } requiring (_ > 0, "min-nr-of-members must be > 0")
@@ -116,7 +169,8 @@ final class ClusterSettings(val config: Config, val systemName: String) {
   val SchedulerTicksPerWheel: Int = cc.getInt("scheduler.ticks-per-wheel")
 
   object Debug {
-    val VerboseHeartbeatLogging = cc.getBoolean("debug.verbose-heartbeat-logging")
+    val VerboseHeartbeatLogging: Boolean = cc.getBoolean("debug.verbose-heartbeat-logging")
+    val VerboseGossipLogging: Boolean = cc.getBoolean("debug.verbose-gossip-logging")
   }
 
 }

akka-cluster/src/main/scala/akka/cluster/CrossDcClusterHeartbeat.scala

@@ -0,0 +1,340 @@
+/*
+ * Copyright (C) 2017 Lightbend Inc. <http://www.lightbend.com/>
+ */
+
+package akka.cluster
+
+import akka.actor.{ Actor, ActorLogging, ActorSelection, Address, NoSerializationVerificationNeeded, RootActorPath }
+import akka.annotation.InternalApi
+import akka.cluster.ClusterEvent._
+import akka.cluster.ClusterSettings.DataCenter
+import akka.event.Logging
+import akka.remote.FailureDetectorRegistry
+import akka.util.ConstantFun
+
+import scala.collection.{ SortedSet, immutable, breakOut }
+
+/**
+ * INTERNAL API
+ *
+ * This actor is will be started on all nodes participating in a cluster,
+ * however unlike the within-dc heartbeat sender ([[ClusterHeartbeatSender]]),
+ * it will only actively work on `n` "oldest" nodes of a given data center.
+ *
+ * It will monitor it's oldest counterparts in other data centers.
+ * For example, a DC configured to have (up to) 4 monitoring actors,
+ * will have 4 such active at any point in time, and those will monitor
+ * the (at most) 4 oldest nodes of each data center.
+ *
+ * This monitoring mode is both simple and predictable, and also uses the assumption that
+ * "nodes which stay around for a long time, become old", and those rarely change. In a way,
+ * they are the "core" of a cluster, while other nodes may be very dynamically changing worked
+ * nodes which aggresively come and go as the traffic in the service changes.
+ */
+@InternalApi
+private[cluster] final class CrossDcHeartbeatSender extends Actor with ActorLogging {
+  import CrossDcHeartbeatSender._
+
+  val cluster = Cluster(context.system)
+  val verboseHeartbeat = cluster.settings.Debug.VerboseHeartbeatLogging
+  import cluster.settings._
+  import cluster.{ scheduler, selfAddress, selfDataCenter, selfUniqueAddress }
+  import context.dispatcher
+
+  // For inspecting if in active state; allows avoiding "becoming active" when already active
+  var activelyMonitoring = false
+
+  val isExternalClusterMember: Member ⇒ Boolean =
+    member ⇒ member.dataCenter != cluster.selfDataCenter
+
+  val crossDcSettings: cluster.settings.CrossDcFailureDetectorSettings =
+    cluster.settings.MultiDataCenter.CrossDcFailureDetectorSettings
+
+  val crossDcFailureDetector = cluster.crossDcFailureDetector
+
+  val selfHeartbeat = ClusterHeartbeatSender.Heartbeat(selfAddress)
+
+  var dataCentersState: CrossDcHeartbeatingState = CrossDcHeartbeatingState.init(
+    selfDataCenter,
+    crossDcFailureDetector,
+    crossDcSettings.NrOfMonitoringActors,
+    SortedSet.empty)
+
+  // start periodic heartbeat to other nodes in cluster
+  val heartbeatTask = scheduler.schedule(
+    PeriodicTasksInitialDelay max HeartbeatInterval,
+    HeartbeatInterval, self, ClusterHeartbeatSender.HeartbeatTick)
+
+  override def preStart(): Unit = {
+    cluster.subscribe(self, classOf[MemberEvent])
+    if (verboseHeartbeat) log.debug("Initialized cross-dc heartbeat sender as DORMANT in DC: [{}]", selfDataCenter)
+  }
+
+  override def postStop(): Unit = {
+    dataCentersState.activeReceivers.foreach(a ⇒ crossDcFailureDetector.remove(a.address))
+    heartbeatTask.cancel()
+    cluster.unsubscribe(self)
+  }
+
+  /**
+   * Looks up and returns the remote cluster heartbeat connection for the specific address.
+   */
+  def heartbeatReceiver(address: Address): ActorSelection =
+    context.actorSelection(ClusterHeartbeatReceiver.path(address))
+
+  def receive: Actor.Receive =
+    dormant orElse introspecting
+
+  /**
+   * In this state no cross-datacenter heartbeats are sent by this actor.
+   * This may be because one of those reasons:
+   *   - no nodes in other DCs were detected yet
+   *   - nodes in other DCs are present, but this node is not tht n-th oldest in this DC (see
+   *     `number-of-cross-datacenter-monitoring-actors`), so it does not have to monitor that other data centers
+   *
+   * In this state it will however listen to cluster events to eventually take over monitoring other DCs
+   * in case it becomes "old enough".
+   */
+  def dormant: Actor.Receive = {
+    case s: CurrentClusterState               ⇒ init(s)
+    case MemberRemoved(m, _)                  ⇒ removeMember(m)
+    case evt: MemberEvent                     ⇒ addMember(evt.member)
+    case ClusterHeartbeatSender.HeartbeatTick ⇒ // ignore...
+  }
+
+  def active: Actor.Receive = {
+    case ClusterHeartbeatSender.HeartbeatTick                ⇒ heartbeat()
+    case ClusterHeartbeatSender.HeartbeatRsp(from)           ⇒ heartbeatRsp(from)
+    case MemberRemoved(m, _)                                 ⇒ removeMember(m)
+    case evt: MemberEvent                                    ⇒ addMember(evt.member)
+    case ClusterHeartbeatSender.ExpectedFirstHeartbeat(from) ⇒ triggerFirstHeartbeat(from)
+  }
+
+  def introspecting: Actor.Receive = {
+    case ReportStatus() ⇒
+      sender() ! {
+        if (activelyMonitoring) CrossDcHeartbeatSender.MonitoringActive(dataCentersState)
+        else CrossDcHeartbeatSender.MonitoringDormant()
+      }
+  }
+
+  def init(snapshot: CurrentClusterState): Unit = {
+    // val unreachable = snapshot.unreachable.collect({ case m if isExternalClusterMember(m) => m.uniqueAddress })
+    // nr of monitored nodes is the same as the number of monitoring nodes (`n` oldest in one DC watch `n` oldest in other)
+    val nodes = snapshot.members
+    val nrOfMonitoredNodes = crossDcSettings.NrOfMonitoringActors
+    dataCentersState = CrossDcHeartbeatingState.init(selfDataCenter, crossDcFailureDetector, nrOfMonitoredNodes, nodes)
+    becomeActiveIfResponsibleForHeartbeat()
+  }
+
+  def addMember(m: Member): Unit =
+    if (CrossDcHeartbeatingState.atLeastInUpState(m)) {
+      // since we only monitor nodes in Up or later states, due to the n-th oldest requirement
+      dataCentersState = dataCentersState.addMember(m)
+      if (verboseHeartbeat && m.dataCenter != selfDataCenter)
+        log.debug("Register member {} for cross DC heartbeat (will only heartbeat if oldest)", m)
+
+      becomeActiveIfResponsibleForHeartbeat()
+    }
+
+  def removeMember(m: Member): Unit =
+    if (m.uniqueAddress == cluster.selfUniqueAddress) {
+      // This cluster node will be shutdown, but stop this actor immediately to avoid further updates
+      context stop self
+    } else {
+      dataCentersState = dataCentersState.removeMember(m)
+      becomeActiveIfResponsibleForHeartbeat()
+    }
+
+  def heartbeat(): Unit = {
+    dataCentersState.activeReceivers foreach { to ⇒
+      if (crossDcFailureDetector.isMonitoring(to.address)) {
+        if (verboseHeartbeat) log.debug("Cluster Node [{}][{}] - (Cross) Heartbeat to [{}]", selfDataCenter, selfAddress, to.address)
+      } else {
+        if (verboseHeartbeat) log.debug("Cluster Node [{}][{}] - First (Cross) Heartbeat to [{}]", selfDataCenter, selfAddress, to.address)
+        // schedule the expected first heartbeat for later, which will give the
+        // other side a chance to reply, and also trigger some resends if needed
+        scheduler.scheduleOnce(HeartbeatExpectedResponseAfter, self, ClusterHeartbeatSender.ExpectedFirstHeartbeat(to))
+      }
+      heartbeatReceiver(to.address) ! selfHeartbeat
+    }
+  }
+
+  def heartbeatRsp(from: UniqueAddress): Unit = {
+    if (verboseHeartbeat) log.debug("Cluster Node [{}][{}] - (Cross) Heartbeat response from [{}]", selfDataCenter, selfAddress, from.address)
+    dataCentersState = dataCentersState.heartbeatRsp(from)
+  }
+
+  def triggerFirstHeartbeat(from: UniqueAddress): Unit =
+    if (dataCentersState.activeReceivers.contains(from) && !crossDcFailureDetector.isMonitoring(from.address)) {
+      if (verboseHeartbeat) log.debug("Cluster Node [{}][{}] - Trigger extra expected (cross) heartbeat from [{}]", selfAddress, from.address)
+      crossDcFailureDetector.heartbeat(from.address)
+    }
+
+  private def selfIsResponsibleForCrossDcHeartbeat(): Boolean = {
+    val activeDcs: Int = dataCentersState.dataCenters.size
+    if (activeDcs > 1) dataCentersState.shouldActivelyMonitorNodes(selfDataCenter, selfUniqueAddress)
+    else false
+  }
+
+  /** Idempotent, become active if this node is n-th oldest and should monitor other nodes */
+  private def becomeActiveIfResponsibleForHeartbeat(): Unit = {
+    if (!activelyMonitoring && selfIsResponsibleForCrossDcHeartbeat()) {
+      log.info("Cross DC heartbeat becoming ACTIVE on this node (for DC: {}), monitoring other DCs oldest nodes", selfDataCenter)
+      activelyMonitoring = true
+
+      context.become(active orElse introspecting)
+    } else if (!activelyMonitoring)
+      if (verboseHeartbeat) log.info("Remaining DORMANT; others in {} handle heartbeating other DCs", selfDataCenter)
+  }
+
+}
+
+/** INTERNAL API */
+@InternalApi
+private[akka] object CrossDcHeartbeatSender {
+
+  // -- messages intended only for local messaging during testing --
+  sealed trait InspectionCommand extends NoSerializationVerificationNeeded
+  final case class ReportStatus()
+
+  sealed trait StatusReport extends NoSerializationVerificationNeeded
+  sealed trait MonitoringStateReport extends StatusReport
+  final case class MonitoringActive(state: CrossDcHeartbeatingState) extends MonitoringStateReport
+  final case class MonitoringDormant() extends MonitoringStateReport
+  // -- end of messages intended only for local messaging during testing --
+}
+
+/** INTERNAL API */
+@InternalApi
+private[cluster] final case class CrossDcHeartbeatingState(
+  selfDataCenter:          DataCenter,
+  failureDetector:         FailureDetectorRegistry[Address],
+  nrOfMonitoredNodesPerDc: Int,
+  state:                   Map[ClusterSettings.DataCenter, SortedSet[Member]]) {
+  import CrossDcHeartbeatingState._
+
+  /**
+   * Decides if `self` node should become active and monitor other nodes with heartbeats.
+   * Only the `nrOfMonitoredNodesPerDc`-oldest nodes in each DC fulfil this role.
+   */
+  def shouldActivelyMonitorNodes(selfDc: ClusterSettings.DataCenter, selfAddress: UniqueAddress): Boolean = {
+    // Since we need ordering of oldests guaranteed, we must only look at Up (or Leaving, Exiting...) nodes
+
+    val selfDcNeighbours: SortedSet[Member] = state.getOrElse(selfDc, emptyMembersSortedSet)
+    val selfDcOldOnes = selfDcNeighbours.filter(atLeastInUpState).take(nrOfMonitoredNodesPerDc)
+
+    // if this node is part of the "n oldest nodes" it should indeed monitor other nodes:
+    val shouldMonitorActively = selfDcOldOnes.exists(_.uniqueAddress == selfAddress)
+    shouldMonitorActively
+  }
+
+  def addMember(m: Member): CrossDcHeartbeatingState = {
+    val dc = m.dataCenter
+
+    // we need to remove the member first, to avoid having "duplicates"
+    // this is because the removal and uniqueness we need is only by uniqueAddress
+    // which is not used by the `ageOrdering`
+    val oldMembersWithoutM = state.getOrElse(dc, emptyMembersSortedSet)
+      .filterNot(_.uniqueAddress == m.uniqueAddress)
+
+    val updatedMembers = oldMembersWithoutM + m
+    val updatedState = this.copy(state = state.updated(dc, updatedMembers))
+
+    // guarding against the case of two members having the same upNumber, in which case the activeReceivers
+    // which are based on the ageOrdering could actually have changed by adding a node. In practice this
+    // should happen rarely, since upNumbers are assigned sequentially, and we only ever compare nodes
+    // in the same DC. If it happens though, we need to remove the previously monitored node from the failure
+    // detector, to prevent both a resource leak and that node actually appearing as unreachable in the gossip (!)
+    val stoppedMonitoringReceivers = updatedState.activeReceiversIn(dc) diff this.activeReceiversIn(dc)
+    stoppedMonitoringReceivers.foreach(m ⇒ failureDetector.remove(m.address)) // at most one element difference
+
+    updatedState
+  }
+
+  def removeMember(m: Member): CrossDcHeartbeatingState = {
+    val dc = m.dataCenter
+    state.get(dc) match {
+      case Some(dcMembers) ⇒
+        val updatedMembers = dcMembers.filterNot(_.uniqueAddress == m.uniqueAddress)
+
+        failureDetector.remove(m.address)
+        copy(state = state.updated(dc, updatedMembers))
+      case None ⇒
+        this // no change needed, was certainly not present (not even its DC was)
+    }
+  }
+
+  /** Lists addresses that this node should send heartbeats to */
+  val activeReceivers: Set[UniqueAddress] = {
+    val otherDcs = state.filter(_._1 != selfDataCenter)
+    val allOtherNodes = otherDcs.values
+
+    allOtherNodes.flatMap(
+      _.take(nrOfMonitoredNodesPerDc)
+        .map(_.uniqueAddress)(breakOut)).toSet
+  }
+
+  /** Lists addresses in diven DataCenter that this node should send heartbeats to */
+  private def activeReceiversIn(dc: DataCenter): Set[UniqueAddress] =
+    if (dc == selfDataCenter) Set.empty // CrossDcHeartbeatSender is not supposed to send within its own Dc
+    else {
+      val otherNodes = state.getOrElse(dc, emptyMembersSortedSet)
+      otherNodes
+        .take(nrOfMonitoredNodesPerDc)
+        .map(_.uniqueAddress)(breakOut)
+    }
+
+  def allMembers: Iterable[Member] =
+    state.values.flatMap(ConstantFun.scalaIdentityFunction)
+
+  def heartbeatRsp(from: UniqueAddress): CrossDcHeartbeatingState = {
+    if (activeReceivers.contains(from)) {
+      failureDetector heartbeat from.address
+    }
+    this
+  }
+
+  def dataCenters: Set[DataCenter] =
+    state.keys.toSet
+
+}
+
+/** INTERNAL API */
+@InternalApi
+private[cluster] object CrossDcHeartbeatingState {
+
+  /** Sorted by age */
+  private def emptyMembersSortedSet: SortedSet[Member] = SortedSet.empty[Member](Member.ageOrdering)
+
+  // Since we need ordering of oldests guaranteed, we must only look at Up (or Leaving, Exiting...) nodes
+  def atLeastInUpState(m: Member): Boolean =
+    m.status != MemberStatus.WeaklyUp && m.status != MemberStatus.Joining
+
+  def init(
+    selfDataCenter:          DataCenter,
+    crossDcFailureDetector:  FailureDetectorRegistry[Address],
+    nrOfMonitoredNodesPerDc: Int,
+    members:                 SortedSet[Member]): CrossDcHeartbeatingState = {
+    new CrossDcHeartbeatingState(
+      selfDataCenter,
+      crossDcFailureDetector,
+      nrOfMonitoredNodesPerDc,
+      state = {
+      // TODO unduplicate this with the logic in MembershipState.ageSortedTopOldestMembersPerDc
+      val groupedByDc = members.filter(atLeastInUpState).groupBy(_.dataCenter)
+
+      if (members.ordering == Member.ageOrdering) {
+        // we already have the right ordering
+        groupedByDc
+      } else {
+        // we need to enforce the ageOrdering for the SortedSet in each DC
+        groupedByDc.map {
+          case (dc, ms) ⇒
+            dc → (SortedSet.empty[Member](Member.ageOrdering) union ms)
+        }
+      }
+    })
+  }
+
+}

akka-cluster/src/main/scala/akka/cluster/Gossip.scala

@@ -4,24 +4,25 @@
 
 package akka.cluster
 
-import scala.collection.immutable
+import scala.collection.{ SortedSet, immutable }
+import ClusterSettings.DataCenter
 import MemberStatus._
+import akka.annotation.InternalApi
+
 import scala.concurrent.duration.Deadline
 
 /**
  * INTERNAL API
  */
 private[cluster] object Gossip {
+  type Timestamp = Long
   val emptyMembers: immutable.SortedSet[Member] = immutable.SortedSet.empty
   val empty: Gossip = new Gossip(Gossip.emptyMembers)
 
   def apply(members: immutable.SortedSet[Member]) =
     if (members.isEmpty) empty else empty.copy(members = members)
 
-  private val leaderMemberStatus = Set[MemberStatus](Up, Leaving)
-  private val convergenceMemberStatus = Set[MemberStatus](Up, Leaving)
-  val convergenceSkipUnreachableWithMemberStatus = Set[MemberStatus](Down, Exiting)
-  val removeUnreachableWithMemberStatus = Set[MemberStatus](Down, Exiting)
+  def vclockName(node: UniqueAddress): String = s"${node.address}-${node.longUid}"
 
 }
 
@@ -59,17 +60,19 @@ private[cluster] object Gossip {
  * removed node telling it to shut itself down.
  */
 @SerialVersionUID(1L)
+@InternalApi
 private[cluster] final case class Gossip(
   members:    immutable.SortedSet[Member], // sorted set of members with their status, sorted by address
   overview:   GossipOverview                       = GossipOverview(),
-  version:  VectorClock                 = VectorClock()) { // vector clock version
+  version:    VectorClock                          = VectorClock(), // vector clock version
+  tombstones: Map[UniqueAddress, Gossip.Timestamp] = Map.empty) {
 
   if (Cluster.isAssertInvariantsEnabled) assertInvariants()
 
   private def assertInvariants(): Unit = {
 
     if (members.exists(_.status == Removed))
-      throw new IllegalArgumentException(s"Live members must have status [${Removed}], " +
+      throw new IllegalArgumentException(s"Live members must not have status [${Removed}], " +
         s"got [${members.filter(_.status == Removed)}]")
 
     val inReachabilityButNotMember = overview.reachability.allObservers diff members.map(_.uniqueAddress)
@@ -86,6 +89,13 @@ private[cluster] final case class Gossip(
   @transient private lazy val membersMap: Map[UniqueAddress, Member] =
     members.map(m ⇒ m.uniqueAddress → m)(collection.breakOut)
 
+  @transient lazy val isMultiDc =
+    if (members.size <= 1) false
+    else {
+      val dc1 = members.head.dataCenter
+      members.exists(_.dataCenter != dc1)
+    }
+
   /**
    * Increments the version for this 'Node'.
    */
@@ -138,15 +148,20 @@ private[cluster] final case class Gossip(
     this copy (overview = overview copy (seen = overview.seen union that.overview.seen))
 
   /**
-   * Merges two Gossip instances including membership tables, and the VectorClock histories.
+   * Merges two Gossip instances including membership tables, tombstones, and the VectorClock histories.
    */
   def merge(that: Gossip): Gossip = {
 
-    // 1. merge vector clocks
-    val mergedVClock = this.version merge that.version
+    // 1. merge sets of tombstones
+    val mergedTombstones = tombstones ++ that.tombstones
+
+    // 2. merge vector clocks (but remove entries for tombstoned nodes)
+    val mergedVClock = mergedTombstones.keys.foldLeft(this.version merge that.version) { (vclock, node) ⇒
+      vclock.prune(VectorClock.Node(Gossip.vclockName(node)))
+    }
 
     // 2. merge members by selecting the single Member with highest MemberStatus out of the Member groups
-    val mergedMembers = Gossip.emptyMembers union Member.pickHighestPriority(this.members, that.members)
+    val mergedMembers = Gossip.emptyMembers union Member.pickHighestPriority(this.members, that.members, mergedTombstones)
 
     // 3. merge reachability table by picking records with highest version
     val mergedReachability = this.overview.reachability.merge(
@@ -156,29 +171,7 @@ private[cluster] final case class Gossip(
     // 4. Nobody can have seen this new gossip yet
     val mergedSeen = Set.empty[UniqueAddress]
 
-    Gossip(mergedMembers, GossipOverview(mergedSeen, mergedReachability), mergedVClock)
-  }
-
-  /**
-   * Checks if we have a cluster convergence. If there are any unreachable nodes then we can't have a convergence -
-   * waiting for user to act (issuing DOWN) or leader to act (issuing DOWN through auto-down).
-   *
-   * @return true if convergence have been reached and false if not
-   */
-  def convergence(selfUniqueAddress: UniqueAddress, exitingConfirmed: Set[UniqueAddress]): Boolean = {
-    // First check that:
-    //   1. we don't have any members that are unreachable, excluding observations from members
-    //      that have status DOWN, or
-    //   2. all unreachable members in the set have status DOWN or EXITING
-    // Else we can't continue to check for convergence
-    // When that is done we check that all members with a convergence
-    // status is in the seen table, i.e. has seen this version
-    val unreachable = reachabilityExcludingDownedObservers.allUnreachableOrTerminated.collect {
-      case node if (node != selfUniqueAddress && !exitingConfirmed(node)) ⇒ member(node)
-    }
-    unreachable.forall(m ⇒ Gossip.convergenceSkipUnreachableWithMemberStatus(m.status)) &&
-      !members.exists(m ⇒ Gossip.convergenceMemberStatus(m.status) &&
-        !(seenByNode(m.uniqueAddress) || exitingConfirmed(m.uniqueAddress)))
+    Gossip(mergedMembers, GossipOverview(mergedSeen, mergedReachability), mergedVClock, mergedTombstones)
   }
 
   lazy val reachabilityExcludingDownedObservers: Reachability = {
@@ -186,29 +179,23 @@ private[cluster] final case class Gossip(
     overview.reachability.removeObservers(downed.map(_.uniqueAddress))
   }
 
-  def isLeader(node: UniqueAddress, selfUniqueAddress: UniqueAddress): Boolean =
-    leader(selfUniqueAddress).contains(node)
-
-  def leader(selfUniqueAddress: UniqueAddress): Option[UniqueAddress] =
-    leaderOf(members, selfUniqueAddress)
-
-  def roleLeader(role: String, selfUniqueAddress: UniqueAddress): Option[UniqueAddress] =
-    leaderOf(members.filter(_.hasRole(role)), selfUniqueAddress)
-
-  def leaderOf(mbrs: immutable.SortedSet[Member], selfUniqueAddress: UniqueAddress): Option[UniqueAddress] = {
-    val reachableMembers =
-      if (overview.reachability.isAllReachable) mbrs.filterNot(_.status == Down)
-      else mbrs.filter(m ⇒ m.status != Down &&
-        (overview.reachability.isReachable(m.uniqueAddress) || m.uniqueAddress == selfUniqueAddress))
-    if (reachableMembers.isEmpty) None
-    else reachableMembers.find(m ⇒ Gossip.leaderMemberStatus(m.status)).
-      orElse(Some(reachableMembers.min(Member.leaderStatusOrdering))).map(_.uniqueAddress)
-  }
+  def allDataCenters: Set[DataCenter] = members.map(_.dataCenter)
 
   def allRoles: Set[String] = members.flatMap(_.roles)
 
   def isSingletonCluster: Boolean = members.size == 1
 
+  /**
+   * @return true if fromAddress should be able to reach toAddress based on the unreachability data and their
+   *         respective data centers
+   */
+  def isReachable(fromAddress: UniqueAddress, toAddress: UniqueAddress): Boolean =
+    if (!hasMember(toAddress)) false
+    else {
+      // as it looks for specific unreachable entires for the node pair we don't have to filter on data center
+      overview.reachability.isReachable(fromAddress, toAddress)
+    }
+
   def member(node: UniqueAddress): Member = {
     membersMap.getOrElse(
       node,
@@ -222,14 +209,60 @@ private[cluster] final case class Gossip(
     members.maxBy(m ⇒ if (m.upNumber == Int.MaxValue) 0 else m.upNumber)
   }
 
+  def removeAll(nodes: Iterable[UniqueAddress], removalTimestamp: Long): Gossip = {
+    nodes.foldLeft(this)((gossip, node) ⇒ gossip.remove(node, removalTimestamp))
+  }
+
+  def update(updatedMembers: immutable.SortedSet[Member]): Gossip = {
+    copy(members = updatedMembers union members)
+  }
+
+  /**
+   * Remove the given member from the set of members and mark it's removal with a tombstone to avoid having it
+   * reintroduced when merging with another gossip that has not seen the removal.
+   */
+  def remove(node: UniqueAddress, removalTimestamp: Long): Gossip = {
+    // removing REMOVED nodes from the `seen` table
+    val newSeen = overview.seen - node
+    // removing REMOVED nodes from the `reachability` table
+    val newReachability = overview.reachability.remove(node :: Nil)
+    val newOverview = overview.copy(seen = newSeen, reachability = newReachability)
+
+    // Clear the VectorClock when member is removed. The change made by the leader is stamped
+    // and will propagate as is if there are no other changes on other nodes.
+    // If other concurrent changes on other nodes (e.g. join) the pruning is also
+    // taken care of when receiving gossips.
+    val newVersion = version.prune(VectorClock.Node(Gossip.vclockName(node)))
+    val newMembers = members.filterNot(_.uniqueAddress == node)
+    val newTombstones = tombstones + (node → removalTimestamp)
+    copy(version = newVersion, members = newMembers, overview = newOverview, tombstones = newTombstones)
+  }
+
+  def markAsDown(member: Member): Gossip = {
+    // replace member (changed status)
+    val newMembers = members - member + member.copy(status = Down)
+    // remove nodes marked as DOWN from the `seen` table
+    val newSeen = overview.seen - member.uniqueAddress
+
+    // update gossip overview
+    val newOverview = overview copy (seen = newSeen)
+    copy(members = newMembers, overview = newOverview) // update gossip
+  }
+
   def prune(removedNode: VectorClock.Node): Gossip = {
     val newVersion = version.prune(removedNode)
     if (newVersion eq version) this
     else copy(version = newVersion)
   }
 
+  def pruneTombstones(removeEarlierThan: Gossip.Timestamp): Gossip = {
+    val newTombstones = tombstones.filter { case (_, timestamp) ⇒ timestamp > removeEarlierThan }
+    if (newTombstones.size == tombstones.size) this
+    else copy(tombstones = newTombstones)
+  }
+
   override def toString =
-    s"Gossip(members = [${members.mkString(", ")}], overview = ${overview}, version = ${version})"
+    s"Gossip(members = [${members.mkString(", ")}], overview = $overview, version = $version, tombstones = $tombstones)"
 }
 
 /**

akka-cluster/src/main/scala/akka/cluster/Member.scala

@@ -6,6 +6,8 @@ package akka.cluster
 
 import akka.actor.Address
 import MemberStatus._
+import akka.annotation.InternalApi
+import akka.cluster.ClusterSettings.DataCenter
 
 import scala.runtime.AbstractFunction2
 
@@ -22,6 +24,10 @@ class Member private[cluster] (
   val status:                    MemberStatus,
   val roles:                     Set[String]) extends Serializable {
 
+  lazy val dataCenter: DataCenter = roles.find(_.startsWith(ClusterSettings.DcRolePrefix))
+    .getOrElse(throw new IllegalStateException("DataCenter undefined, should not be possible"))
+    .substring(ClusterSettings.DcRolePrefix.length)
+
   def address: Address = uniqueAddress.address
 
   override def hashCode = uniqueAddress.##
@@ -29,7 +35,11 @@ class Member private[cluster] (
     case m: Member ⇒ uniqueAddress == m.uniqueAddress
     case _         ⇒ false
   }
-  override def toString = s"Member(address = ${address}, status = ${status})"
+  override def toString =
+    if (dataCenter == ClusterSettings.DefaultDataCenter)
+      s"Member(address = $address, status = $status)"
+    else
+      s"Member(address = $address, dataCenter = $dataCenter, status = $status)"
 
   def hasRole(role: String): Boolean = roles.contains(role)
 
@@ -43,7 +53,9 @@ class Member private[cluster] (
    * Is this member older, has been part of cluster longer, than another
    * member. It is only correct when comparing two existing members in a
    * cluster. A member that joined after removal of another member may be
-   * considered older than the removed member.
+   * considered older than the removed member. Note that is only makes
+   * sense to compare with other members inside of one data center (upNumber has
+   * a higher risk of being reused across data centers). // TODO should we enforce this to compare only within DCs?
    */
   def isOlderThan(other: Member): Boolean =
     if (upNumber == other.upNumber)
@@ -84,7 +96,8 @@ object Member {
   /**
    * INTERNAL API
    */
-  private[cluster] def removed(node: UniqueAddress): Member = new Member(node, Int.MaxValue, Removed, Set.empty)
+  private[cluster] def removed(node: UniqueAddress): Member =
+    new Member(node, Int.MaxValue, Removed, Set(ClusterSettings.DcRolePrefix + "-N/A"))
 
   /**
    * `Address` ordering type class, sorts addresses by host and port.
@@ -133,16 +146,24 @@ object Member {
     (a, b) ⇒ a.isOlderThan(b)
   }
 
-  def pickHighestPriority(a: Set[Member], b: Set[Member]): Set[Member] = {
+  @deprecated("Was accidentally made a public API, internal", since = "2.5.4")
+  def pickHighestPriority(a: Set[Member], b: Set[Member]): Set[Member] =
+    pickHighestPriority(a, b, Map.empty)
+
+  /**
+   * INTERNAL API.
+   */
+  @InternalApi
+  private[akka] def pickHighestPriority(a: Set[Member], b: Set[Member], tombstones: Map[UniqueAddress, Long]): Set[Member] = {
     // group all members by Address => Seq[Member]
     val groupedByAddress = (a.toSeq ++ b.toSeq).groupBy(_.uniqueAddress)
     // pick highest MemberStatus
-    (Member.none /: groupedByAddress) {
+    groupedByAddress.foldLeft(Member.none) {
       case (acc, (_, members)) ⇒
         if (members.size == 2) acc + members.reduceLeft(highestPriorityOf)
         else {
           val m = members.head
-          if (Gossip.removeUnreachableWithMemberStatus(m.status)) acc // removed
+          if (tombstones.contains(m.uniqueAddress) || MembershipState.removeUnreachableWithMemberStatus(m.status)) acc // removed
           else acc + m
         }
     }

akka-cluster/src/main/scala/akka/cluster/MembershipState.scala

@@ -0,0 +1,328 @@
+/**
+ * Copyright (C) 2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import java.util.{ ArrayList, Collections }
+import java.util.concurrent.ThreadLocalRandom
+
+import scala.collection.immutable
+import scala.collection.SortedSet
+import akka.cluster.ClusterSettings.DataCenter
+import akka.cluster.MemberStatus._
+import akka.annotation.InternalApi
+
+import scala.annotation.tailrec
+import scala.collection.breakOut
+import scala.util.Random
+
+/**
+ * INTERNAL API
+ */
+@InternalApi private[akka] object MembershipState {
+  import MemberStatus._
+  private val leaderMemberStatus = Set[MemberStatus](Up, Leaving)
+  private val convergenceMemberStatus = Set[MemberStatus](Up, Leaving)
+  val convergenceSkipUnreachableWithMemberStatus = Set[MemberStatus](Down, Exiting)
+  val removeUnreachableWithMemberStatus = Set[MemberStatus](Down, Exiting)
+}
+
+/**
+ * INTERNAL API
+ */
+@InternalApi private[akka] final case class MembershipState(
+  latestGossip:       Gossip,
+  selfUniqueAddress:  UniqueAddress,
+  selfDc:             DataCenter,
+  crossDcConnections: Int) {
+
+  import MembershipState._
+
+  lazy val selfMember = latestGossip.member(selfUniqueAddress)
+
+  def members: immutable.SortedSet[Member] = latestGossip.members
+
+  def overview: GossipOverview = latestGossip.overview
+
+  def seen(): MembershipState = copy(latestGossip = latestGossip.seen(selfUniqueAddress))
+
+  /**
+   * Checks if we have a cluster convergence. If there are any in data center node pairs that cannot reach each other
+   * then we can't have a convergence until those nodes reach each other again or one of them is downed
+   *
+   * @return true if convergence have been reached and false if not
+   */
+  def convergence(exitingConfirmed: Set[UniqueAddress]): Boolean = {
+
+    // If another member in the data center that is UP or LEAVING and has not seen this gossip or is exiting
+    // convergence cannot be reached
+    def memberHinderingConvergenceExists =
+      members.exists(member ⇒
+        member.dataCenter == selfDc &&
+          convergenceMemberStatus(member.status) &&
+          !(latestGossip.seenByNode(member.uniqueAddress) || exitingConfirmed(member.uniqueAddress)))
+
+    // Find cluster members in the data center that are unreachable from other members of the data center
+    // excluding observations from members outside of the data center, that have status DOWN or is passed in as confirmed exiting.
+    val unreachableInDc = dcReachabilityExcludingDownedObservers.allUnreachableOrTerminated.collect {
+      case node if node != selfUniqueAddress && !exitingConfirmed(node) ⇒ latestGossip.member(node)
+    }
+    // unreachables outside of the data center or with status DOWN or EXITING does not affect convergence
+    val allUnreachablesCanBeIgnored =
+      unreachableInDc.forall(unreachable ⇒ convergenceSkipUnreachableWithMemberStatus(unreachable.status))
+
+    allUnreachablesCanBeIgnored && !memberHinderingConvergenceExists
+  }
+
+  /**
+   * @return Reachability excluding observations from nodes outside of the data center, but including observed unreachable
+   *         nodes outside of the data center
+   */
+  lazy val dcReachability: Reachability =
+    overview.reachability.removeObservers(members.collect { case m if m.dataCenter != selfDc ⇒ m.uniqueAddress })
+
+  /**
+   * @return Reachability excluding observations from nodes outside of the data center and observations within self data center,
+   *        but including observed unreachable nodes outside of the data center
+   */
+  lazy val dcReachabilityWithoutObservationsWithin: Reachability =
+    dcReachability.filterRecords { r ⇒ latestGossip.member(r.subject).dataCenter != selfDc }
+
+  /**
+   * @return reachability for data center nodes, with observations from outside the data center or from downed nodes filtered out
+   */
+  lazy val dcReachabilityExcludingDownedObservers: Reachability = {
+    val membersToExclude = members.collect { case m if m.status == Down || m.dataCenter != selfDc ⇒ m.uniqueAddress }
+    overview.reachability.removeObservers(membersToExclude).remove(members.collect { case m if m.dataCenter != selfDc ⇒ m.uniqueAddress })
+  }
+
+  lazy val dcReachabilityNoOutsideNodes: Reachability =
+    overview.reachability.remove(members.collect { case m if m.dataCenter != selfDc ⇒ m.uniqueAddress })
+
+  /**
+   * @return Up to `crossDcConnections` oldest members for each DC
+   */
+  lazy val ageSortedTopOldestMembersPerDc: Map[DataCenter, SortedSet[Member]] =
+    // TODO make this recursive and bail early when size reached to make it fast for large clusters
+    latestGossip.members.foldLeft(Map.empty[DataCenter, SortedSet[Member]]) { (acc, member) ⇒
+      acc.get(member.dataCenter) match {
+        case Some(set) ⇒
+          if (set.size < crossDcConnections) acc + (member.dataCenter → (set + member))
+          else acc
+        case None ⇒ acc + (member.dataCenter → (SortedSet.empty(Member.ageOrdering) + member))
+      }
+    }
+
+  /**
+   * @return true if toAddress should be reachable from the fromDc in general, within a data center
+   *         this means only caring about data center local observations, across data centers it
+   *         means caring about all observations for the toAddress.
+   */
+  def isReachableExcludingDownedObservers(toAddress: UniqueAddress): Boolean =
+    if (!latestGossip.hasMember(toAddress)) false
+    else {
+      val to = latestGossip.member(toAddress)
+
+      // if member is in the same data center, we ignore cross data center unreachability
+      if (selfDc == to.dataCenter) dcReachabilityExcludingDownedObservers.isReachable(toAddress)
+      // if not it is enough that any non-downed node observed it as unreachable
+      else latestGossip.reachabilityExcludingDownedObservers.isReachable(toAddress)
+    }
+
+  def dcMembers: SortedSet[Member] =
+    members.filter(_.dataCenter == selfDc)
+
+  def isLeader(node: UniqueAddress): Boolean =
+    leader.contains(node)
+
+  def leader: Option[UniqueAddress] =
+    leaderOf(members)
+
+  def roleLeader(role: String): Option[UniqueAddress] =
+    leaderOf(members.filter(_.hasRole(role)))
+
+  def leaderOf(mbrs: immutable.SortedSet[Member]): Option[UniqueAddress] = {
+    val reachability = dcReachability
+
+    val reachableMembersInDc =
+      if (reachability.isAllReachable) mbrs.filter(m ⇒ m.dataCenter == selfDc && m.status != Down)
+      else mbrs.filter(m ⇒
+        m.dataCenter == selfDc &&
+          m.status != Down &&
+          (reachability.isReachable(m.uniqueAddress) || m.uniqueAddress == selfUniqueAddress))
+    if (reachableMembersInDc.isEmpty) None
+    else reachableMembersInDc.find(m ⇒ leaderMemberStatus(m.status))
+      .orElse(Some(reachableMembersInDc.min(Member.leaderStatusOrdering)))
+      .map(_.uniqueAddress)
+  }
+
+  def isInSameDc(node: UniqueAddress): Boolean =
+    node == selfUniqueAddress || latestGossip.member(node).dataCenter == selfDc
+
+  def validNodeForGossip(node: UniqueAddress): Boolean =
+    node != selfUniqueAddress &&
+      ((isInSameDc(node) && isReachableExcludingDownedObservers(node)) ||
+        // if cross DC we need to check pairwise unreachable observation
+        overview.reachability.isReachable(selfUniqueAddress, node))
+
+}
+
+/**
+ * INTERNAL API
+ */
+@InternalApi private[akka] class GossipTargetSelector(
+  reduceGossipDifferentViewProbability: Double,
+  crossDcGossipProbability:             Double) {
+
+  final def gossipTarget(state: MembershipState): Option[UniqueAddress] = {
+    selectRandomNode(gossipTargets(state))
+  }
+
+  final def gossipTargets(state: MembershipState): Vector[UniqueAddress] =
+    if (state.latestGossip.isMultiDc) multiDcGossipTargets(state)
+    else localDcGossipTargets(state)
+
+  /**
+   * Select `n` random nodes to gossip to (used to quickly inform the rest of the cluster when leaving for example)
+   */
+  def randomNodesForFullGossip(state: MembershipState, n: Int): Vector[UniqueAddress] =
+    if (state.latestGossip.isMultiDc && state.ageSortedTopOldestMembersPerDc(state.selfDc).contains(state.selfMember)) {
+      // this node is one of the N oldest in the cluster, gossip to one cross-dc but mostly locally
+      val randomLocalNodes = Random.shuffle(state.members.toVector.collect {
+        case m if m.dataCenter == state.selfDc && state.validNodeForGossip(m.uniqueAddress) ⇒ m.uniqueAddress
+      })
+
+      @tailrec
+      def selectOtherDcNode(randomizedDcs: List[DataCenter]): Option[UniqueAddress] =
+        randomizedDcs match {
+          case Nil ⇒ None // couldn't find a single cross-dc-node to talk to
+          case dc :: tail ⇒
+            state.ageSortedTopOldestMembersPerDc(dc).collectFirst {
+              case m if state.validNodeForGossip(m.uniqueAddress) ⇒ m.uniqueAddress
+            } match {
+              case Some(addr) ⇒ Some(addr)
+              case None       ⇒ selectOtherDcNode(tail)
+            }
+
+        }
+      val otherDcs = Random.shuffle((state.ageSortedTopOldestMembersPerDc.keySet - state.selfDc).toList)
+
+      selectOtherDcNode(otherDcs) match {
+        case Some(node) ⇒ randomLocalNodes.take(n - 1) :+ node
+        case None       ⇒ randomLocalNodes.take(n)
+      }
+
+    } else {
+      // single dc or not among the N oldest - select local nodes
+      val selectedNodes = state.members.toVector.collect {
+        case m if m.dataCenter == state.selfDc && state.validNodeForGossip(m.uniqueAddress) ⇒ m.uniqueAddress
+      }
+
+      if (selectedNodes.size <= n) selectedNodes
+      else Random.shuffle(selectedNodes).take(n)
+    }
+
+  /**
+   * Chooses a set of possible gossip targets that is in the same dc. If the cluster is not multi dc this
+   * means it is a choice among all nodes of the cluster.
+   */
+  protected def localDcGossipTargets(state: MembershipState): Vector[UniqueAddress] = {
+    val latestGossip = state.latestGossip
+    val firstSelection: Vector[UniqueAddress] =
+      if (preferNodesWithDifferentView(state)) {
+        // If it's time to try to gossip to some nodes with a different view
+        // gossip to a random alive same dc member with preference to a member with older gossip version
+        latestGossip.members.collect {
+          case m if m.dataCenter == state.selfDc && !latestGossip.seenByNode(m.uniqueAddress) && state.validNodeForGossip(m.uniqueAddress) ⇒
+            m.uniqueAddress
+        }(breakOut)
+      } else Vector.empty
+
+    // Fall back to localGossip
+    if (firstSelection.isEmpty) {
+      latestGossip.members.toVector.collect {
+        case m if m.dataCenter == state.selfDc && state.validNodeForGossip(m.uniqueAddress) ⇒ m.uniqueAddress
+      }
+    } else firstSelection
+
+  }
+
+  /**
+   * Choose cross-dc nodes if this one of the N oldest nodes, and if not fall back to gosip locally in the dc
+   */
+  protected def multiDcGossipTargets(state: MembershipState): Vector[UniqueAddress] = {
+    val latestGossip = state.latestGossip
+    // only a fraction of the time across data centers
+    if (selectDcLocalNodes()) localDcGossipTargets(state)
+    else {
+      val nodesPerDc = state.ageSortedTopOldestMembersPerDc
+
+      // only do cross DC gossip if this node is among the N oldest
+
+      if (!nodesPerDc(state.selfDc).contains(state.selfMember)) localDcGossipTargets(state)
+      else {
+        @tailrec
+        def findFirstDcWithValidNodes(left: List[DataCenter]): Vector[UniqueAddress] =
+          left match {
+            case dc :: tail ⇒
+
+              val validNodes = nodesPerDc(dc).collect {
+                case member if state.validNodeForGossip(member.uniqueAddress) ⇒
+                  member.uniqueAddress
+              }
+
+              if (validNodes.nonEmpty) validNodes.toVector
+              else findFirstDcWithValidNodes(tail) // no valid nodes in dc, try next
+
+            case Nil ⇒
+              Vector.empty
+          }
+
+        // chose another DC at random
+        val otherDcsInRandomOrder = dcsInRandomOrder((nodesPerDc - state.selfDc).keys.toList)
+        val nodes = findFirstDcWithValidNodes(otherDcsInRandomOrder)
+        if (nodes.nonEmpty) nodes
+        // no other dc with reachable nodes, fall back to local gossip
+        else localDcGossipTargets(state)
+      }
+    }
+  }
+
+  /**
+   * For large clusters we should avoid shooting down individual
+   * nodes. Therefore the probability is reduced for large clusters.
+   */
+  protected def adjustedGossipDifferentViewProbability(clusterSize: Int): Double = {
+    val low = reduceGossipDifferentViewProbability
+    val high = low * 3
+    // start reduction when cluster is larger than configured ReduceGossipDifferentViewProbability
+    if (clusterSize <= low)
+      reduceGossipDifferentViewProbability
+    else {
+      // don't go lower than 1/10 of the configured GossipDifferentViewProbability
+      val minP = reduceGossipDifferentViewProbability / 10
+      if (clusterSize >= high)
+        minP
+      else {
+        // linear reduction of the probability with increasing number of nodes
+        // from ReduceGossipDifferentViewProbability at ReduceGossipDifferentViewProbability nodes
+        // to ReduceGossipDifferentViewProbability / 10 at ReduceGossipDifferentViewProbability * 3 nodes
+        // i.e. default from 0.8 at 400 nodes, to 0.08 at 1600 nodes
+        val k = (minP - reduceGossipDifferentViewProbability) / (high - low)
+        reduceGossipDifferentViewProbability + (clusterSize - low) * k
+      }
+    }
+  }
+
+  protected def selectDcLocalNodes(): Boolean = ThreadLocalRandom.current.nextDouble() > crossDcGossipProbability
+
+  protected def preferNodesWithDifferentView(state: MembershipState): Boolean =
+    ThreadLocalRandom.current.nextDouble() < adjustedGossipDifferentViewProbability(state.latestGossip.members.size)
+
+  protected def dcsInRandomOrder(dcs: List[DataCenter]): List[DataCenter] =
+    Random.shuffle(dcs)
+
+  protected def selectRandomNode(nodes: IndexedSeq[UniqueAddress]): Option[UniqueAddress] =
+    if (nodes.isEmpty) None
+    else Some(nodes(ThreadLocalRandom.current.nextInt(nodes.size)))
+}

akka-cluster/src/main/scala/akka/cluster/Reachability.scala

@@ -3,6 +3,8 @@
  */
 package akka.cluster
 
+import akka.annotation.InternalApi
+
 import scala.collection.immutable
 import scala.collection.breakOut
 
@@ -40,12 +42,18 @@ private[cluster] object Reachability {
  * record, and thereby it is always possible to determine which record is newest when
  * merging two instances.
  *
+ * By default, each observer treats every other node as reachable. That allows to
+ * introduce the invariant that if an observer sees all nodes as reachable, no
+ * records should be kept at all. Therefore, in a running cluster with full
+ * reachability, no records need to be kept at all.
+ *
  * Aggregated status of a subject node is defined as (in this order):
  * - Terminated if any observer node considers it as Terminated
  * - Unreachable if any observer node considers it as Unreachable
  * - Reachable otherwise, i.e. no observer node considers it as Unreachable
  */
 @SerialVersionUID(1L)
+@InternalApi
 private[cluster] class Reachability private (
   val records:  immutable.IndexedSeq[Reachability.Record],
   val versions: Map[UniqueAddress, Long]) extends Serializable {
@@ -53,6 +61,8 @@ private[cluster] class Reachability private (
   import Reachability._
 
   private class Cache {
+    // `allUnreachable` contains all nodes that have been observed as Unreachable by at least one other node
+    // `allTerminated` contains all nodes that have been observed as Terminated by at least one other node
     val (observerRowsMap, allUnreachable, allTerminated) = {
       if (records.isEmpty) {
         val observerRowsMap = Map.empty[UniqueAddress, Map[UniqueAddress, Reachability.Record]]
@@ -116,15 +126,19 @@ private[cluster] class Reachability private (
     val newVersions = versions.updated(observer, v)
     val newRecord = Record(observer, subject, status, v)
     observerRows(observer) match {
+      // don't record Reachable observation if nothing has been noted so far
       case None if status == Reachable ⇒ this
+      // otherwise, create new instance including this first observation
       case None ⇒
         new Reachability(records :+ newRecord, newVersions)
 
+      // otherwise, update old observations
       case Some(oldObserverRows) ⇒
 
         oldObserverRows.get(subject) match {
           case None ⇒
             if (status == Reachable && oldObserverRows.forall { case (_, r) ⇒ r.status == Reachable }) {
+              // FIXME: how should we have gotten into this state?
               // all Reachable, prune by removing the records of the observer, and bump the version
               new Reachability(records.filterNot(_.observer == observer), newVersions)
             } else
@@ -156,6 +170,10 @@ private[cluster] class Reachability private (
       (this.observerRows(observer), other.observerRows(observer)) match {
         case (None, None) ⇒
         case (Some(rows1), Some(rows2)) ⇒
+          // We throw away a complete set of records based on the version here. Couldn't we lose records here? No,
+          // because the observer gossips always the complete set of records. (That's hard to see in the model, because
+          // records also contain the version number for which they were introduced but actually the version number
+          // corresponds to the whole set of records of one observer at one point in time.
           val rows = if (observerVersion1 > observerVersion2) rows1 else rows2
           recordBuilder ++= rows.collect { case (_, r) if allowed(r.subject) ⇒ r }
         case (Some(rows1), None) ⇒
@@ -191,6 +209,9 @@ private[cluster] class Reachability private (
       Reachability(newRecords, newVersions)
     }
 
+  def filterRecords(f: Record ⇒ Boolean) =
+    Reachability(records.filter(f), versions)
+
   def status(observer: UniqueAddress, subject: UniqueAddress): ReachabilityStatus =
     observerRows(observer) match {
       case None ⇒ Reachable
@@ -205,22 +226,39 @@ private[cluster] class Reachability private (
     else if (cache.allUnreachable(node)) Unreachable
     else Reachable
 
+  /**
+   * @return true if the given node is seen as Reachable, i.e. there's no negative (Unreachable, Terminated) observation
+   * record known for that the node.
+   */
   def isReachable(node: UniqueAddress): Boolean = isAllReachable || !allUnreachableOrTerminated.contains(node)
 
+  /**
+   * @return true if the given observer node can reach the subject node.
+   */
   def isReachable(observer: UniqueAddress, subject: UniqueAddress): Boolean =
     status(observer, subject) == Reachable
 
+  /**
+   * @return true if there's no negative (Unreachable, Terminated) observation record at all for
+   * any node
+   */
   def isAllReachable: Boolean = records.isEmpty
 
   /**
-   * Doesn't include terminated
+   * @return all nodes that are Unreachable (i.e. they have been reported as Unreachable by at least one other node).
+   * This does not include nodes observed to be Terminated.
    */
   def allUnreachable: Set[UniqueAddress] = cache.allUnreachable
 
+  /**
+   * @return all nodes that are Unreachable or Terminated (i.e. they have been reported as Unreachable or Terminated
+   * by at least one other node).
+   */
   def allUnreachableOrTerminated: Set[UniqueAddress] = cache.allUnreachableOrTerminated
 
   /**
-   * Doesn't include terminated
+   * @return all nodes that have been observed as Unreachable by the given observer.
+   * This doesn't include nodes observed as Terminated.
    */
   def allUnreachableFrom(observer: UniqueAddress): Set[UniqueAddress] =
     observerRows(observer) match {
@@ -255,7 +293,7 @@ private[cluster] class Reachability private (
   // only used for testing
   override def equals(obj: Any): Boolean = obj match {
     case other: Reachability ⇒
-      records.size == other.records.size && versions == versions &&
+      records.size == other.records.size && versions == other.versions &&
         cache.observerRowsMap == other.cache.observerRowsMap
     case _ ⇒ false
   }

akka-cluster/src/main/scala/akka/cluster/protobuf/ClusterMessageSerializer.scala

@@ -247,7 +247,7 @@ class ClusterMessageSerializer(val system: ExtendedActorSystem) extends BaseSeri
 
   private def gossipToProto(gossip: Gossip): cm.Gossip.Builder = {
     val allMembers = gossip.members.toVector
-    val allAddresses: Vector[UniqueAddress] = allMembers.map(_.uniqueAddress)
+    val allAddresses: Vector[UniqueAddress] = allMembers.map(_.uniqueAddress) ++ gossip.tombstones.keys
     val addressMapping = allAddresses.zipWithIndex.toMap
     val allRoles = allMembers.foldLeft(Set.empty[String])((acc, m) ⇒ acc union m.roles).to[Vector]
     val roleMapping = allRoles.zipWithIndex.toMap
@@ -274,6 +274,12 @@ class ClusterMessageSerializer(val system: ExtendedActorSystem) extends BaseSeri
       }
     }
 
+    def tombstoneToProto(t: (UniqueAddress, Long)): cm.Tombstone =
+      cm.Tombstone.newBuilder()
+        .setAddressIndex(mapUniqueAddress(t._1))
+        .setTimestamp(t._2)
+        .build()
+
     val reachability = reachabilityToProto(gossip.overview.reachability)
     val members = gossip.members.map(memberToProto)
     val seen = gossip.overview.seen.map(mapUniqueAddress)
@@ -282,8 +288,12 @@ class ClusterMessageSerializer(val system: ExtendedActorSystem) extends BaseSeri
       addAllObserverReachability(reachability.map(_.build).asJava)
 
     cm.Gossip.newBuilder().addAllAllAddresses(allAddresses.map(uniqueAddressToProto(_).build).asJava).
-      addAllAllRoles(allRoles.asJava).addAllAllHashes(allHashes.asJava).addAllMembers(members.map(_.build).asJava).
-      setOverview(overview).setVersion(vectorClockToProto(gossip.version, hashMapping))
+      addAllAllRoles(allRoles.asJava)
+      .addAllAllHashes(allHashes.asJava)
+      .addAllMembers(members.map(_.build).asJava)
+      .setOverview(overview)
+      .setVersion(vectorClockToProto(gossip.version, hashMapping))
+      .addAllTombstones(gossip.tombstones.map(tombstoneToProto).asJava)
   }
 
   private def vectorClockToProto(version: VectorClock, hashMapping: Map[String, Int]): cm.VectorClock.Builder = {
@@ -339,15 +349,35 @@ class ClusterMessageSerializer(val system: ExtendedActorSystem) extends BaseSeri
 
     def memberFromProto(member: cm.Member) =
       new Member(addressMapping(member.getAddressIndex), member.getUpNumber, memberStatusFromInt(member.getStatus.getNumber),
-        member.getRolesIndexesList.asScala.map(roleMapping(_))(breakOut))
+        rolesFromProto(member.getRolesIndexesList.asScala))
+
+    def rolesFromProto(roleIndexes: Seq[Integer]): Set[String] = {
+      var containsDc = false
+      var roles = Set.empty[String]
+
+      for {
+        roleIndex ← roleIndexes
+        role = roleMapping(roleIndex)
+      } {
+        if (role.startsWith(ClusterSettings.DcRolePrefix)) containsDc = true
+        roles += role
+      }
+
+      if (!containsDc) roles + (ClusterSettings.DcRolePrefix + "default")
+      else roles
+    }
+
+    def tombstoneFromProto(tombstone: cm.Tombstone): (UniqueAddress, Long) =
+      (addressMapping(tombstone.getAddressIndex), tombstone.getTimestamp)
 
     val members: immutable.SortedSet[Member] = gossip.getMembersList.asScala.map(memberFromProto)(breakOut)
 
     val reachability = reachabilityFromProto(gossip.getOverview.getObserverReachabilityList.asScala)
     val seen: Set[UniqueAddress] = gossip.getOverview.getSeenList.asScala.map(addressMapping(_))(breakOut)
     val overview = GossipOverview(seen, reachability)
+    val tombstones: Map[UniqueAddress, Long] = gossip.getTombstonesList.asScala.map(tombstoneFromProto)(breakOut)
 
-    Gossip(members, overview, vectorClockFromProto(gossip.getVersion, hashMapping))
+    Gossip(members, overview, vectorClockFromProto(gossip.getVersion, hashMapping), tombstones)
   }
 
   private def vectorClockFromProto(version: cm.VectorClock, hashMapping: immutable.Seq[String]) = {

akka-cluster/src/multi-jvm/scala/akka/cluster/MBeanSpec.scala

@@ -120,6 +120,7 @@ abstract class MBeanSpec
              |    {
              |      "address": "${sortedNodes(0)}",
              |      "roles": [
+             |        "dc-default",
              |        "testNode"
              |      ],
              |      "status": "Up"
@@ -127,6 +128,7 @@ abstract class MBeanSpec
              |    {
              |      "address": "${sortedNodes(1)}",
              |      "roles": [
+             |        "dc-default",
              |        "testNode"
              |      ],
              |      "status": "Up"
@@ -134,6 +136,7 @@ abstract class MBeanSpec
              |    {
              |      "address": "${sortedNodes(2)}",
              |      "roles": [
+             |        "dc-default",
              |        "testNode"
              |      ],
              |      "status": "Up"
@@ -141,6 +144,7 @@ abstract class MBeanSpec
              |    {
              |      "address": "${sortedNodes(3)}",
              |      "roles": [
+             |        "dc-default",
              |        "testNode"
              |      ],
              |      "status": "Up"

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcClusterSpec.scala

@@ -0,0 +1,165 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import akka.cluster.MemberStatus.Up
+import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec }
+import akka.remote.transport.ThrottlerTransportAdapter.Direction
+import com.typesafe.config.ConfigFactory
+
+import scala.concurrent.duration._
+
+class MultiDcSpecConfig(crossDcConnections: Int = 5) extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+  val fifth = role("fifth")
+
+  commonConfig(ConfigFactory.parseString(
+    s"""
+      akka.loglevel = INFO
+      akka.cluster.multi-data-center.cross-data-center-connections = $crossDcConnections
+    """).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+  nodeConfig(first, second)(ConfigFactory.parseString(
+    """
+      akka.cluster.multi-data-center.self-data-center = "dc1"
+    """))
+
+  nodeConfig(third, fourth, fifth)(ConfigFactory.parseString(
+    """
+      akka.cluster.multi-data-center.self-data-center = "dc2"
+    """))
+
+  testTransport(on = true)
+}
+
+object MultiDcNormalConfig extends MultiDcSpecConfig()
+
+class MultiDcMultiJvmNode1 extends MultiDcSpec(MultiDcNormalConfig)
+class MultiDcMultiJvmNode2 extends MultiDcSpec(MultiDcNormalConfig)
+class MultiDcMultiJvmNode3 extends MultiDcSpec(MultiDcNormalConfig)
+class MultiDcMultiJvmNode4 extends MultiDcSpec(MultiDcNormalConfig)
+class MultiDcMultiJvmNode5 extends MultiDcSpec(MultiDcNormalConfig)
+
+object MultiDcFewCrossDcConnectionsConfig extends MultiDcSpecConfig(1)
+
+class MultiDcFewCrossDcMultiJvmNode1 extends MultiDcSpec(MultiDcFewCrossDcConnectionsConfig)
+class MultiDcFewCrossDcMultiJvmNode2 extends MultiDcSpec(MultiDcFewCrossDcConnectionsConfig)
+class MultiDcFewCrossDcMultiJvmNode3 extends MultiDcSpec(MultiDcFewCrossDcConnectionsConfig)
+class MultiDcFewCrossDcMultiJvmNode4 extends MultiDcSpec(MultiDcFewCrossDcConnectionsConfig)
+class MultiDcFewCrossDcMultiJvmNode5 extends MultiDcSpec(MultiDcFewCrossDcConnectionsConfig)
+
+abstract class MultiDcSpec(config: MultiDcSpecConfig)
+  extends MultiNodeSpec(config)
+  with MultiNodeClusterSpec {
+
+  import config._
+
+  "A cluster with multiple data centers" must {
+    "be able to form" in {
+
+      runOn(first) {
+        cluster.join(first)
+      }
+      runOn(second, third, fourth) {
+        cluster.join(first)
+      }
+      enterBarrier("form-cluster-join-attempt")
+
+      runOn(first, second, third, fourth) {
+        within(20.seconds) {
+          awaitAssert(clusterView.members.filter(_.status == MemberStatus.Up) should have size (4))
+        }
+      }
+
+      enterBarrier("cluster started")
+    }
+
+    "have a leader per data center" in {
+      runOn(first, second) {
+        cluster.settings.SelfDataCenter should ===("dc1")
+        clusterView.leader shouldBe defined
+        val dc1 = Set(address(first), address(second))
+        dc1 should contain(clusterView.leader.get)
+      }
+      runOn(third, fourth) {
+        cluster.settings.SelfDataCenter should ===("dc2")
+        clusterView.leader shouldBe defined
+        val dc2 = Set(address(third), address(fourth))
+        dc2 should contain(clusterView.leader.get)
+      }
+
+      enterBarrier("leader per data center")
+    }
+
+    "be able to have data center member changes while there is inter data center unreachability" in within(20.seconds) {
+      runOn(first) {
+        testConductor.blackhole(first, third, Direction.Both).await
+      }
+      enterBarrier("inter-data-center unreachability")
+
+      runOn(fifth) {
+        cluster.join(third)
+      }
+
+      runOn(third, fourth, fifth) {
+        // should be able to join and become up since the
+        // unreachable is between dc1 and dc2,
+        within(10.seconds) {
+          awaitAssert(clusterView.members.filter(_.status == MemberStatus.Up) should have size (5))
+        }
+      }
+
+      runOn(first) {
+        testConductor.passThrough(first, third, Direction.Both).await
+      }
+
+      // should be able to join and become up since the
+      // unreachable is between dc1 and dc2,
+      within(10.seconds) {
+        awaitAssert(clusterView.members.filter(_.status == MemberStatus.Up) should have size (5))
+      }
+
+      enterBarrier("inter-data-center unreachability end")
+    }
+
+    "be able to have data center member changes while there is unreachability in another data center" in within(20.seconds) {
+      runOn(first) {
+        testConductor.blackhole(first, second, Direction.Both).await
+      }
+      enterBarrier("other-data-center-internal-unreachable")
+
+      runOn(third) {
+        cluster.join(fifth)
+        // should be able to join and leave
+        // since the unreachable nodes are inside of dc1
+        cluster.leave(fourth)
+
+        awaitAssert(clusterView.members.map(_.address) should not contain (address(fourth)))
+        awaitAssert(clusterView.members.collect { case m if m.status == Up ⇒ m.address } should contain(address(fifth)))
+      }
+
+      enterBarrier("other-data-center-internal-unreachable changed")
+
+      runOn(first) {
+        testConductor.passThrough(first, second, Direction.Both).await
+      }
+      enterBarrier("other-datac-enter-internal-unreachable end")
+    }
+
+    "be able to down a member of another data-center" in within(20.seconds) {
+      runOn(fifth) {
+        cluster.down(address(second))
+      }
+
+      runOn(first, third, fifth) {
+        awaitAssert(clusterView.members.map(_.address) should not contain (address(second)))
+      }
+      enterBarrier("cross-data-center-downed")
+    }
+
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcHeartbeatTakingOverSpec.scala

@@ -0,0 +1,199 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import akka.actor.ActorSelection
+import akka.annotation.InternalApi
+import akka.remote.testconductor.RoleName
+import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec }
+import akka.testkit._
+import com.typesafe.config.ConfigFactory
+
+import scala.collection.immutable
+import scala.collection.immutable.SortedSet
+import scala.concurrent.duration._
+
+object MultiDcHeartbeatTakingOverSpecMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first") //   alpha
+  val second = role("second") // alpha
+  val third = role("third") //   alpha
+
+  val fourth = role("fourth") // beta
+  val fifth = role("fifth") //   beta
+
+  nodeConfig(first, second, third)(ConfigFactory.parseString(
+    """
+    akka {
+      cluster.multi-data-center.self-data-center = alpha
+    }
+    """))
+
+  nodeConfig(fourth, fifth)(ConfigFactory.parseString(
+    """
+    akka {
+      cluster.multi-data-center.self-data-center = beta
+    }
+    """))
+
+  commonConfig(ConfigFactory.parseString(
+    """
+    akka {
+      actor.provider = cluster
+
+      loggers = ["akka.testkit.TestEventListener"]
+      loglevel = INFO
+
+      remote.log-remote-lifecycle-events = off
+
+      cluster {
+        debug.verbose-heartbeat-logging = off
+
+        multi-data-center {
+          cross-data-center-connections = 2
+        }
+      }
+    }
+    """))
+
+}
+
+class MultiDcHeartbeatTakingOverSpecMultiJvmNode1 extends MultiDcHeartbeatTakingOverSpec
+class MultiDcHeartbeatTakingOverSpecMultiJvmNode2 extends MultiDcHeartbeatTakingOverSpec
+class MultiDcHeartbeatTakingOverSpecMultiJvmNode3 extends MultiDcHeartbeatTakingOverSpec
+class MultiDcHeartbeatTakingOverSpecMultiJvmNode4 extends MultiDcHeartbeatTakingOverSpec
+class MultiDcHeartbeatTakingOverSpecMultiJvmNode5 extends MultiDcHeartbeatTakingOverSpec
+
+abstract class MultiDcHeartbeatTakingOverSpec extends MultiNodeSpec(MultiDcHeartbeatTakingOverSpecMultiJvmSpec)
+  with MultiNodeClusterSpec {
+
+  "A 2-dc cluster" must {
+
+    val observer: TestProbe = TestProbe("alpha-observer")
+
+    val crossDcHeartbeatSenderPath = "/system/cluster/core/daemon/crossDcHeartbeatSender"
+    val selectCrossDcHeartbeatSender: ActorSelection = system.actorSelection(crossDcHeartbeatSenderPath)
+
+    // these will be filled in during the initial phase of the test -----------
+    var expectedAlphaHeartbeaterNodes: SortedSet[Member] = SortedSet.empty
+    var expectedAlphaHeartbeaterRoles: SortedSet[RoleName] = SortedSet.empty
+
+    var expectedBetaHeartbeaterNodes: SortedSet[Member] = SortedSet.empty
+    var expectedBetaHeartbeaterRoles: SortedSet[RoleName] = SortedSet.empty
+
+    var expectedNoActiveHeartbeatSenderRoles: Set[RoleName] = Set.empty
+    // end of these will be filled in during the initial phase of the test -----------
+
+    def refreshOldestMemberHeartbeatStatuses() = {
+      expectedAlphaHeartbeaterNodes = takeNOldestMembers(_.dataCenter == "alpha", 2)
+      expectedAlphaHeartbeaterRoles = membersAsRoles(expectedAlphaHeartbeaterNodes)
+
+      expectedBetaHeartbeaterNodes = takeNOldestMembers(_.dataCenter == "beta", 2)
+      expectedBetaHeartbeaterRoles = membersAsRoles(expectedBetaHeartbeaterNodes)
+
+      expectedNoActiveHeartbeatSenderRoles = roles.toSet -- (expectedAlphaHeartbeaterRoles union expectedBetaHeartbeaterRoles)
+    }
+
+    "collect information on oldest nodes" taggedAs LongRunningTest in {
+      // allow all nodes to join:
+      awaitClusterUp(roles: _*)
+
+      refreshOldestMemberHeartbeatStatuses()
+      info(s"expectedAlphaHeartbeaterNodes = ${expectedAlphaHeartbeaterNodes.map(_.address.port.get)}")
+      info(s"expectedBetaHeartbeaterNodes = ${expectedBetaHeartbeaterNodes.map(_.address.port.get)}")
+      info(s"expectedNoActiveHeartbeatSenderRoles = ${expectedNoActiveHeartbeatSenderRoles.map(_.port.get)}")
+
+      expectedAlphaHeartbeaterRoles.size should ===(2)
+      expectedBetaHeartbeaterRoles.size should ===(2)
+
+      enterBarrier("found-expectations")
+    }
+
+    "be healthy" taggedAs LongRunningTest in within(5.seconds) {
+      implicit val sender = observer.ref
+      runOn(expectedAlphaHeartbeaterRoles.toList: _*) {
+        awaitAssert {
+          selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+          observer.expectMsgType[CrossDcHeartbeatSender.MonitoringActive]
+        }
+      }
+      runOn(expectedBetaHeartbeaterRoles.toList: _*) {
+        awaitAssert {
+          selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+          observer.expectMsgType[CrossDcHeartbeatSender.MonitoringActive]
+        }
+      }
+      runOn(expectedNoActiveHeartbeatSenderRoles.toList: _*) {
+        awaitAssert {
+          selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+          observer.expectMsgType[CrossDcHeartbeatSender.MonitoringDormant]
+        }
+      }
+
+      enterBarrier("sunny-weather-done")
+    }
+
+    "other node must become oldest when current DC-oldest Leaves" taggedAs LongRunningTest in {
+      val observer = TestProbe("alpha-observer-prime")
+
+      // we leave one of the current oldest nodes of the `alpha` DC,
+      // since it has 3 members the "not yet oldest" one becomes oldest and should start monitoring across datacenter
+      val preLeaveOldestAlphaRole = expectedAlphaHeartbeaterRoles.head
+      val preLeaveOldestAlphaAddress = expectedAlphaHeartbeaterNodes.find(_.address.port.get == preLeaveOldestAlphaRole.port.get).get.address
+      runOn(preLeaveOldestAlphaRole) {
+        info(s"Leaving: ${preLeaveOldestAlphaAddress}")
+        cluster.leave(cluster.selfAddress)
+      }
+
+      awaitMemberRemoved(preLeaveOldestAlphaAddress)
+      enterBarrier("wat")
+
+      // refresh our view about who is currently monitoring things in alpha:
+      refreshOldestMemberHeartbeatStatuses()
+
+      enterBarrier("after-alpha-monitoring-node-left")
+
+      implicit val sender = observer.ref
+      val expectedAlphaMonitoringNodesAfterLeaving = (takeNOldestMembers(_.dataCenter == "alpha", 3).filterNot(_.status == MemberStatus.Exiting))
+      runOn(membersAsRoles(expectedAlphaMonitoringNodesAfterLeaving).toList: _*) {
+        awaitAssert({
+
+          selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+
+          try {
+            observer.expectMsgType[CrossDcHeartbeatSender.MonitoringActive](5.seconds)
+            info(s"Got confirmation from ${observer.lastSender} that it is actively monitoring now")
+          } catch {
+            case ex: Throwable ⇒
+              throw new AssertionError(s"Monitoring was Dormant on ${cluster.selfAddress}, where we expected it to be active!", ex)
+          }
+        }, 20.seconds)
+      }
+      enterBarrier("confirmed-heartbeating-take-over")
+    }
+  }
+
+  /**
+   * INTERNAL API
+   * Returns `Up` (or in "later" status, like Leaving etc, but never `Joining` or `WeaklyUp`) members,
+   * sorted by Member.ageOrdering (from oldest to youngest). This restriction on status is needed to
+   * strongly guaratnee the order of "oldest" members, as they're linearized by the order in which they become Up
+   * (since marking that transition is a Leader action).
+   */
+  private def membersByAge(): immutable.SortedSet[Member] =
+    SortedSet.empty(Member.ageOrdering)
+      .union(cluster.state.members.filter(m ⇒ m.status != MemberStatus.Joining && m.status != MemberStatus.WeaklyUp))
+
+  /** INTERNAL API */
+  @InternalApi
+  private[cluster] def takeNOldestMembers(memberFilter: Member ⇒ Boolean, n: Int): immutable.SortedSet[Member] =
+    membersByAge()
+      .filter(memberFilter)
+      .take(n)
+
+  private def membersAsRoles(ms: SortedSet[Member]): SortedSet[RoleName] = {
+    val res = ms.flatMap(m ⇒ roleName(m.address))
+    require(res.size == ms.size, s"Not all members were converted to roles! Got: ${ms}, found ${res}")
+    res
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSplitBrainSpec.scala

@@ -0,0 +1,187 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import akka.cluster.ClusterEvent.{ CurrentClusterState, DataCenterReachabilityEvent, ReachableDataCenter, UnreachableDataCenter }
+import akka.remote.testconductor.RoleName
+import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec }
+import akka.remote.transport.ThrottlerTransportAdapter.Direction
+import akka.testkit.TestProbe
+import com.typesafe.config.ConfigFactory
+
+import scala.concurrent.duration._
+
+object MultiDcSplitBrainMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+
+  commonConfig(ConfigFactory.parseString(
+    """
+      akka.loglevel = DEBUG
+      akka.cluster.debug.verbose-heartbeat-logging = on
+      akka.remote.netty.tcp.connection-timeout = 5 s # speedup in case of connection issue
+      akka.remote.retry-gate-closed-for = 1 s
+      akka.cluster.multi-data-center {
+        failure-detector {
+          acceptable-heartbeat-pause = 4s
+          heartbeat-interval = 1s
+        }
+      }
+    """).withFallback(MultiNodeClusterSpec.clusterConfig))
+
+  nodeConfig(first, second)(ConfigFactory.parseString(
+    """
+      akka.cluster.multi-data-center.self-data-center = "dc1"
+    """))
+
+  nodeConfig(third, fourth)(ConfigFactory.parseString(
+    """
+      akka.cluster.multi-data-center.self-data-center = "dc2"
+    """))
+
+  testTransport(on = true)
+}
+
+class MultiDcSplitBrainMultiJvmNode1 extends MultiDcSplitBrainSpec
+class MultiDcSplitBrainMultiJvmNode2 extends MultiDcSplitBrainSpec
+class MultiDcSplitBrainMultiJvmNode3 extends MultiDcSplitBrainSpec
+class MultiDcSplitBrainMultiJvmNode4 extends MultiDcSplitBrainSpec
+
+abstract class MultiDcSplitBrainSpec
+  extends MultiNodeSpec(MultiDcSplitBrainMultiJvmSpec)
+  with MultiNodeClusterSpec {
+
+  import MultiDcSplitBrainMultiJvmSpec._
+
+  val dc1 = List(first, second)
+  val dc2 = List(third, fourth)
+  var barrierCounter = 0
+
+  def splitDataCenters(notMembers: Set[RoleName]): Unit = {
+    val memberNodes = (dc1 ++ dc2).filterNot(notMembers)
+    val probe = TestProbe()
+    runOn(memberNodes: _*) {
+      cluster.subscribe(probe.ref, classOf[DataCenterReachabilityEvent])
+      probe.expectMsgType[CurrentClusterState]
+    }
+    enterBarrier(s"split-$barrierCounter")
+    barrierCounter += 1
+
+    runOn(first) {
+      for (dc1Node ← dc1; dc2Node ← dc2) {
+        testConductor.blackhole(dc1Node, dc2Node, Direction.Both).await
+      }
+    }
+
+    enterBarrier(s"after-split-$barrierCounter")
+    barrierCounter += 1
+
+    runOn(memberNodes: _*) {
+      probe.expectMsgType[UnreachableDataCenter](15.seconds)
+      cluster.unsubscribe(probe.ref)
+      runOn(dc1: _*) {
+        awaitAssert {
+          cluster.state.unreachableDataCenters should ===(Set("dc2"))
+        }
+      }
+      runOn(dc2: _*) {
+        awaitAssert {
+          cluster.state.unreachableDataCenters should ===(Set("dc1"))
+        }
+      }
+      cluster.state.unreachable should ===(Set.empty)
+    }
+    enterBarrier(s"after-split-verified-$barrierCounter")
+    barrierCounter += 1
+  }
+
+  def unsplitDataCenters(notMembers: Set[RoleName]): Unit = {
+    val memberNodes = (dc1 ++ dc2).filterNot(notMembers)
+    val probe = TestProbe()
+    runOn(memberNodes: _*) {
+      cluster.subscribe(probe.ref, classOf[ReachableDataCenter])
+      probe.expectMsgType[CurrentClusterState]
+    }
+    enterBarrier(s"unsplit-$barrierCounter")
+    barrierCounter += 1
+
+    runOn(first) {
+      for (dc1Node ← dc1; dc2Node ← dc2) {
+        testConductor.passThrough(dc1Node, dc2Node, Direction.Both).await
+      }
+    }
+
+    enterBarrier(s"after-unsplit-$barrierCounter")
+    barrierCounter += 1
+
+    runOn(memberNodes: _*) {
+      probe.expectMsgType[ReachableDataCenter](25.seconds)
+      cluster.unsubscribe(probe.ref)
+      awaitAssert {
+        cluster.state.unreachableDataCenters should ===(Set.empty)
+      }
+    }
+    enterBarrier(s"after-unsplit-verified-$barrierCounter")
+    barrierCounter += 1
+
+  }
+
+  "A cluster with multiple data centers" must {
+    "be able to form two data centers" in {
+      awaitClusterUp(first, second, third)
+    }
+
+    "be able to have a data center member join while there is inter data center split" in within(20.seconds) {
+      // introduce a split between data centers
+      splitDataCenters(notMembers = Set(fourth))
+
+      runOn(fourth) {
+        cluster.join(third)
+      }
+      enterBarrier("inter-data-center unreachability")
+
+      // should be able to join and become up since the
+      // split is between dc1 and dc2
+      runOn(third, fourth) {
+        awaitAssert(clusterView.members.collect {
+          case m if m.dataCenter == "dc2" && m.status == MemberStatus.Up ⇒ m.address
+        } should ===(Set(address(third), address(fourth))))
+      }
+      enterBarrier("dc2-join-completed")
+
+      unsplitDataCenters(notMembers = Set.empty)
+
+      runOn(dc1: _*) {
+        awaitAssert(clusterView.members.collect {
+          case m if m.dataCenter == "dc2" && m.status == MemberStatus.Up ⇒ m.address
+        } should ===(Set(address(third), address(fourth))))
+      }
+
+      enterBarrier("inter-data-center-split-1-done")
+    }
+
+    "be able to have data center member leave while there is inter data center split" in within(20.seconds) {
+      splitDataCenters(notMembers = Set.empty)
+
+      runOn(fourth) {
+        cluster.leave(fourth)
+      }
+
+      runOn(third) {
+        awaitAssert(clusterView.members.filter(_.address == address(fourth)) should ===(Set.empty))
+      }
+      enterBarrier("node-4-left")
+
+      unsplitDataCenters(notMembers = Set(fourth))
+
+      runOn(first, second) {
+        awaitAssert(clusterView.members.filter(_.address == address(fourth)) should ===(Set.empty))
+      }
+      enterBarrier("inter-data-center-split-2-done")
+    }
+
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiDcSunnyWeatherSpec.scala

@@ -0,0 +1,166 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import akka.annotation.InternalApi
+import akka.remote.testconductor.RoleName
+import akka.remote.testkit.{ MultiNodeConfig, MultiNodeSpec }
+import akka.testkit._
+import com.typesafe.config.ConfigFactory
+
+import scala.collection.immutable
+import scala.collection.immutable.SortedSet
+import scala.concurrent.duration._
+
+object MultiDcSunnyWeatherMultiJvmSpec extends MultiNodeConfig {
+  val first = role("first")
+  val second = role("second")
+  val third = role("third")
+  val fourth = role("fourth")
+  val fifth = role("fifth")
+
+  nodeConfig(first, second, third)(ConfigFactory.parseString(
+    """
+    akka {
+      cluster.multi-data-center.self-data-center = alpha
+    }
+    """))
+
+  nodeConfig(fourth, fifth)(ConfigFactory.parseString(
+    """
+    akka {
+      cluster.multi-data-center.self-data-center = beta
+    }
+    """))
+
+  commonConfig(ConfigFactory.parseString(
+    """
+    akka {
+      actor.provider = cluster
+
+      loggers = ["akka.testkit.TestEventListener"]
+      loglevel = INFO
+
+      remote.log-remote-lifecycle-events = off
+
+      cluster {
+        debug.verbose-heartbeat-logging = off
+
+        multi-data-center {
+          cross-data-center-connections = 2
+        }
+      }
+    }
+    """))
+
+}
+
+class MultiDcSunnyWeatherMultiJvmNode1 extends MultiDcSunnyWeatherSpec
+class MultiDcSunnyWeatherMultiJvmNode2 extends MultiDcSunnyWeatherSpec
+class MultiDcSunnyWeatherMultiJvmNode3 extends MultiDcSunnyWeatherSpec
+class MultiDcSunnyWeatherMultiJvmNode4 extends MultiDcSunnyWeatherSpec
+class MultiDcSunnyWeatherMultiJvmNode5 extends MultiDcSunnyWeatherSpec
+
+abstract class MultiDcSunnyWeatherSpec extends MultiNodeSpec(MultiDcSunnyWeatherMultiJvmSpec)
+  with MultiNodeClusterSpec {
+
+  "A normal cluster" must {
+    "be healthy" taggedAs LongRunningTest in {
+
+      val observer = TestProbe("alpha-observer")
+
+      // allow all nodes to join:
+      awaitClusterUp(roles: _*)
+
+      val crossDcHeartbeatSenderPath = "/system/cluster/core/daemon/crossDcHeartbeatSender"
+      val selectCrossDcHeartbeatSender = system.actorSelection(crossDcHeartbeatSenderPath)
+
+      val expectedAlphaHeartbeaterNodes = takeNOldestMembers(_.dataCenter == "alpha", 2)
+      val expectedAlphaHeartbeaterRoles = membersAsRoles(expectedAlphaHeartbeaterNodes)
+
+      val expectedBetaHeartbeaterNodes = takeNOldestMembers(_.dataCenter == "beta", 2)
+      val expectedBetaHeartbeaterRoles = membersAsRoles(expectedBetaHeartbeaterNodes)
+
+      val expectedNoActiveHeartbeatSenderRoles = roles.toSet -- (expectedAlphaHeartbeaterRoles union expectedBetaHeartbeaterRoles)
+
+      enterBarrier("found-expectations")
+
+      info(s"expectedAlphaHeartbeaterNodes = ${expectedAlphaHeartbeaterNodes.map(_.address.port.get)}")
+      info(s"expectedBetaHeartbeaterNodes = ${expectedBetaHeartbeaterNodes.map(_.address.port.get)}")
+      info(s"expectedNoActiveHeartbeatSenderRoles = ${expectedNoActiveHeartbeatSenderRoles.map(_.port.get)}")
+
+      expectedAlphaHeartbeaterRoles.size should ===(2)
+      expectedBetaHeartbeaterRoles.size should ===(2)
+
+      implicit val sender = observer.ref
+      runOn(expectedAlphaHeartbeaterRoles.toList: _*) {
+        selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+        val status = observer.expectMsgType[CrossDcHeartbeatSender.MonitoringActive](5.seconds)
+      }
+      runOn(expectedBetaHeartbeaterRoles.toList: _*) {
+        selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+        val status = observer.expectMsgType[CrossDcHeartbeatSender.MonitoringActive](5.seconds)
+      }
+      runOn(expectedNoActiveHeartbeatSenderRoles.toList: _*) {
+        selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+        val status = observer.expectMsgType[CrossDcHeartbeatSender.MonitoringDormant](5.seconds)
+      }
+
+      enterBarrier("done")
+    }
+
+    "never heartbeat to itself or members of same its own data center" taggedAs LongRunningTest in {
+
+      val observer = TestProbe("alpha-observer")
+
+      val crossDcHeartbeatSenderPath = "/system/cluster/core/daemon/crossDcHeartbeatSender"
+      val selectCrossDcHeartbeatSender = system.actorSelection(crossDcHeartbeatSenderPath)
+
+      enterBarrier("checking-activeReceivers")
+
+      implicit val sender = observer.ref
+      selectCrossDcHeartbeatSender ! CrossDcHeartbeatSender.ReportStatus()
+      observer.expectMsgType[CrossDcHeartbeatSender.MonitoringStateReport](5.seconds) match {
+        case CrossDcHeartbeatSender.MonitoringDormant() ⇒ // ok ...
+        case CrossDcHeartbeatSender.MonitoringActive(state) ⇒
+
+          // must not heartbeat myself
+          state.activeReceivers should not contain cluster.selfUniqueAddress
+
+          // not any of the members in the same datacenter; it's "cross-dc" after all
+          val myDataCenterMembers = state.state.getOrElse(cluster.selfDataCenter, Set.empty)
+          myDataCenterMembers foreach { myDcMember ⇒
+            state.activeReceivers should not contain myDcMember.uniqueAddress
+          }
+
+      }
+
+      enterBarrier("done-checking-activeReceivers")
+    }
+  }
+
+  /**
+   * INTERNAL API
+   * Returns `Up` (or in "later" status, like Leaving etc, but never `Joining` or `WeaklyUp`) members,
+   * sorted by Member.ageOrdering (from oldest to youngest). This restriction on status is needed to
+   * strongly guaratnee the order of "oldest" members, as they're linearized by the order in which they become Up
+   * (since marking that transition is a Leader action).
+   */
+  private def membersByAge(): immutable.SortedSet[Member] =
+    SortedSet.empty(Member.ageOrdering)
+      .union(cluster.state.members.filter(m ⇒ m.status != MemberStatus.Joining && m.status != MemberStatus.WeaklyUp))
+
+  /** INTERNAL API */
+  @InternalApi
+  private[cluster] def takeNOldestMembers(memberFilter: Member ⇒ Boolean, n: Int): immutable.SortedSet[Member] =
+    membersByAge()
+      .filter(memberFilter)
+      .take(n)
+
+  private def membersAsRoles(ms: immutable.Set[Member]): immutable.Set[RoleName] = {
+    val res = ms.flatMap(m ⇒ roleName(m.address))
+    require(res.size == ms.size, s"Not all members were converted to roles! Got: ${ms}, found ${res}")
+    res
+  }
+}

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

@@ -14,7 +14,7 @@ import akka.remote.testconductor.RoleName
 import akka.remote.testkit.{ FlightRecordingSupport, MultiNodeSpec, STMultiNodeSpec }
 import akka.testkit._
 import akka.testkit.TestEvent._
-import akka.actor.{ ActorSystem, Address }
+import akka.actor.{ Actor, ActorRef, ActorSystem, Address, Deploy, PoisonPill, Props, RootActorPath }
 import akka.event.Logging.ErrorLevel
 
 import scala.concurrent.duration._
@@ -22,9 +22,9 @@ import scala.collection.immutable
 import java.util.concurrent.ConcurrentHashMap
 
 import akka.remote.DefaultFailureDetectorRegistry
-import akka.actor.ActorRef
-import akka.actor.Actor
-import akka.actor.RootActorPath
+import akka.cluster.ClusterEvent.{ CurrentClusterState, MemberEvent, MemberExited, MemberRemoved }
+
+import scala.concurrent.Await
 
 object MultiNodeClusterSpec {
 
@@ -305,11 +305,53 @@ trait MultiNodeClusterSpec extends Suite with STMultiNodeSpec with WatchedByCoro
       awaitAssert(clusterView.members.size should ===(numberOfMembers))
       awaitAssert(clusterView.members.map(_.status) should ===(Set(MemberStatus.Up)))
       // clusterView.leader is updated by LeaderChanged, await that to be updated also
-      val expectedLeader = clusterView.members.headOption.map(_.address)
+      val expectedLeader = clusterView.members.collectFirst {
+        case m if m.dataCenter == cluster.settings.SelfDataCenter ⇒ m.address
+      }
       awaitAssert(clusterView.leader should ===(expectedLeader))
     }
   }
 
+  def awaitMemberRemoved(toBeRemovedAddress: Address, timeout: FiniteDuration = 25.seconds): Unit = within(timeout) {
+    if (toBeRemovedAddress == cluster.selfAddress) {
+      enterBarrier("registered-listener")
+
+      cluster.leave(toBeRemovedAddress)
+      enterBarrier("member-left")
+
+      awaitCond(cluster.isTerminated, remaining)
+      enterBarrier("member-shutdown")
+    } else {
+      val exitingLatch = TestLatch()
+
+      val awaiter = system.actorOf(Props(new Actor {
+        def receive = {
+          case MemberRemoved(m, _) if m.address == toBeRemovedAddress ⇒
+            exitingLatch.countDown()
+          case _ ⇒
+          // ignore
+        }
+      }).withDeploy(Deploy.local))
+      cluster.subscribe(awaiter, classOf[MemberEvent])
+      enterBarrier("registered-listener")
+
+      // in the meantime member issues leave
+      enterBarrier("member-left")
+
+      // verify that the member is EXITING
+      try Await.result(exitingLatch, timeout) catch {
+        case cause: Exception ⇒
+          throw new AssertionError(s"Member ${toBeRemovedAddress} was not removed within ${timeout}!", cause)
+      }
+      awaiter ! PoisonPill // you've done your job, now die 
+
+      enterBarrier("member-shutdown")
+      markNodeAsUnavailable(toBeRemovedAddress)
+    }
+
+    enterBarrier("member-totally-shutdown")
+  }
+
   def awaitAllReachable(): Unit =
     awaitAssert(clusterView.unreachableMembers should ===(Set.empty))
 

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeChurnSpec.scala

@@ -23,6 +23,7 @@ object NodeChurnMultiJvmSpec extends MultiNodeConfig {
   commonConfig(debugConfig(on = false).
     withFallback(ConfigFactory.parseString("""
       akka.cluster.auto-down-unreachable-after = 1s
+      akka.cluster.prune-gossip-tombstones-after = 1s
       akka.remote.log-frame-size-exceeding = 1200b
       akka.remote.artery.advanced {
         idle-cpu-level = 1

akka-cluster/src/multi-jvm/scala/akka/cluster/QuickRestartSpec.scala

@@ -91,7 +91,7 @@ abstract class QuickRestartSpec
             Cluster(system).state.members.size should ===(totalNumberOfNodes)
             Cluster(system).state.members.map(_.status == MemberStatus.Up)
             // use the role to test that it is the new incarnation that joined, sneaky
-            Cluster(system).state.members.flatMap(_.roles) should ===(Set(s"round-$n"))
+            Cluster(system).state.members.flatMap(_.roles) should ===(Set(s"round-$n", ClusterSettings.DcRolePrefix + "default"))
           }
         }
         enterBarrier("members-up-" + n)

akka-cluster/src/test/scala/akka/cluster/ClusterConfigSpec.scala

@@ -5,9 +5,13 @@
 package akka.cluster
 
 import language.postfixOps
+import scala.concurrent.duration._
+
+import com.typesafe.config.ConfigFactory
+
 import akka.testkit.AkkaSpec
 import akka.dispatch.Dispatchers
-import scala.concurrent.duration._
+
 import akka.remote.PhiAccrualFailureDetector
 import akka.util.Helpers.ConfigOps
 import akka.actor.Address
@@ -41,7 +45,8 @@ class ClusterConfigSpec extends AkkaSpec {
       DownRemovalMargin should ===(Duration.Zero)
       MinNrOfMembers should ===(1)
       MinNrOfMembersOfRole should ===(Map.empty[String, Int])
-      Roles should ===(Set.empty[String])
+      SelfDataCenter should ===("default")
+      Roles should ===(Set(ClusterSettings.DcRolePrefix + "default"))
       JmxEnabled should ===(true)
       UseDispatcher should ===(Dispatchers.DefaultDispatcherId)
       GossipDifferentViewProbability should ===(0.8 +- 0.0001)
@@ -49,5 +54,20 @@ class ClusterConfigSpec extends AkkaSpec {
       SchedulerTickDuration should ===(33 millis)
       SchedulerTicksPerWheel should ===(512)
     }
+
+    "be able to parse non-default cluster config elements" in {
+      val settings = new ClusterSettings(ConfigFactory.parseString(
+        """
+          |akka {
+          |  cluster {
+          |    roles = [ "hamlet" ]
+          |    multi-data-center.self-data-center = "blue"
+          |  }
+          |}
+        """.stripMargin).withFallback(ConfigFactory.load()), system.name)
+      import settings._
+      Roles should ===(Set("hamlet", ClusterSettings.DcRolePrefix + "blue"))
+      SelfDataCenter should ===("blue")
+    }
   }
 }

akka-cluster/src/test/scala/akka/cluster/ClusterDomainEventPublisherSpec.scala

@@ -19,6 +19,7 @@ import akka.testkit.ImplicitSender
 import akka.actor.ActorRef
 import akka.remote.RARP
 import akka.testkit.TestProbe
+import akka.cluster.ClusterSettings.{ DataCenter, DefaultDataCenter }
 
 object ClusterDomainEventPublisherSpec {
   val config = """
@@ -36,6 +37,9 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
     else "akka.tcp"
 
   var publisher: ActorRef = _
+
+  final val OtherDataCenter = "dc2"
+
   val aUp = TestMember(Address(protocol, "sys", "a", 2552), Up)
   val aLeaving = aUp.copy(status = Leaving)
   val aExiting = aLeaving.copy(status = Exiting)
@@ -47,17 +51,38 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
   val cRemoved = cUp.copy(status = Removed)
   val a51Up = TestMember(Address(protocol, "sys", "a", 2551), Up)
   val dUp = TestMember(Address(protocol, "sys", "d", 2552), Up, Set("GRP"))
+  val eUp = TestMember(Address(protocol, "sys", "e", 2552), Up, Set("GRP"), OtherDataCenter)
+
+  private def state(gossip: Gossip, self: UniqueAddress, dc: DataCenter) =
+    MembershipState(gossip, self, DefaultDataCenter, crossDcConnections = 5)
+
+  val emptyMembershipState = state(Gossip.empty, aUp.uniqueAddress, DefaultDataCenter)
 
   val g0 = Gossip(members = SortedSet(aUp)).seen(aUp.uniqueAddress)
+  val state0 = state(g0, aUp.uniqueAddress, DefaultDataCenter)
   val g1 = Gossip(members = SortedSet(aUp, cJoining)).seen(aUp.uniqueAddress).seen(cJoining.uniqueAddress)
+  val state1 = state(g1, aUp.uniqueAddress, DefaultDataCenter)
   val g2 = Gossip(members = SortedSet(aUp, bExiting, cUp)).seen(aUp.uniqueAddress)
+  val state2 = state(g2, aUp.uniqueAddress, DefaultDataCenter)
   val g3 = g2.seen(bExiting.uniqueAddress).seen(cUp.uniqueAddress)
+  val state3 = state(g3, aUp.uniqueAddress, DefaultDataCenter)
   val g4 = Gossip(members = SortedSet(a51Up, aUp, bExiting, cUp)).seen(aUp.uniqueAddress)
+  val state4 = state(g4, aUp.uniqueAddress, DefaultDataCenter)
   val g5 = Gossip(members = SortedSet(a51Up, aUp, bExiting, cUp)).seen(aUp.uniqueAddress).seen(bExiting.uniqueAddress).seen(cUp.uniqueAddress).seen(a51Up.uniqueAddress)
+  val state5 = state(g5, aUp.uniqueAddress, DefaultDataCenter)
   val g6 = Gossip(members = SortedSet(aLeaving, bExiting, cUp)).seen(aUp.uniqueAddress)
+  val state6 = state(g6, aUp.uniqueAddress, DefaultDataCenter)
   val g7 = Gossip(members = SortedSet(aExiting, bExiting, cUp)).seen(aUp.uniqueAddress)
+  val state7 = state(g7, aUp.uniqueAddress, DefaultDataCenter)
   val g8 = Gossip(members = SortedSet(aUp, bExiting, cUp, dUp), overview = GossipOverview(reachability =
     Reachability.empty.unreachable(aUp.uniqueAddress, dUp.uniqueAddress))).seen(aUp.uniqueAddress)
+  val state8 = state(g8, aUp.uniqueAddress, DefaultDataCenter)
+  val g9 = Gossip(members = SortedSet(aUp, bExiting, cUp, dUp, eUp), overview = GossipOverview(reachability =
+    Reachability.empty.unreachable(aUp.uniqueAddress, eUp.uniqueAddress)))
+  val state9 = state(g9, aUp.uniqueAddress, DefaultDataCenter)
+  val g10 = Gossip(members = SortedSet(aUp, bExiting, cUp, dUp, eUp), overview = GossipOverview(reachability =
+    Reachability.empty))
+  val state10 = state(g10, aUp.uniqueAddress, DefaultDataCenter)
 
   // created in beforeEach
   var memberSubscriber: TestProbe = _
@@ -69,7 +94,7 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
     system.eventStream.subscribe(memberSubscriber.ref, ClusterShuttingDown.getClass)
 
     publisher = system.actorOf(Props[ClusterDomainEventPublisher])
-    publisher ! PublishChanges(g0)
+    publisher ! PublishChanges(state0)
     memberSubscriber.expectMsg(MemberUp(aUp))
     memberSubscriber.expectMsg(LeaderChanged(Some(aUp.address)))
   }
@@ -77,19 +102,19 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
   "ClusterDomainEventPublisher" must {
 
     "publish MemberJoined" in {
-      publisher ! PublishChanges(g1)
+      publisher ! PublishChanges(state1)
       memberSubscriber.expectMsg(MemberJoined(cJoining))
     }
 
     "publish MemberUp" in {
-      publisher ! PublishChanges(g2)
-      publisher ! PublishChanges(g3)
+      publisher ! PublishChanges(state2)
+      publisher ! PublishChanges(state3)
       memberSubscriber.expectMsg(MemberExited(bExiting))
       memberSubscriber.expectMsg(MemberUp(cUp))
     }
 
     "publish leader changed" in {
-      publisher ! PublishChanges(g4)
+      publisher ! PublishChanges(state4)
       memberSubscriber.expectMsg(MemberUp(a51Up))
       memberSubscriber.expectMsg(MemberExited(bExiting))
       memberSubscriber.expectMsg(MemberUp(cUp))
@@ -98,17 +123,17 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
     }
 
     "publish leader changed when old leader leaves and is removed" in {
-      publisher ! PublishChanges(g3)
+      publisher ! PublishChanges(state3)
       memberSubscriber.expectMsg(MemberExited(bExiting))
       memberSubscriber.expectMsg(MemberUp(cUp))
-      publisher ! PublishChanges(g6)
+      publisher ! PublishChanges(state6)
       memberSubscriber.expectMsg(MemberLeft(aLeaving))
-      publisher ! PublishChanges(g7)
+      publisher ! PublishChanges(state7)
       memberSubscriber.expectMsg(MemberExited(aExiting))
       memberSubscriber.expectMsg(LeaderChanged(Some(cUp.address)))
       memberSubscriber.expectNoMsg(500 millis)
       // at the removed member a an empty gossip is the last thing
-      publisher ! PublishChanges(Gossip.empty)
+      publisher ! PublishChanges(emptyMembershipState)
       memberSubscriber.expectMsg(MemberRemoved(aRemoved, Exiting))
       memberSubscriber.expectMsg(MemberRemoved(bRemoved, Exiting))
       memberSubscriber.expectMsg(MemberRemoved(cRemoved, Up))
@@ -116,13 +141,13 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
     }
 
     "not publish leader changed when same leader" in {
-      publisher ! PublishChanges(g4)
+      publisher ! PublishChanges(state4)
       memberSubscriber.expectMsg(MemberUp(a51Up))
       memberSubscriber.expectMsg(MemberExited(bExiting))
       memberSubscriber.expectMsg(MemberUp(cUp))
       memberSubscriber.expectMsg(LeaderChanged(Some(a51Up.address)))
 
-      publisher ! PublishChanges(g5)
+      publisher ! PublishChanges(state5)
       memberSubscriber.expectNoMsg(500 millis)
     }
 
@@ -130,9 +155,11 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
       val subscriber = TestProbe()
       publisher ! Subscribe(subscriber.ref, InitialStateAsSnapshot, Set(classOf[RoleLeaderChanged]))
       subscriber.expectMsgType[CurrentClusterState]
-      publisher ! PublishChanges(Gossip(members = SortedSet(cJoining, dUp)))
-      subscriber.expectMsg(RoleLeaderChanged("GRP", Some(dUp.address)))
-      publisher ! PublishChanges(Gossip(members = SortedSet(cUp, dUp)))
+      publisher ! PublishChanges(state(Gossip(members = SortedSet(cJoining, dUp)), dUp.uniqueAddress, DefaultDataCenter))
+      subscriber.expectMsgAllOf(
+        RoleLeaderChanged("GRP", Some(dUp.address)),
+        RoleLeaderChanged(ClusterSettings.DcRolePrefix + ClusterSettings.DefaultDataCenter, Some(dUp.address)))
+      publisher ! PublishChanges(state(Gossip(members = SortedSet(cUp, dUp)), dUp.uniqueAddress, DefaultDataCenter))
       subscriber.expectMsg(RoleLeaderChanged("GRP", Some(cUp.address)))
     }
 
@@ -142,24 +169,34 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
       subscriber.expectMsgType[CurrentClusterState]
       // but only to the new subscriber
       memberSubscriber.expectNoMsg(500 millis)
-
     }
 
     "send events corresponding to current state when subscribe" in {
       val subscriber = TestProbe()
-      publisher ! PublishChanges(g8)
+      publisher ! PublishChanges(state8)
       publisher ! Subscribe(subscriber.ref, InitialStateAsEvents, Set(classOf[MemberEvent], classOf[ReachabilityEvent]))
       subscriber.receiveN(4).toSet should be(Set(MemberUp(aUp), MemberUp(cUp), MemberUp(dUp), MemberExited(bExiting)))
       subscriber.expectMsg(UnreachableMember(dUp))
       subscriber.expectNoMsg(500 millis)
     }
 
+    "send datacenter reachability events" in {
+      val subscriber = TestProbe()
+      publisher ! PublishChanges(state9)
+      publisher ! Subscribe(subscriber.ref, InitialStateAsEvents, Set(classOf[DataCenterReachabilityEvent]))
+      subscriber.expectMsg(UnreachableDataCenter(OtherDataCenter))
+      subscriber.expectNoMsg(500 millis)
+      publisher ! PublishChanges(state10)
+      subscriber.expectMsg(ReachableDataCenter(OtherDataCenter))
+      subscriber.expectNoMsg(500 millis)
+    }
+
     "support unsubscribe" in {
       val subscriber = TestProbe()
       publisher ! Subscribe(subscriber.ref, InitialStateAsSnapshot, Set(classOf[MemberEvent]))
       subscriber.expectMsgType[CurrentClusterState]
       publisher ! Unsubscribe(subscriber.ref, Some(classOf[MemberEvent]))
-      publisher ! PublishChanges(g3)
+      publisher ! PublishChanges(state3)
       subscriber.expectNoMsg(500 millis)
       // but memberSubscriber is still subscriber
       memberSubscriber.expectMsg(MemberExited(bExiting))
@@ -170,10 +207,10 @@ class ClusterDomainEventPublisherSpec extends AkkaSpec(ClusterDomainEventPublish
       val subscriber = TestProbe()
       publisher ! Subscribe(subscriber.ref, InitialStateAsSnapshot, Set(classOf[SeenChanged]))
       subscriber.expectMsgType[CurrentClusterState]
-      publisher ! PublishChanges(g2)
+      publisher ! PublishChanges(state2)
       subscriber.expectMsgType[SeenChanged]
       subscriber.expectNoMsg(500 millis)
-      publisher ! PublishChanges(g3)
+      publisher ! PublishChanges(state3)
       subscriber.expectMsgType[SeenChanged]
       subscriber.expectNoMsg(500 millis)
     }

akka-cluster/src/test/scala/akka/cluster/ClusterDomainEventSpec.scala

@@ -7,6 +7,7 @@ package akka.cluster
 import org.scalatest.WordSpec
 import org.scalatest.Matchers
 import akka.actor.Address
+
 import scala.collection.immutable.SortedSet
 
 class ClusterDomainEventSpec extends WordSpec with Matchers {
@@ -38,30 +39,36 @@ class ClusterDomainEventSpec extends WordSpec with Matchers {
   private[cluster] def converge(gossip: Gossip): (Gossip, Set[UniqueAddress]) =
     ((gossip, Set.empty[UniqueAddress]) /: gossip.members) { case ((gs, as), m) ⇒ (gs.seen(m.uniqueAddress), as + m.uniqueAddress) }
 
+  private def state(g: Gossip): MembershipState =
+    state(g, selfDummyAddress)
+
+  private def state(g: Gossip, self: UniqueAddress): MembershipState =
+    MembershipState(g, self, ClusterSettings.DefaultDataCenter, crossDcConnections = 5)
+
   "Domain events" must {
 
     "be empty for the same gossip" in {
       val g1 = Gossip(members = SortedSet(aUp))
 
-      diffUnreachable(g1, g1, selfDummyAddress) should ===(Seq.empty)
+      diffUnreachable(state(g1), state(g1)) should ===(Seq.empty)
     }
 
     "be produced for new members" in {
       val (g1, _) = converge(Gossip(members = SortedSet(aUp)))
       val (g2, s2) = converge(Gossip(members = SortedSet(aUp, bUp, eJoining)))
 
-      diffMemberEvents(g1, g2) should ===(Seq(MemberUp(bUp), MemberJoined(eJoining)))
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
+      diffMemberEvents(state(g1), state(g2)) should ===(Seq(MemberUp(bUp), MemberJoined(eJoining)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq.empty)
+      diffSeen(state(g1), state(g2)) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
     }
 
     "be produced for changed status of members" in {
       val (g1, _) = converge(Gossip(members = SortedSet(aJoining, bUp, cUp)))
       val (g2, s2) = converge(Gossip(members = SortedSet(aUp, bUp, cLeaving, eJoining)))
 
-      diffMemberEvents(g1, g2) should ===(Seq(MemberUp(aUp), MemberLeft(cLeaving), MemberJoined(eJoining)))
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
+      diffMemberEvents(state(g1), state(g2)) should ===(Seq(MemberUp(aUp), MemberLeft(cLeaving), MemberJoined(eJoining)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq.empty)
+      diffSeen(state(g1), state(g2)) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
     }
 
     "be produced for members in unreachable" in {
@@ -73,10 +80,72 @@ class ClusterDomainEventSpec extends WordSpec with Matchers {
         unreachable(aUp.uniqueAddress, bDown.uniqueAddress)
       val g2 = Gossip(members = SortedSet(aUp, cUp, bDown, eDown), overview = GossipOverview(reachability = reachability2))
 
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq(UnreachableMember(bDown)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq(UnreachableMember(bDown)))
       // never include self member in unreachable
-      diffUnreachable(g1, g2, bDown.uniqueAddress) should ===(Seq())
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq.empty)
+
+      diffUnreachable(
+        state(g1, bDown.uniqueAddress),
+        state(g2, bDown.uniqueAddress)) should ===(Seq())
+      diffSeen(state(g1), state(g2)) should ===(Seq.empty)
+    }
+
+    "be produced for reachability observations between data centers" in {
+      val dc2AMemberUp = TestMember(Address("akka.tcp", "sys", "dc2A", 2552), Up, Set.empty, "dc2")
+      val dc2AMemberDown = TestMember(Address("akka.tcp", "sys", "dc2A", 2552), Down, Set.empty, "dc2")
+      val dc2BMemberUp = TestMember(Address("akka.tcp", "sys", "dc2B", 2552), Up, Set.empty, "dc2")
+
+      val dc3AMemberUp = TestMember(Address("akka.tcp", "sys", "dc3A", 2552), Up, Set.empty, "dc3")
+      val dc3BMemberUp = TestMember(Address("akka.tcp", "sys", "dc3B", 2552), Up, Set.empty, "dc3")
+
+      val reachability1 = Reachability.empty
+      val g1 = Gossip(members = SortedSet(aUp, bUp, dc2AMemberUp, dc2BMemberUp, dc3AMemberUp, dc3BMemberUp), overview = GossipOverview(reachability = reachability1))
+
+      val reachability2 = reachability1
+        .unreachable(aUp.uniqueAddress, dc2AMemberDown.uniqueAddress)
+        .unreachable(dc2BMemberUp.uniqueAddress, dc2AMemberDown.uniqueAddress)
+      val g2 = Gossip(members = SortedSet(aUp, bUp, dc2AMemberDown, dc2BMemberUp, dc3AMemberUp, dc3BMemberUp), overview = GossipOverview(reachability = reachability2))
+
+      Set(aUp, bUp, dc2AMemberUp, dc2BMemberUp, dc3AMemberUp, dc3BMemberUp).foreach { member ⇒
+        val otherDc =
+          if (member.dataCenter == ClusterSettings.DefaultDataCenter) Seq("dc2")
+          else Seq()
+
+        diffUnreachableDataCenter(
+          MembershipState(g1, member.uniqueAddress, member.dataCenter, crossDcConnections = 5),
+          MembershipState(g2, member.uniqueAddress, member.dataCenter, crossDcConnections = 5)) should ===(otherDc.map(UnreachableDataCenter))
+
+        diffReachableDataCenter(
+          MembershipState(g2, member.uniqueAddress, member.dataCenter, crossDcConnections = 5),
+          MembershipState(g1, member.uniqueAddress, member.dataCenter, crossDcConnections = 5)) should ===(otherDc.map(ReachableDataCenter))
+      }
+    }
+
+    "not be produced for same reachability observations between data centers" in {
+      val dc2AMemberUp = TestMember(Address("akka.tcp", "sys", "dc2A", 2552), Up, Set.empty, "dc2")
+      val dc2AMemberDown = TestMember(Address("akka.tcp", "sys", "dc2A", 2552), Down, Set.empty, "dc2")
+
+      val reachability1 = Reachability.empty
+      val g1 = Gossip(members = SortedSet(aUp, dc2AMemberUp), overview = GossipOverview(reachability = reachability1))
+
+      val reachability2 = reachability1
+        .unreachable(aUp.uniqueAddress, dc2AMemberDown.uniqueAddress)
+      val g2 = Gossip(members = SortedSet(aUp, dc2AMemberDown), overview = GossipOverview(reachability = reachability2))
+
+      diffUnreachableDataCenter(
+        MembershipState(g1, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5),
+        MembershipState(g1, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5)) should ===(Seq())
+
+      diffUnreachableDataCenter(
+        MembershipState(g2, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5),
+        MembershipState(g2, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5)) should ===(Seq())
+
+      diffReachableDataCenter(
+        MembershipState(g1, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5),
+        MembershipState(g1, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5)) should ===(Seq())
+
+      diffReachableDataCenter(
+        MembershipState(g2, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5),
+        MembershipState(g2, aUp.uniqueAddress, aUp.dataCenter, crossDcConnections = 5)) should ===(Seq())
     }
 
     "be produced for members becoming reachable after unreachable" in {
@@ -90,62 +159,81 @@ class ClusterDomainEventSpec extends WordSpec with Matchers {
         reachable(aUp.uniqueAddress, bUp.uniqueAddress)
       val g2 = Gossip(members = SortedSet(aUp, cUp, bUp, eUp), overview = GossipOverview(reachability = reachability2))
 
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq(UnreachableMember(cUp)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq(UnreachableMember(cUp)))
       // never include self member in unreachable
-      diffUnreachable(g1, g2, cUp.uniqueAddress) should ===(Seq())
-      diffReachable(g1, g2, selfDummyAddress) should ===(Seq(ReachableMember(bUp)))
+      diffUnreachable(
+        state(g1, cUp.uniqueAddress),
+        state(g2, cUp.uniqueAddress)) should ===(Seq())
+      diffReachable(state(g1), state(g2)) should ===(Seq(ReachableMember(bUp)))
       // never include self member in reachable
-      diffReachable(g1, g2, bUp.uniqueAddress) should ===(Seq())
+      diffReachable(
+        state(g1, bUp.uniqueAddress),
+        state(g2, bUp.uniqueAddress)) should ===(Seq())
     }
 
     "be produced for removed members" in {
       val (g1, _) = converge(Gossip(members = SortedSet(aUp, dExiting)))
       val (g2, s2) = converge(Gossip(members = SortedSet(aUp)))
 
-      diffMemberEvents(g1, g2) should ===(Seq(MemberRemoved(dRemoved, Exiting)))
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
+      diffMemberEvents(state(g1), state(g2)) should ===(Seq(MemberRemoved(dRemoved, Exiting)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq.empty)
+      diffSeen(state(g1), state(g2)) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
     }
 
     "be produced for convergence changes" in {
       val g1 = Gossip(members = SortedSet(aUp, bUp, eJoining)).seen(aUp.uniqueAddress).seen(bUp.uniqueAddress).seen(eJoining.uniqueAddress)
       val g2 = Gossip(members = SortedSet(aUp, bUp, eJoining)).seen(aUp.uniqueAddress).seen(bUp.uniqueAddress)
 
-      diffMemberEvents(g1, g2) should ===(Seq.empty)
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = Set(aUp.address, bUp.address))))
-      diffMemberEvents(g2, g1) should ===(Seq.empty)
-      diffUnreachable(g2, g1, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g2, g1, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = Set(aUp.address, bUp.address, eJoining.address))))
+      diffMemberEvents(state(g1), state(g2)) should ===(Seq.empty)
+      diffUnreachable(state(g1), state(g2)) should ===(Seq.empty)
+      diffSeen(state(g1), state(g2)) should ===(Seq(SeenChanged(convergence = true, seenBy = Set(aUp.address, bUp.address))))
+      diffMemberEvents(state(g2), state(g1)) should ===(Seq.empty)
+      diffUnreachable(state(g2), state(g1)) should ===(Seq.empty)
+      diffSeen(state(g2), state(g1)) should ===(Seq(SeenChanged(convergence = true, seenBy = Set(aUp.address, bUp.address, eJoining.address))))
     }
 
     "be produced for leader changes" in {
       val (g1, _) = converge(Gossip(members = SortedSet(aUp, bUp, eJoining)))
       val (g2, s2) = converge(Gossip(members = SortedSet(bUp, eJoining)))
 
-      diffMemberEvents(g1, g2) should ===(Seq(MemberRemoved(aRemoved, Up)))
-      diffUnreachable(g1, g2, selfDummyAddress) should ===(Seq.empty)
-      diffSeen(g1, g2, selfDummyAddress) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
-      diffLeader(g1, g2, selfDummyAddress) should ===(Seq(LeaderChanged(Some(bUp.address))))
+      diffMemberEvents(state(g1), state(g2)) should ===(Seq(MemberRemoved(aRemoved, Up)))
+      diffUnreachable(state(g1), state(g2)) should ===(Seq.empty)
+      diffSeen(state(g1), state(g2)) should ===(Seq(SeenChanged(convergence = true, seenBy = s2.map(_.address))))
+      diffLeader(state(g1), state(g2)) should ===(Seq(LeaderChanged(Some(bUp.address))))
     }
 
-    "be produced for role leader changes" in {
+    "be produced for role leader changes in the same data center" in {
       val g0 = Gossip.empty
       val g1 = Gossip(members = SortedSet(aUp, bUp, cUp, dLeaving, eJoining))
       val g2 = Gossip(members = SortedSet(bUp, cUp, dExiting, eJoining))
-      diffRolesLeader(g0, g1, selfDummyAddress) should ===(
+      diffRolesLeader(state(g0), state(g1)) should ===(
         Set(
+          // since this role is implicitly added
+          RoleLeaderChanged(ClusterSettings.DcRolePrefix + ClusterSettings.DefaultDataCenter, Some(aUp.address)),
           RoleLeaderChanged("AA", Some(aUp.address)),
           RoleLeaderChanged("AB", Some(aUp.address)),
           RoleLeaderChanged("BB", Some(bUp.address)),
           RoleLeaderChanged("DD", Some(dLeaving.address)),
           RoleLeaderChanged("DE", Some(dLeaving.address)),
           RoleLeaderChanged("EE", Some(eUp.address))))
-      diffRolesLeader(g1, g2, selfDummyAddress) should ===(
+      diffRolesLeader(state(g1), state(g2)) should ===(
         Set(
+          RoleLeaderChanged(ClusterSettings.DcRolePrefix + ClusterSettings.DefaultDataCenter, Some(bUp.address)),
           RoleLeaderChanged("AA", None),
           RoleLeaderChanged("AB", Some(bUp.address)),
           RoleLeaderChanged("DE", Some(eJoining.address))))
     }
+
+    "not be produced for role leader changes in other data centers" in {
+      val g0 = Gossip.empty
+      val s0 = state(g0).copy(selfDc = "dc2")
+      val g1 = Gossip(members = SortedSet(aUp, bUp, cUp, dLeaving, eJoining))
+      val s1 = state(g1).copy(selfDc = "dc2")
+      val g2 = Gossip(members = SortedSet(bUp, cUp, dExiting, eJoining))
+      val s2 = state(g2).copy(selfDc = "dc2")
+
+      diffRolesLeader(s0, s1) should ===(Set.empty)
+      diffRolesLeader(s1, s2) should ===(Set.empty)
+    }
   }
 }

akka-cluster/src/test/scala/akka/cluster/GossipSpec.scala

@@ -7,6 +7,9 @@ package akka.cluster
 import org.scalatest.WordSpec
 import org.scalatest.Matchers
 import akka.actor.Address
+import akka.cluster.Gossip.vclockName
+import akka.cluster.ClusterSettings.DefaultDataCenter
+
 import scala.collection.immutable.SortedSet
 
 class GossipSpec extends WordSpec with Matchers {
@@ -25,43 +28,57 @@ class GossipSpec extends WordSpec with Matchers {
   val e2 = TestMember(e1.address, Up)
   val e3 = TestMember(e1.address, Down)
 
+  val dc1a1 = TestMember(Address("akka.tcp", "sys", "a", 2552), Up, Set.empty, dataCenter = "dc1")
+  val dc1b1 = TestMember(Address("akka.tcp", "sys", "b", 2552), Up, Set.empty, dataCenter = "dc1")
+  val dc2c1 = TestMember(Address("akka.tcp", "sys", "c", 2552), Up, Set.empty, dataCenter = "dc2")
+  val dc2d1 = TestMember(Address("akka.tcp", "sys", "d", 2552), Up, Set.empty, dataCenter = "dc2")
+  val dc2d2 = TestMember(dc2d1.address, status = Down, roles = Set.empty, dataCenter = dc2d1.dataCenter)
+
+  private def state(g: Gossip, selfMember: Member = a1): MembershipState =
+    MembershipState(g, selfMember.uniqueAddress, selfMember.dataCenter, crossDcConnections = 5)
+
   "A Gossip" must {
 
+    "have correct test setup" in {
+      List(a1, a2, b1, b2, c1, c2, c3, d1, e1, e2, e3).foreach(m ⇒
+        m.dataCenter should ===(DefaultDataCenter))
+    }
+
     "reach convergence when it's empty" in {
-      Gossip.empty.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(Gossip.empty).convergence(Set.empty) should ===(true)
     }
 
     "reach convergence for one node" in {
       val g1 = Gossip(members = SortedSet(a1)).seen(a1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(g1).convergence(Set.empty) should ===(true)
     }
 
     "not reach convergence until all have seen version" in {
       val g1 = Gossip(members = SortedSet(a1, b1)).seen(a1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(false)
+      state(g1).convergence(Set.empty) should ===(false)
     }
 
     "reach convergence for two nodes" in {
       val g1 = Gossip(members = SortedSet(a1, b1)).seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(g1).convergence(Set.empty) should ===(true)
     }
 
     "reach convergence, skipping joining" in {
       // e1 is joining
       val g1 = Gossip(members = SortedSet(a1, b1, e1)).seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(g1).convergence(Set.empty) should ===(true)
     }
 
     "reach convergence, skipping down" in {
       // e3 is down
       val g1 = Gossip(members = SortedSet(a1, b1, e3)).seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(g1).convergence(Set.empty) should ===(true)
     }
 
     "reach convergence, skipping Leaving with exitingConfirmed" in {
       // c1 is Leaving
       val g1 = Gossip(members = SortedSet(a1, b1, c1)).seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set(c1.uniqueAddress)) should ===(true)
+      state(g1).convergence(Set(c1.uniqueAddress)) should ===(true)
     }
 
     "reach convergence, skipping unreachable Leaving with exitingConfirmed" in {
@@ -69,16 +86,16 @@ class GossipSpec extends WordSpec with Matchers {
       val r1 = Reachability.empty.unreachable(b1.uniqueAddress, c1.uniqueAddress)
       val g1 = Gossip(members = SortedSet(a1, b1, c1), overview = GossipOverview(reachability = r1))
         .seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(a1.uniqueAddress, Set(c1.uniqueAddress)) should ===(true)
+      state(g1).convergence(Set(c1.uniqueAddress)) should ===(true)
     }
 
     "not reach convergence when unreachable" in {
       val r1 = Reachability.empty.unreachable(b1.uniqueAddress, a1.uniqueAddress)
       val g1 = (Gossip(members = SortedSet(a1, b1), overview = GossipOverview(reachability = r1)))
         .seen(a1.uniqueAddress).seen(b1.uniqueAddress)
-      g1.convergence(b1.uniqueAddress, Set.empty) should ===(false)
+      state(g1, b1).convergence(Set.empty) should ===(false)
       // but from a1's point of view (it knows that itself is not unreachable)
-      g1.convergence(a1.uniqueAddress, Set.empty) should ===(true)
+      state(g1).convergence(Set.empty) should ===(true)
     }
 
     "reach convergence when downed node has observed unreachable" in {
@@ -86,7 +103,7 @@ class GossipSpec extends WordSpec with Matchers {
       val r1 = Reachability.empty.unreachable(e3.uniqueAddress, a1.uniqueAddress)
       val g1 = (Gossip(members = SortedSet(a1, b1, e3), overview = GossipOverview(reachability = r1)))
         .seen(a1.uniqueAddress).seen(b1.uniqueAddress).seen(e3.uniqueAddress)
-      g1.convergence(b1.uniqueAddress, Set.empty) should ===(true)
+      state(g1, b1).convergence(Set.empty) should ===(true)
     }
 
     "merge members by status priority" in {
@@ -133,21 +150,33 @@ class GossipSpec extends WordSpec with Matchers {
     }
 
     "have leader as first member based on ordering, except Exiting status" in {
-      Gossip(members = SortedSet(c2, e2)).leader(c2.uniqueAddress) should ===(Some(c2.uniqueAddress))
-      Gossip(members = SortedSet(c3, e2)).leader(c3.uniqueAddress) should ===(Some(e2.uniqueAddress))
-      Gossip(members = SortedSet(c3)).leader(c3.uniqueAddress) should ===(Some(c3.uniqueAddress))
+      state(Gossip(members = SortedSet(c2, e2)), c2).leader should ===(Some(c2.uniqueAddress))
+      state(Gossip(members = SortedSet(c3, e2)), c3).leader should ===(Some(e2.uniqueAddress))
+      state(Gossip(members = SortedSet(c3)), c3).leader should ===(Some(c3.uniqueAddress))
     }
 
     "have leader as first reachable member based on ordering" in {
       val r1 = Reachability.empty.unreachable(e2.uniqueAddress, c2.uniqueAddress)
       val g1 = Gossip(members = SortedSet(c2, e2), overview = GossipOverview(reachability = r1))
-      g1.leader(e2.uniqueAddress) should ===(Some(e2.uniqueAddress))
+      state(g1, e2).leader should ===(Some(e2.uniqueAddress))
       // but when c2 is selfUniqueAddress
-      g1.leader(c2.uniqueAddress) should ===(Some(c2.uniqueAddress))
+      state(g1, c2).leader should ===(Some(c2.uniqueAddress))
     }
 
     "not have Down member as leader" in {
-      Gossip(members = SortedSet(e3)).leader(e3.uniqueAddress) should ===(None)
+      state(Gossip(members = SortedSet(e3)), e3).leader should ===(None)
+    }
+
+    "have a leader per data center" in {
+      val g1 = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+
+      // dc1a1 being leader of dc1
+      state(g1, dc1a1).leader should ===(Some(dc1a1.uniqueAddress))
+      state(g1, dc1b1).leader should ===(Some(dc1a1.uniqueAddress))
+
+      // and dc2c1 being leader of dc2
+      state(g1, dc2c1).leader should ===(Some(dc2c1.uniqueAddress))
+      state(g1, dc2d1).leader should ===(Some(dc2c1.uniqueAddress))
     }
 
     "merge seen table correctly" in {
@@ -182,5 +211,255 @@ class GossipSpec extends WordSpec with Matchers {
       val g3 = Gossip(members = SortedSet(a2, b1.copyUp(3), e2.copyUp(4)))
       g3.youngestMember should ===(e2)
     }
+
+    "reach convergence per data center" in {
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .seen(dc1a1.uniqueAddress)
+        .seen(dc1b1.uniqueAddress)
+        .seen(dc2c1.uniqueAddress)
+        .seen(dc2d1.uniqueAddress)
+      state(g, dc1a1).leader should ===(Some(dc1a1.uniqueAddress))
+      state(g, dc1a1).convergence(Set.empty) should ===(true)
+
+      state(g, dc2c1).leader should ===(Some(dc2c1.uniqueAddress))
+      state(g, dc2c1).convergence(Set.empty) should ===(true)
+    }
+
+    "reach convergence per data center even if members of another data center has not seen the gossip" in {
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .seen(dc1a1.uniqueAddress)
+        .seen(dc1b1.uniqueAddress)
+        .seen(dc2c1.uniqueAddress)
+      // dc2d1 has not seen the gossip
+
+      // so dc1 can reach convergence
+      state(g, dc1a1).leader should ===(Some(dc1a1.uniqueAddress))
+      state(g, dc1a1).convergence(Set.empty) should ===(true)
+
+      // but dc2 cannot
+      state(g, dc2c1).leader should ===(Some(dc2c1.uniqueAddress))
+      state(g, dc2c1).convergence(Set.empty) should ===(false)
+    }
+
+    "reach convergence per data center even if another data center contains unreachable" in {
+      val r1 = Reachability.empty.unreachable(dc2c1.uniqueAddress, dc2d1.uniqueAddress)
+
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1), overview = GossipOverview(reachability = r1))
+        .seen(dc1a1.uniqueAddress)
+        .seen(dc1b1.uniqueAddress)
+        .seen(dc2c1.uniqueAddress)
+        .seen(dc2d1.uniqueAddress)
+
+      // this data center doesn't care about dc2 having reachability problems and can reach convergence
+      state(g, dc1a1).leader should ===(Some(dc1a1.uniqueAddress))
+      state(g, dc1a1).convergence(Set.empty) should ===(true)
+
+      // this data center is cannot reach convergence because of unreachability within the data center
+      state(g, dc2c1).leader should ===(Some(dc2c1.uniqueAddress))
+      state(g, dc2c1).convergence(Set.empty) should ===(false)
+    }
+
+    "reach convergence per data center even if there is unreachable nodes in another data center" in {
+      val r1 = Reachability.empty
+        .unreachable(dc1a1.uniqueAddress, dc2d1.uniqueAddress)
+        .unreachable(dc2d1.uniqueAddress, dc1a1.uniqueAddress)
+
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1), overview = GossipOverview(reachability = r1))
+        .seen(dc1a1.uniqueAddress)
+        .seen(dc1b1.uniqueAddress)
+        .seen(dc2c1.uniqueAddress)
+        .seen(dc2d1.uniqueAddress)
+
+      // neither data center is affected by the inter data center unreachability as far as convergence goes
+      state(g, dc1a1).leader should ===(Some(dc1a1.uniqueAddress))
+      state(g, dc1a1).convergence(Set.empty) should ===(true)
+
+      state(g, dc2c1).leader should ===(Some(dc2c1.uniqueAddress))
+      state(g, dc2c1).convergence(Set.empty) should ===(true)
+    }
+
+    "ignore cross data center unreachability when determining inside of data center reachability" in {
+      val r1 = Reachability.empty
+        .unreachable(dc1a1.uniqueAddress, dc2c1.uniqueAddress)
+        .unreachable(dc2c1.uniqueAddress, dc1a1.uniqueAddress)
+
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1), overview = GossipOverview(reachability = r1))
+
+      // inside of the data center we don't care about the cross data center unreachability
+      g.isReachable(dc1a1.uniqueAddress, dc1b1.uniqueAddress) should ===(true)
+      g.isReachable(dc1b1.uniqueAddress, dc1a1.uniqueAddress) should ===(true)
+      g.isReachable(dc2c1.uniqueAddress, dc2d1.uniqueAddress) should ===(true)
+      g.isReachable(dc2d1.uniqueAddress, dc2c1.uniqueAddress) should ===(true)
+
+      state(g, dc1a1).isReachableExcludingDownedObservers(dc1b1.uniqueAddress) should ===(true)
+      state(g, dc1b1).isReachableExcludingDownedObservers(dc1a1.uniqueAddress) should ===(true)
+      state(g, dc2c1).isReachableExcludingDownedObservers(dc2d1.uniqueAddress) should ===(true)
+      state(g, dc2d1).isReachableExcludingDownedObservers(dc2c1.uniqueAddress) should ===(true)
+
+      // between data centers it matters though
+      g.isReachable(dc1a1.uniqueAddress, dc2c1.uniqueAddress) should ===(false)
+      g.isReachable(dc2c1.uniqueAddress, dc1a1.uniqueAddress) should ===(false)
+      // this isReachable method only says false for specific unreachable entries between the nodes
+      g.isReachable(dc1b1.uniqueAddress, dc2c1.uniqueAddress) should ===(true)
+      g.isReachable(dc2d1.uniqueAddress, dc1a1.uniqueAddress) should ===(true)
+
+      // this one looks at all unreachable-entries for the to-address
+      state(g, dc1a1).isReachableExcludingDownedObservers(dc2c1.uniqueAddress) should ===(false)
+      state(g, dc1b1).isReachableExcludingDownedObservers(dc2c1.uniqueAddress) should ===(false)
+      state(g, dc2c1).isReachableExcludingDownedObservers(dc1a1.uniqueAddress) should ===(false)
+      state(g, dc2d1).isReachableExcludingDownedObservers(dc1a1.uniqueAddress) should ===(false)
+
+      // between the two other nodes there is no unreachability
+      g.isReachable(dc1b1.uniqueAddress, dc2d1.uniqueAddress) should ===(true)
+      g.isReachable(dc2d1.uniqueAddress, dc1b1.uniqueAddress) should ===(true)
+
+      state(g, dc1b1).isReachableExcludingDownedObservers(dc2d1.uniqueAddress) should ===(true)
+      state(g, dc2d1).isReachableExcludingDownedObservers(dc1b1.uniqueAddress) should ===(true)
+    }
+
+    "not returning a downed data center leader" in {
+      val g = Gossip(members = SortedSet(dc1a1.copy(Down), dc1b1))
+      state(g, dc1b1).leaderOf(g.members) should ===(Some(dc1b1.uniqueAddress))
+    }
+
+    "ignore cross data center unreachability when determining data center leader" in {
+      val r1 = Reachability.empty
+        .unreachable(dc1a1.uniqueAddress, dc2d1.uniqueAddress)
+        .unreachable(dc2d1.uniqueAddress, dc1a1.uniqueAddress)
+
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1), overview = GossipOverview(reachability = r1))
+
+      state(g, dc1a1).leaderOf(g.members) should ===(Some(dc1a1.uniqueAddress))
+      state(g, dc1b1).leaderOf(g.members) should ===(Some(dc1a1.uniqueAddress))
+
+      state(g, dc2c1).leaderOf(g.members) should ===(Some(dc2c1.uniqueAddress))
+      state(g, dc2d1).leaderOf(g.members) should ===(Some(dc2c1.uniqueAddress))
+    }
+
+    // TODO test coverage for when leaderOf returns None - I have not been able to figure it out
+
+    "clear out a bunch of stuff when removing a node" in {
+      val g = Gossip(
+        members = SortedSet(dc1a1, dc1b1, dc2d2),
+        overview = GossipOverview(reachability =
+          Reachability.empty
+            .unreachable(dc1b1.uniqueAddress, dc2d2.uniqueAddress)
+            .unreachable(dc2d2.uniqueAddress, dc1b1.uniqueAddress)
+        ))
+        .:+(VectorClock.Node(Gossip.vclockName(dc1b1.uniqueAddress)))
+        .:+(VectorClock.Node(Gossip.vclockName(dc2d2.uniqueAddress)))
+        .remove(dc1b1.uniqueAddress, System.currentTimeMillis())
+
+      g.seenBy should not contain (dc1b1.uniqueAddress)
+      g.overview.reachability.records.map(_.observer) should not contain (dc1b1.uniqueAddress)
+      g.overview.reachability.records.map(_.subject) should not contain (dc1b1.uniqueAddress)
+
+      // sort order should be kept
+      g.members.toList should ===(List(dc1a1, dc2d2))
+      g.version.versions.keySet should not contain (VectorClock.Node(Gossip.vclockName(dc1b1.uniqueAddress)))
+      g.version.versions.keySet should contain(VectorClock.Node(Gossip.vclockName(dc2d2.uniqueAddress)))
+    }
+
+    "not reintroduce members from out-of data center gossip when merging" in {
+      // dc1 does not know about any unreachability nor that the node has been downed
+      val gdc1 = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .seen(dc1b1.uniqueAddress)
+        .seen(dc2c1.uniqueAddress)
+        .:+(VectorClock.Node(vclockName(dc2d1.uniqueAddress))) // just to make sure these are also pruned
+
+      // dc2 has downed the dc2d1 node, seen it as unreachable and removed it
+      val gdc2 = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .seen(dc1a1.uniqueAddress)
+        .remove(dc2d1.uniqueAddress, System.currentTimeMillis())
+
+      gdc2.tombstones.keys should contain(dc2d1.uniqueAddress)
+      gdc2.members should not contain (dc2d1)
+      gdc2.overview.reachability.records.filter(r ⇒ r.subject == dc2d1.uniqueAddress || r.observer == dc2d1.uniqueAddress) should be(empty)
+      gdc2.overview.reachability.versions.keys should not contain (dc2d1.uniqueAddress)
+
+      // when we merge the two, it should not be reintroduced
+      val merged1 = gdc2 merge gdc1
+      merged1.members should ===(SortedSet(dc1a1, dc1b1, dc2c1))
+
+      merged1.tombstones.keys should contain(dc2d1.uniqueAddress)
+      merged1.members should not contain (dc2d1)
+      merged1.overview.reachability.records.filter(r ⇒ r.subject == dc2d1.uniqueAddress || r.observer == dc2d1.uniqueAddress) should be(empty)
+      merged1.overview.reachability.versions.keys should not contain (dc2d1.uniqueAddress)
+      merged1.version.versions.keys should not contain (VectorClock.Node(vclockName(dc2d1.uniqueAddress)))
+    }
+
+    "correctly prune vector clocks based on tombstones when merging" in {
+      val gdc1 = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .:+(VectorClock.Node(vclockName(dc1a1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc1b1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc2c1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc2d1.uniqueAddress)))
+        .remove(dc1b1.uniqueAddress, System.currentTimeMillis())
+
+      gdc1.version.versions.keySet should not contain (VectorClock.Node(vclockName(dc1b1.uniqueAddress)))
+
+      val gdc2 = Gossip(members = SortedSet(dc1a1, dc1b1, dc2c1, dc2d1))
+        .seen(dc1a1.uniqueAddress)
+        .:+(VectorClock.Node(vclockName(dc1a1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc1b1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc2c1.uniqueAddress)))
+        .:+(VectorClock.Node(vclockName(dc2d1.uniqueAddress)))
+        .remove(dc2c1.uniqueAddress, System.currentTimeMillis())
+
+      gdc2.version.versions.keySet should not contain (VectorClock.Node(vclockName(dc2c1.uniqueAddress)))
+
+      // when we merge the two, the nodes should not be reintroduced
+      val merged1 = gdc2 merge gdc1
+      merged1.members should ===(SortedSet(dc1a1, dc2d1))
+
+      merged1.version.versions.keySet should ===(Set(
+        VectorClock.Node(vclockName(dc1a1.uniqueAddress)),
+        VectorClock.Node(vclockName(dc2d1.uniqueAddress))))
+    }
+
+    "prune old tombstones" in {
+      val timestamp = 352684800
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1))
+        .remove(dc1b1.uniqueAddress, timestamp)
+
+      g.tombstones.keys should contain(dc1b1.uniqueAddress)
+
+      val pruned = g.pruneTombstones(timestamp + 1)
+
+      // when we merge the two, it should not be reintroduced
+      pruned.tombstones.keys should not contain (dc1b1.uniqueAddress)
+    }
+
+    "mark a node as down" in {
+      val g = Gossip(members = SortedSet(dc1a1, dc1b1))
+        .seen(dc1a1.uniqueAddress)
+        .seen(dc1b1.uniqueAddress)
+        .markAsDown(dc1b1)
+
+      g.member(dc1b1.uniqueAddress).status should ===(MemberStatus.Down)
+      g.overview.seen should not contain (dc1b1.uniqueAddress)
+
+      // obviously the other member should be unaffected
+      g.member(dc1a1.uniqueAddress).status should ===(dc1a1.status)
+      g.overview.seen should contain(dc1a1.uniqueAddress)
+    }
+
+    "update members" in {
+      val joining = TestMember(Address("akka.tcp", "sys", "d", 2552), Joining, Set.empty, dataCenter = "dc2")
+      val g = Gossip(members = SortedSet(dc1a1, joining))
+
+      g.member(joining.uniqueAddress).status should ===(Joining)
+      val oldMembers = g.members
+
+      val updated = g.update(SortedSet(joining.copy(status = Up)))
+
+      updated.member(joining.uniqueAddress).status should ===(Up)
+
+      // obviously the other member should be unaffected
+      updated.member(dc1a1.uniqueAddress).status should ===(dc1a1.status)
+
+      // order should be kept
+      updated.members.toList.map(_.uniqueAddress) should ===(List(dc1a1.uniqueAddress, joining.uniqueAddress))
+    }
   }
 }

akka-cluster/src/test/scala/akka/cluster/GossipTargetSelectorSpec.scala

@@ -0,0 +1,206 @@
+/**
+ * Copyright (C) 2009-2017 Lightbend Inc. <http://www.lightbend.com>
+ */
+package akka.cluster
+
+import akka.actor.Address
+import akka.cluster.ClusterSettings.DataCenter
+import akka.cluster.MemberStatus.Up
+import org.scalatest.{ Matchers, WordSpec }
+
+import scala.collection.immutable.SortedSet
+
+class GossipTargetSelectorSpec extends WordSpec with Matchers {
+
+  val aDc1 = TestMember(Address("akka.tcp", "sys", "a", 2552), Up, Set.empty, dataCenter = "dc1")
+  val bDc1 = TestMember(Address("akka.tcp", "sys", "b", 2552), Up, Set.empty, dataCenter = "dc1")
+  val cDc1 = TestMember(Address("akka.tcp", "sys", "c", 2552), Up, Set.empty, dataCenter = "dc1")
+
+  val eDc2 = TestMember(Address("akka.tcp", "sys", "e", 2552), Up, Set.empty, dataCenter = "dc2")
+  val fDc2 = TestMember(Address("akka.tcp", "sys", "f", 2552), Up, Set.empty, dataCenter = "dc2")
+
+  val gDc3 = TestMember(Address("akka.tcp", "sys", "g", 2552), Up, Set.empty, dataCenter = "dc3")
+  val hDc3 = TestMember(Address("akka.tcp", "sys", "h", 2552), Up, Set.empty, dataCenter = "dc3")
+
+  val defaultSelector = new GossipTargetSelector(
+    reduceGossipDifferentViewProbability = 400,
+    crossDcGossipProbability = 0.2
+  )
+
+  "The gossip target selection" should {
+
+    "select local nodes in a multi dc setting when chance says so" in {
+      val alwaysLocalSelector = new GossipTargetSelector(400, 0.2) {
+        override protected def selectDcLocalNodes: Boolean = true
+      }
+
+      val state = MembershipState(Gossip(SortedSet(aDc1, bDc1, eDc2, fDc2)), aDc1, aDc1.dataCenter, crossDcConnections = 5)
+      val gossipTo = alwaysLocalSelector.gossipTargets(state)
+
+      // only one other local node
+      gossipTo should ===(Vector[UniqueAddress](bDc1))
+    }
+
+    "select cross dc nodes when chance says so" in {
+      val alwaysCrossDcSelector = new GossipTargetSelector(400, 0.2) {
+        override protected def selectDcLocalNodes: Boolean = false
+      }
+
+      val state = MembershipState(Gossip(SortedSet(aDc1, bDc1, eDc2, fDc2)), aDc1, aDc1.dataCenter, crossDcConnections = 5)
+      val gossipTo = alwaysCrossDcSelector.gossipTargets(state)
+
+      // only one other local node
+      gossipTo should (contain(eDc2.uniqueAddress) or contain(fDc2.uniqueAddress))
+    }
+
+    "select local nodes that hasn't seen the gossip when chance says so" in {
+      val alwaysLocalSelector = new GossipTargetSelector(400, 0.2) {
+        override protected def preferNodesWithDifferentView(state: MembershipState): Boolean = true
+      }
+
+      val state = MembershipState(
+        Gossip(SortedSet(aDc1, bDc1, cDc1)).seen(bDc1),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 5
+      )
+      val gossipTo = alwaysLocalSelector.gossipTargets(state)
+
+      // a1 is self, b1 has seen so only option is c1
+      gossipTo should ===(Vector[UniqueAddress](cDc1))
+    }
+
+    "select among all local nodes regardless if they saw the gossip already when chance says so" in {
+      val alwaysLocalSelector = new GossipTargetSelector(400, 0.2) {
+        override protected def preferNodesWithDifferentView(state: MembershipState): Boolean = false
+      }
+
+      val state = MembershipState(
+        Gossip(SortedSet(aDc1, bDc1, cDc1)).seen(bDc1),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 5
+      )
+      val gossipTo = alwaysLocalSelector.gossipTargets(state)
+
+      // a1 is self, b1 is the only that has seen
+      gossipTo should ===(Vector[UniqueAddress](bDc1, cDc1))
+    }
+
+    "not choose unreachable nodes" in {
+      val alwaysLocalSelector = new GossipTargetSelector(400, 0.2) {
+        override protected def preferNodesWithDifferentView(state: MembershipState): Boolean = false
+      }
+
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, cDc1),
+          overview = GossipOverview(
+            reachability = Reachability.empty.unreachable(aDc1, bDc1))),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 5)
+      val gossipTo = alwaysLocalSelector.gossipTargets(state)
+
+      // a1 cannot reach b1 so only option is c1
+      gossipTo should ===(Vector[UniqueAddress](cDc1))
+    }
+
+    "continue with the next dc when doing cross dc and no node where suitable" in {
+      val selector = new GossipTargetSelector(400, 0.2) {
+        override protected def selectDcLocalNodes: Boolean = false
+        override protected def dcsInRandomOrder(dcs: List[DataCenter]): List[DataCenter] = dcs.sorted // sort on name
+      }
+
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, eDc2, fDc2, gDc3, hDc3),
+          overview = GossipOverview(
+            reachability = Reachability.empty
+              .unreachable(aDc1, eDc2)
+              .unreachable(aDc1, fDc2))),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 5)
+      val gossipTo = selector.gossipTargets(state)
+      gossipTo should ===(Vector[UniqueAddress](gDc3, hDc3))
+    }
+
+    "not care about seen/unseen for cross dc" in {
+      val selector = new GossipTargetSelector(400, 0.2) {
+        override protected def selectDcLocalNodes: Boolean = false
+        override protected def dcsInRandomOrder(dcs: List[DataCenter]): List[DataCenter] = dcs.sorted // sort on name
+      }
+
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, eDc2, fDc2, gDc3, hDc3)
+        ).seen(fDc2).seen(hDc3),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 5)
+      val gossipTo = selector.gossipTargets(state)
+      gossipTo should ===(Vector[UniqueAddress](eDc2, fDc2))
+    }
+
+    "limit the numbers of chosen cross dc nodes to the crossDcConnections setting" in {
+      val selector = new GossipTargetSelector(400, 0.2) {
+        override protected def selectDcLocalNodes: Boolean = false
+        override protected def dcsInRandomOrder(dcs: List[DataCenter]): List[DataCenter] = dcs.sorted // sort on name
+      }
+
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, eDc2, fDc2, gDc3, hDc3)),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 1)
+      val gossipTo = selector.gossipTargets(state)
+      gossipTo should ===(Vector[UniqueAddress](eDc2))
+    }
+
+    "select N random local nodes when single dc" in {
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, cDc1)),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 1) // means only a e and g are oldest
+
+      val randomNodes = defaultSelector.randomNodesForFullGossip(state, 3)
+
+      randomNodes.toSet should ===(Set[UniqueAddress](bDc1, cDc1))
+    }
+
+    "select N random local nodes when not self among oldest" in {
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, cDc1, eDc2, fDc2, gDc3, hDc3)),
+        bDc1,
+        bDc1.dataCenter,
+        crossDcConnections = 1) // means only a, e and g are oldest
+
+      val randomNodes = defaultSelector.randomNodesForFullGossip(state, 3)
+
+      randomNodes.toSet should ===(Set[UniqueAddress](aDc1, cDc1))
+    }
+
+    "select N-1 random local nodes plus one cross dc oldest node when self among oldest" in {
+      val state = MembershipState(
+        Gossip(
+          members = SortedSet(aDc1, bDc1, cDc1, eDc2, fDc2)),
+        aDc1,
+        aDc1.dataCenter,
+        crossDcConnections = 1) // means only a and e are oldest
+
+      val randomNodes = defaultSelector.randomNodesForFullGossip(state, 3)
+
+      randomNodes.toSet should ===(Set[UniqueAddress](bDc1, cDc1, eDc2))
+    }
+
+  }
+
+  // made the test so much easier to read
+  import scala.language.implicitConversions
+  private implicit def memberToUniqueAddress(m: Member): UniqueAddress = m.uniqueAddress
+}

akka-cluster/src/test/scala/akka/cluster/TestMember.scala

@@ -9,6 +9,6 @@ object TestMember {
   def apply(address: Address, status: MemberStatus): Member =
     apply(address, status, Set.empty)
 
-  def apply(address: Address, status: MemberStatus, roles: Set[String]): Member =
-    new Member(UniqueAddress(address, 0L), Int.MaxValue, status, roles)
+  def apply(address: Address, status: MemberStatus, roles: Set[String], dataCenter: ClusterSettings.DataCenter = ClusterSettings.DefaultDataCenter): Member =
+    new Member(UniqueAddress(address, 0L), Int.MaxValue, status, roles + (ClusterSettings.DcRolePrefix + dataCenter))
 }

akka-cluster/src/test/scala/akka/cluster/protobuf/ClusterMessageSerializerSpec.scala

@@ -16,16 +16,18 @@ class ClusterMessageSerializerSpec extends AkkaSpec(
 
   val serializer = new ClusterMessageSerializer(system.asInstanceOf[ExtendedActorSystem])
 
-  def checkSerialization(obj: AnyRef): Unit = {
+  def roundtrip[T <: AnyRef](obj: T): T = {
     val blob = serializer.toBinary(obj)
-    val ref = serializer.fromBinary(blob, obj.getClass)
-    obj match {
-      case env: GossipEnvelope ⇒
-        val env2 = obj.asInstanceOf[GossipEnvelope]
+    serializer.fromBinary(blob, obj.getClass).asInstanceOf[T]
+  }
+
+  def checkSerialization(obj: AnyRef): Unit = {
+    (obj, roundtrip(obj)) match {
+      case (env: GossipEnvelope, env2: GossipEnvelope) ⇒
         env2.from should ===(env.from)
         env2.to should ===(env.to)
         env2.gossip should ===(env.gossip)
-      case _ ⇒
+      case (_, ref) ⇒
         ref should ===(obj)
     }
 
@@ -35,10 +37,10 @@ class ClusterMessageSerializerSpec extends AkkaSpec(
 
   val a1 = TestMember(Address("akka.tcp", "sys", "a", 2552), Joining, Set.empty)
   val b1 = TestMember(Address("akka.tcp", "sys", "b", 2552), Up, Set("r1"))
-  val c1 = TestMember(Address("akka.tcp", "sys", "c", 2552), Leaving, Set("r2"))
-  val d1 = TestMember(Address("akka.tcp", "sys", "d", 2552), Exiting, Set("r1", "r2"))
+  val c1 = TestMember(Address("akka.tcp", "sys", "c", 2552), Leaving, Set.empty, "foo")
+  val d1 = TestMember(Address("akka.tcp", "sys", "d", 2552), Exiting, Set("r1"), "foo")
   val e1 = TestMember(Address("akka.tcp", "sys", "e", 2552), Down, Set("r3"))
-  val f1 = TestMember(Address("akka.tcp", "sys", "f", 2552), Removed, Set("r2", "r3"))
+  val f1 = TestMember(Address("akka.tcp", "sys", "f", 2552), Removed, Set("r3"), "foo")
 
   "ClusterMessages" must {
 
@@ -65,9 +67,11 @@ class ClusterMessageSerializerSpec extends AkkaSpec(
       val g2 = (g1 :+ node3 :+ node4).seen(a1.uniqueAddress).seen(c1.uniqueAddress)
       val reachability3 = Reachability.empty.unreachable(a1.uniqueAddress, e1.uniqueAddress).unreachable(b1.uniqueAddress, e1.uniqueAddress)
       val g3 = g2.copy(members = SortedSet(a1, b1, c1, d1, e1), overview = g2.overview.copy(reachability = reachability3))
+      val g4 = g1.remove(d1.uniqueAddress, 352684800)
       checkSerialization(GossipEnvelope(a1.uniqueAddress, uniqueAddress2, g1))
       checkSerialization(GossipEnvelope(a1.uniqueAddress, uniqueAddress2, g2))
       checkSerialization(GossipEnvelope(a1.uniqueAddress, uniqueAddress2, g3))
+      checkSerialization(GossipEnvelope(a1.uniqueAddress, uniqueAddress2, g4))
 
       checkSerialization(GossipStatus(a1.uniqueAddress, g1.version))
       checkSerialization(GossipStatus(a1.uniqueAddress, g2.version))
@@ -110,6 +114,12 @@ class ClusterMessageSerializerSpec extends AkkaSpec(
       }
 
     }
+
+    "add a default data center role if none is present" in {
+      val env = roundtrip(GossipEnvelope(a1.uniqueAddress, d1.uniqueAddress, Gossip(SortedSet(a1, d1))))
+      env.gossip.members.head.roles should be(Set(ClusterSettings.DcRolePrefix + "default"))
+      env.gossip.members.tail.head.roles should be(Set("r1", ClusterSettings.DcRolePrefix + "foo"))
+    }
   }
   "Cluster router pool" must {
     "be serializable with no role" in {
@@ -142,8 +152,7 @@ class ClusterMessageSerializerSpec extends AkkaSpec(
     "be serializable with many roles" in {
       checkSerialization(ClusterRouterPool(
         RoundRobinPool(
-          nrOfInstances = 4
-        ),
+          nrOfInstances = 4),
         ClusterRouterPoolSettings(
           totalInstances = 2,
           maxInstancesPerNode = 5,

akka-distributed-data/src/main/mima-filters/2.5.4.backwards.excludes

@@ -0,0 +1,4 @@
+# #23231 multi-DC Sharding
+ProblemFilters.exclude[IncompatibleResultTypeProblem]("akka.cluster.ddata.Replicator.leader")
+ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.ddata.Replicator.receiveLeaderChanged")
+ProblemFilters.exclude[IncompatibleMethTypeProblem]("akka.cluster.ddata.Replicator.leader_=")

akka-distributed-data/src/main/scala/akka/cluster/ddata/DistributedData.scala

@@ -34,7 +34,7 @@ class DistributedData(system: ExtendedActorSystem) extends Extension {
    * Returns true if this member is not tagged with the role configured for the
    * replicas.
    */
-  def isTerminated: Boolean = Cluster(system).isTerminated || !settings.role.forall(Cluster(system).selfRoles.contains)
+  def isTerminated: Boolean = Cluster(system).isTerminated || !settings.roles.subsetOf(Cluster(system).selfRoles)
 
   /**
    * `ActorRef` of the [[Replicator]] .

akka-distributed-data/src/main/scala/akka/cluster/ddata/Replicator.scala

@@ -48,6 +48,9 @@ import akka.actor.Cancellable
 import scala.util.control.NonFatal
 import akka.cluster.ddata.Key.KeyId
 import akka.annotation.InternalApi
+import scala.collection.immutable.TreeSet
+import akka.cluster.MemberStatus
+import scala.annotation.varargs
 
 object ReplicatorSettings {
 
@@ -98,8 +101,8 @@ object ReplicatorSettings {
 }
 
 /**
- * @param role Replicas are running on members tagged with this role.
- *   All members are used if undefined.
+ * @param roles Replicas are running on members tagged with these roles.
+ *   The member must have all given roles. All members are used if empty.
  * @param gossipInterval How often the Replicator should send out gossip information.
  * @param notifySubscribersInterval How often the subscribers will be notified
  *   of changes, if any.
@@ -124,7 +127,7 @@ object ReplicatorSettings {
  *        in the `Set`.
  */
 final class ReplicatorSettings(
-  val role:                           Option[String],
+  val roles:                          Set[String],
   val gossipInterval:                 FiniteDuration,
   val notifySubscribersInterval:      FiniteDuration,
   val maxDeltaElements:               Int,
@@ -138,10 +141,29 @@ final class ReplicatorSettings(
   val deltaCrdtEnabled:               Boolean,
   val maxDeltaSize:                   Int) {
 
+  // for backwards compatibility
+  def this(
+    role:                           Option[String],
+    gossipInterval:                 FiniteDuration,
+    notifySubscribersInterval:      FiniteDuration,
+    maxDeltaElements:               Int,
+    dispatcher:                     String,
+    pruningInterval:                FiniteDuration,
+    maxPruningDissemination:        FiniteDuration,
+    durableStoreProps:              Either[(String, Config), Props],
+    durableKeys:                    Set[KeyId],
+    pruningMarkerTimeToLive:        FiniteDuration,
+    durablePruningMarkerTimeToLive: FiniteDuration,
+    deltaCrdtEnabled:               Boolean,
+    maxDeltaSize:                   Int) =
+    this(role.toSet, gossipInterval, notifySubscribersInterval, maxDeltaElements, dispatcher, pruningInterval,
+      maxPruningDissemination, durableStoreProps, durableKeys, pruningMarkerTimeToLive, durablePruningMarkerTimeToLive,
+      deltaCrdtEnabled, maxDeltaSize)
+
   // For backwards compatibility
   def this(role: Option[String], gossipInterval: FiniteDuration, notifySubscribersInterval: FiniteDuration,
            maxDeltaElements: Int, dispatcher: String, pruningInterval: FiniteDuration, maxPruningDissemination: FiniteDuration) =
-    this(role, gossipInterval, notifySubscribersInterval, maxDeltaElements, dispatcher, pruningInterval,
+    this(roles = role.toSet, gossipInterval, notifySubscribersInterval, maxDeltaElements, dispatcher, pruningInterval,
       maxPruningDissemination, Right(Props.empty), Set.empty, 6.hours, 10.days, true, 200)
 
   // For backwards compatibility
@@ -161,9 +183,20 @@ final class ReplicatorSettings(
       maxPruningDissemination, durableStoreProps, durableKeys, pruningMarkerTimeToLive, durablePruningMarkerTimeToLive,
       deltaCrdtEnabled, 200)
 
-  def withRole(role: String): ReplicatorSettings = copy(role = ReplicatorSettings.roleOption(role))
+  def withRole(role: String): ReplicatorSettings = copy(roles = ReplicatorSettings.roleOption(role).toSet)
 
-  def withRole(role: Option[String]): ReplicatorSettings = copy(role = role)
+  def withRole(role: Option[String]): ReplicatorSettings = copy(roles = role.toSet)
+
+  @varargs
+  def withRoles(roles: String*): ReplicatorSettings = copy(roles = roles.toSet)
+
+  /**
+   * INTERNAL API
+   */
+  @InternalApi private[akka] def withRoles(roles: Set[String]): ReplicatorSettings = copy(roles = roles)
+
+  // for backwards compatibility
+  def role: Option[String] = roles.headOption
 
   def withGossipInterval(gossipInterval: FiniteDuration): ReplicatorSettings =
     copy(gossipInterval = gossipInterval)
@@ -216,7 +249,7 @@ final class ReplicatorSettings(
     copy(maxDeltaSize = maxDeltaSize)
 
   private def copy(
-    role:                           Option[String]                  = role,
+    roles:                          Set[String]                     = roles,
     gossipInterval:                 FiniteDuration                  = gossipInterval,
     notifySubscribersInterval:      FiniteDuration                  = notifySubscribersInterval,
     maxDeltaElements:               Int                             = maxDeltaElements,
@@ -229,7 +262,7 @@ final class ReplicatorSettings(
     durablePruningMarkerTimeToLive: FiniteDuration                  = durablePruningMarkerTimeToLive,
     deltaCrdtEnabled:               Boolean                         = deltaCrdtEnabled,
     maxDeltaSize:                   Int                             = maxDeltaSize): ReplicatorSettings =
-    new ReplicatorSettings(role, gossipInterval, notifySubscribersInterval, maxDeltaElements, dispatcher,
+    new ReplicatorSettings(roles, gossipInterval, notifySubscribersInterval, maxDeltaElements, dispatcher,
       pruningInterval, maxPruningDissemination, durableStoreProps, durableKeys,
       pruningMarkerTimeToLive, durablePruningMarkerTimeToLive, deltaCrdtEnabled, maxDeltaSize)
 }
@@ -989,8 +1022,8 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
 
   require(!cluster.isTerminated, "Cluster node must not be terminated")
   require(
-    role.forall(cluster.selfRoles.contains),
-    s"This cluster member [${selfAddress}] doesn't have the role [$role]")
+    roles.subsetOf(cluster.selfRoles),
+    s"This cluster member [${selfAddress}] doesn't have all the roles [${roles.mkString(", ")}]")
 
   //Start periodic gossip to random nodes in cluster
   import context.dispatcher
@@ -1058,8 +1091,10 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
   var weaklyUpNodes: Set[Address] = Set.empty
 
   var removedNodes: Map[UniqueAddress, Long] = Map.empty
-  var leader: Option[Address] = None
-  def isLeader: Boolean = leader.exists(_ == selfAddress)
+  // all nodes sorted with the leader first
+  var leader: TreeSet[Member] = TreeSet.empty(Member.leaderStatusOrdering)
+  def isLeader: Boolean =
+    leader.nonEmpty && leader.head.address == selfAddress && leader.head.status == MemberStatus.Up
 
   // for pruning timeouts are based on clock that is only increased when all nodes are reachable
   var previousClockTime = System.nanoTime()
@@ -1100,9 +1135,9 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
   override def preStart(): Unit = {
     if (hasDurableKeys)
       durableStore ! LoadAll
-    val leaderChangedClass = if (role.isDefined) classOf[RoleLeaderChanged] else classOf[LeaderChanged]
+    // not using LeaderChanged/RoleLeaderChanged because here we need one node independent of data center
     cluster.subscribe(self, initialStateMode = InitialStateAsEvents,
-      classOf[MemberEvent], classOf[ReachabilityEvent], leaderChangedClass)
+      classOf[MemberEvent], classOf[ReachabilityEvent])
   }
 
   override def postStop(): Unit = {
@@ -1114,7 +1149,7 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
     clockTask.cancel()
   }
 
-  def matchingRole(m: Member): Boolean = role.forall(m.hasRole)
+  def matchingRole(m: Member): Boolean = roles.subsetOf(m.roles)
 
   override val supervisorStrategy = {
     def fromDurableStore: Boolean = sender() == durableStore && sender() != context.system.deadLetters
@@ -1205,11 +1240,9 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
     case MemberWeaklyUp(m)                      ⇒ receiveWeaklyUpMemberUp(m)
     case MemberUp(m)                            ⇒ receiveMemberUp(m)
     case MemberRemoved(m, _)                    ⇒ receiveMemberRemoved(m)
-    case _: MemberEvent                         ⇒ // not of interest
+    case evt: MemberEvent                       ⇒ receiveOtherMemberEvent(evt.member)
     case UnreachableMember(m)                   ⇒ receiveUnreachable(m)
     case ReachableMember(m)                     ⇒ receiveReachable(m)
-    case LeaderChanged(leader)                  ⇒ receiveLeaderChanged(leader, None)
-    case RoleLeaderChanged(role, leader)        ⇒ receiveLeaderChanged(leader, Some(role))
     case GetKeyIds                              ⇒ receiveGetKeyIds()
     case Delete(key, consistency, req)          ⇒ receiveDelete(key, consistency, req)
     case RemovedNodePruningTick                 ⇒ receiveRemovedNodePruningTick()
@@ -1696,15 +1729,20 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
       weaklyUpNodes += m.address
 
   def receiveMemberUp(m: Member): Unit =
-    if (matchingRole(m) && m.address != selfAddress) {
+    if (matchingRole(m)) {
+      leader += m
+      if (m.address != selfAddress) {
         nodes += m.address
         weaklyUpNodes -= m.address
       }
+    }
 
   def receiveMemberRemoved(m: Member): Unit = {
     if (m.address == selfAddress)
       context stop self
     else if (matchingRole(m)) {
+      // filter, it's possible that the ordering is changed since it based on MemberStatus
+      leader = leader.filterNot(_.uniqueAddress == m.uniqueAddress)
       nodes -= m.address
       weaklyUpNodes -= m.address
       log.debug("adding removed node [{}] from MemberRemoved", m.uniqueAddress)
@@ -1714,15 +1752,19 @@ final class Replicator(settings: ReplicatorSettings) extends Actor with ActorLog
     }
   }
 
+  def receiveOtherMemberEvent(m: Member): Unit =
+    if (matchingRole(m)) {
+      // replace, it's possible that the ordering is changed since it based on MemberStatus
+      leader = leader.filterNot(_.uniqueAddress == m.uniqueAddress)
+      leader += m
+    }
+
   def receiveUnreachable(m: Member): Unit =
     if (matchingRole(m)) unreachable += m.address
 
   def receiveReachable(m: Member): Unit =
     if (matchingRole(m)) unreachable -= m.address
 
-  def receiveLeaderChanged(leaderOption: Option[Address], roleOption: Option[String]): Unit =
-    if (roleOption == role) leader = leaderOption
-
   def receiveClockTick(): Unit = {
     val now = System.nanoTime()
     if (unreachable.isEmpty)

akka-distributed-data/src/multi-jvm/scala/akka/cluster/ddata/DurablePruningSpec.scala

@@ -16,6 +16,7 @@ import com.typesafe.config.ConfigFactory
 import akka.actor.ActorSystem
 import akka.actor.ActorRef
 import scala.concurrent.Await
+import akka.cluster.MemberStatus
 
 object DurablePruningSpec extends MultiNodeConfig {
   val first = role("first")
@@ -73,6 +74,13 @@ class DurablePruningSpec extends MultiNodeSpec(DurablePruningSpec) with STMultiN
       val replicator2 = startReplicator(sys2)
       val probe2 = TestProbe()(sys2)
       Cluster(sys2).join(node(first).address)
+      awaitAssert({
+        Cluster(system).state.members.size should ===(4)
+        Cluster(system).state.members.map(_.status) should ===(Set(MemberStatus.Up))
+        Cluster(sys2).state.members.size should ===(4)
+        Cluster(sys2).state.members.map(_.status) should ===(Set(MemberStatus.Up))
+      }, 10.seconds)
+      enterBarrier("joined")
 
       within(5.seconds) {
         awaitAssert {

akka-docs/src/main/paradox/java/cluster-dc.md

@@ -0,0 +1 @@
+../scala/cluster-dc.md

akka-docs/src/main/paradox/scala/cluster-dc.md

@@ -0,0 +1,191 @@
+# Cluster across multiple data centers
+
+This chapter describes how @ref[Akka Cluster](cluster-usage.md) can be used across
+multiple data centers, availability zones or regions.
+
+## Motivation
+
+There can be many reasons for using more than one data center, such as:
+
+* Redundancy to tolerate failures in one location and still be operational.
+* Serve request from a location near the user to provide better responsiveness.
+* Balance the load over many servers.
+
+It's possible to run an ordinary Akka Cluster with default settings that spans multiple
+data centers but that may result in problems like:
+
+* Management of Cluster membership is stalled during network partitions as described in a 
+  separate section below. This means that nodes would not be able to be added and removed 
+  during network partitions across data centers.  
+* More frequent false positive failure detection for network connections across data centers.
+  It's not possible to have different settings for the failure detection within vs. across
+  data centers.
+* Downing/removal of nodes in the case of network partitions should typically be treated
+  differently for failures within vs. across data centers. For network partitions across the 
+  system should typically not down the unreachable nodes, but instead wait until it heals or
+  an decision is made by a human or external monitoring system. For failures within same
+  data center automatic, more aggressive, downing mechanisms can be employed for quick fail over.
+* Quick fail over of Cluster Singleton and Cluster Sharding from one data center to another
+  is difficult to do in a safe way. There is a risk that singletons or sharded entities become 
+  active on both sides of a network partition.
+* Lack of location information makes it difficult to optimize communication to prefer nodes
+  that are close over distant nodes. E.g. a cluster aware router would be more efficient
+  if it would prefer routing messages to nodes in the own data center. 
+
+To avoid some of these problems one can run a separate Akka Cluster per data center and use another
+communication channel between the data centers, such as HTTP, an external message broker or
+[Cluster Client](cluster-singleton.md). However, many of the nice tools that are built on
+top of the Cluster membership information are lost. For example, it wouldn't be possible
+to use [Distributed Data](distributed-data.md) across the separate clusters.
+
+We often recommend implementing a micro-service as one Akka Cluster. The external API of the
+service would be HTTP or a message broker, and not Akka Remoting or Cluster, but the internal
+communication within the service that is running on several nodes would use ordinary actor 
+messaging or the tools based on Akka Cluster. When deploying this service to multiple data
+centers it would be inconvenient if the internal communication could not use ordinary actor 
+messaging because it was separated into several Akka Clusters. The benefit of using Akka
+messaging internally is performance as well as ease of development and reasoning about
+your domain in terms of Actors.
+
+Therefore, it's possible to make the Akka Cluster aware of data centers so that one Akka
+Cluster can span multiple data centers and still be tolerant to network partitions.
+
+## Defining the data centers
+
+The features are based on the idea that nodes can be assigned to a group of nodes
+by setting the `akka.cluster.multi-data-center.self-data-center` configuration property.
+A node can only belong to one data center and if nothing is specified a node will belong 
+to the `default` data center.
+
+The grouping of nodes is not limited to the physical boundaries of data centers,
+even though that is the primary use case. It could also be used as a logical grouping
+for other reasons, such as isolation of certain nodes to improve stability or splitting
+up a large cluster into smaller groups of nodes for better scalability. 
+
+## Membership
+
+Some @ref[membership transitions](common/cluster.md#membership-lifecycle) are managed by 
+one node called the @ref[leader](common/cluster.md#leader). There is one leader per data center
+and it is responsible for these transitions for the members within the same data center. Members of
+other data centers are managed independently by leader of the respective data center. These actions
+cannot be performed while there are any unreachability observations among the nodes in the data center, 
+but unreachability across different data centers don't influence the progress of membership management
+within a data center. Nodes can be added and removed also when there are network partitions between
+data centers, which is impossible if nodes are not grouped into data centers.
+
+![cluster-dc.png](../images/cluster-dc.png)
+
+User actions like joining, leaving, and downing can be sent to any node in the cluster,
+not only to the nodes in the data center of the node. Seed nodes are also global.
+
+The data center membership is implemented by adding the data center name prefixed with `"dc-"` to the 
+@ref[roles](cluster-usage.md#node-roles) of the member and thereby this information is known
+by all other members in the cluster. This is an implementation detail, but it can be good to know
+if you see this in log messages.
+
+You can retrieve information about what data center a member belongs to:
+
+Scala
+:  @@snip [ClusterDocSpec.scala]($code$/scala/docs/cluster/ClusterDocSpec.scala) { #dcAccess }
+
+Java
+:  @@snip [ClusterDocTest.java]($code$/java/jdocs/cluster/ClusterDocTest.java) { #dcAccess }
+
+## Failure Detection
+
+@ref[Failure detection](cluster-usage.md#failure-detector) is performed by sending heartbeat messages
+to detect if a node is unreachable. This is done more frequently and with more certainty among
+the nodes in the same data center than across data centers. The failure detection across different data centers should
+be interpreted as an indication of problem with the network link between the data centers.
+
+Two different failure detectors can be configured for these two purposes:
+
+* `akka.cluster.failure-detector` for failure detection within own data center
+* `akka.cluster.multi-data-center.failure-detector` for failure detection across different data centers
+
+When @ref[subscribing to cluster events](cluster-usage.md#cluster-subscriber) the `UnreachableMember` and
+`ReachableMember` events are for observations within the own data center. The same data center as where the
+subscription was registered.
+
+For cross data center unreachability notifications you can subscribe to `UnreachableDataCenter` and `ReachableDataCenter`
+events.
+
+Heartbeat messages for failure detection across data centers are only performed between a number of the
+oldest nodes on each side. The number of nodes is configured with `akka.cluster.multi-data-center.cross-data-center-connections`.
+The reason for only using a limited number of nodes is to keep the number of connections across data
+centers low. The same nodes are also used for the gossip protocol when disseminating the membership
+information across data centers. Within a data center all nodes are involved in gossip and failure detection.
+
+This influence how rolling upgrades should be performed. Don't stop all of the oldest that are used for gossip
+at the same time. Stop one or a few at a time so that new nodes can take over the responsibility.
+It's best to leave the oldest nodes until last.
+
+## Cluster Singleton
+
+The [Cluster Singleton](cluster-singleton.md) is a singleton per data center. If you start the 
+`ClusterSingletonManager` on all nodes and you have defined 3 different data centers there will be
+3 active singleton instances in the cluster, one in each data center. This is taken care of automatically,
+but is important to be aware of. Designing the system for one singleton per data center makes it possible
+for the system to be available also during network partitions between data centers.
+
+The reason why the singleton is per data center and not global is that membership information is not
+guaranteed to be consistent across data centers when using one leader per data center and that makes it
+difficult to select a single global singleton. 
+
+If you need a global singleton you have to pick one data center to host that singleton and only start the
+`ClusterSingletonManager` on nodes of that data center. If the data center is unreachable from another data center the
+singleton is inaccessible, which is a reasonable trade-off when selecting consistency over availability.
+
+The `ClusterSingletonProxy` is by default routing messages to the singleton in the own data center, but
+it can be started with a `data-center` parameter in the `ClusterSingletonProxySettings` to define that 
+it should route messages to a singleton located in another data center. That is useful for example when
+having a global singleton in one data center and accessing it from other data centers.
+
+This is how to create a singleton proxy for a specific data center:
+
+Scala
+:  @@snip [ClusterSingletonManagerSpec.scala]($akka$/akka-cluster-tools/src/multi-jvm/scala/akka/cluster/singleton/ClusterSingletonManagerSpec.scala) { #create-singleton-proxy-dc }
+
+Java
+:  @@snip [ClusterSingletonManagerTest.java]($akka$/akka-cluster-tools/src/test/java/akka/cluster/singleton/ClusterSingletonManagerTest.java) { #create-singleton-proxy-dc }
+
+If using the own data center as the `withDataCenter` parameter that would be a proxy for the singleton in the own data center, which
+is also the default if `withDataCenter` is not given.
+
+## Cluster Sharding
+
+The coordinator in [Cluster Sharding](cluster-sharding.md) is a Cluster Singleton and therefore,
+as explained above, Cluster Sharding is also per data center. Each data center will have its own coordinator
+and regions, isolated from other data centers. If you start an entity type with the same name on all 
+nodes and you have defined 3 different data centers and then send messages to the same entity id to
+sharding regions in all data centers you will end up with 3 active entity instances for that entity id,
+one in each data center. 
+
+Especially when used together with Akka Persistence that is based on the single-writer principle
+it is important to avoid running the same entity at multiple locations at the same time with a
+shared data store. Lightbend's Akka Team is working on a solution that will support multiple active
+entities that will work nicely together with Cluster Sharding across multiple data centers.
+
+If you need global entities you have to pick one data center to host that entity type and only start
+`ClusterSharding` on nodes of that data center. If the data center is unreachable from another data center the
+entities are inaccessible, which is a reasonable trade-off when selecting consistency over availability.
+
+The Cluster Sharding proxy is by default routing messages to the shard regions in the own data center, but
+it can be started with a `data-center` parameter to define that it should route messages to a shard region
+located in another data center. That is useful for example when having global entities in one data center and 
+accessing them from other data centers.
+
+This is how to create a sharding proxy for a specific data center:
+
+Scala
+:  @@snip [ClusterShardingSpec.scala]($akka$/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sharding/ClusterShardingSpec.scala) { #proxy-dc }
+
+Java
+:  @@snip [ClusterShardingTest.java]($code$/java/jdocs/sharding/ClusterShardingTest.java) { #proxy-dc }
+
+Another way to manage global entities is to make sure that certain entity ids are located in 
+only one data center by routing the messages to the right region. For example, the routing function
+could be that odd entity ids are routed to data center A and even entity ids to data center B.
+Before sending the message to the local region actor you make the decision of which data center it should
+be routed to. Messages for another data center can be sent with a sharding proxy as explained above and
+messages for the own data center are sent to the local region.

akka-docs/src/main/paradox/scala/cluster-sharding.md

@@ -36,7 +36,7 @@ See @ref:[Downing](cluster-usage.md#automatic-vs-manual-downing).
 This is how an entity actor may look like:
 
 Scala
-:  @@snip [ClusterShardingSpec.scala]($akka$/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sharding/ClusterShardingSpec.scala) { #counter-actor }
+:  @@snip [ClusterShardingSpec.scala]($akka$/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sharding/ClusterShardingSpec.scala) { #proxy-dc }
 
 Java
 :  @@snip [ClusterShardingTest.java]($code$/java/jdocs/sharding/ClusterShardingTest.java) { #counter-actor }

akka-docs/src/main/paradox/scala/index-network.md

@@ -12,6 +12,7 @@
 * [cluster-sharding](cluster-sharding.md)
 * [cluster-metrics](cluster-metrics.md)
 * [distributed-data](distributed-data.md)
+* [cluster-dc](cluster-dc.md)
 * [remoting](remoting.md)
 * [remoting-artery](remoting-artery.md)
 * [serialization](serialization.md)

akka-docs/src/test/java/jdocs/cluster/ClusterDocTest.java

@@ -5,6 +5,7 @@ package jdocs.cluster;
 
 import akka.testkit.javadsl.TestKit;
 import com.typesafe.config.ConfigFactory;
+import java.util.Set;
 import jdocs.AbstractJavaTest;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -12,6 +13,7 @@ import org.junit.Test;
 
 import akka.actor.ActorSystem;
 import akka.cluster.Cluster;
+import akka.cluster.Member;
 
 
 public class ClusterDocTest extends AbstractJavaTest {
@@ -39,4 +41,19 @@ public class ClusterDocTest extends AbstractJavaTest {
 
   }
   
+  // compile only 
+  @SuppressWarnings("unused")
+  public void demonstrateDataCenter() {
+    //#dcAccess
+    final Cluster cluster = Cluster.get(system);
+    // this node's data center
+    String dc = cluster.selfDataCenter();
+    // all known data centers
+    Set<String> allDc = cluster.state().getAllDataCenters();
+    // a specific member's data center
+    Member aMember = cluster.state().getMembers().iterator().next();
+    String aDc = aMember.dataCenter();
+    //#dcAccess
+  }
+
 }

akka-docs/src/test/java/jdocs/sharding/ClusterShardingTest.java

@@ -6,6 +6,7 @@ package jdocs.sharding;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 
+import java.util.Optional;
 import scala.concurrent.duration.Duration;
 
 import akka.actor.AbstractActor;
@@ -101,6 +102,15 @@ public class ClusterShardingTest {
     ClusterSharding.get(system).start("SupervisedCounter",
         Props.create(CounterSupervisor.class), settings, messageExtractor);
     //#counter-supervisor-start
+
+    //#proxy-dc
+    ActorRef counterProxyDcB =
+      ClusterSharding.get(system).startProxy(
+        "Counter",
+        Optional.empty(),
+        Optional.of("B"), // data center name
+        messageExtractor);
+    //#proxy-dc
   }
 
   public void demonstrateUsage2() {

akka-docs/src/test/scala/docs/cluster/ClusterDocSpec.scala

@@ -5,6 +5,7 @@ package scala.docs.cluster
 
 import akka.cluster.Cluster
 import akka.testkit.AkkaSpec
+import docs.CompileOnlySpec
 
 object ClusterDocSpec {
 
@@ -15,13 +16,28 @@ object ClusterDocSpec {
     """
 }
 
-class ClusterDocSpec extends AkkaSpec(ClusterDocSpec.config) {
+class ClusterDocSpec extends AkkaSpec(ClusterDocSpec.config) with CompileOnlySpec {
 
-  "demonstrate leave" in {
+  "demonstrate leave" in compileOnlySpec {
     //#leave
     val cluster = Cluster(system)
     cluster.leave(cluster.selfAddress)
     //#leave
   }
 
+  "demonstrate data center" in compileOnlySpec {
+    {
+      //#dcAccess
+      val cluster = Cluster(system)
+      // this node's data center
+      val dc = cluster.selfDataCenter
+      // all known data centers
+      val allDc = cluster.state.allDataCenters
+      // a specific member's data center
+      val aMember = cluster.state.members.head
+      val aDc = aMember.dataCenter
+      //#dcAccess
+    }
+  }
+
 }