=clu #13802 Introduce max-total-nr-of-instances for cluster aware routers

akka-cluster-metrics/src/main/scala/akka/cluster/metrics/ClusterMetricsRouting.scala

@@ -21,6 +21,7 @@ import akka.cluster.ClusterEvent.CurrentClusterState
 import akka.dispatch.Dispatchers
 import akka.japi.Util.immutableSeq
 import akka.routing._
+import akka.cluster.routing.ClusterRouterSettingsBase
 
 /**
  * Load balancing of messages to cluster nodes based on cluster metric data.
@@ -128,7 +129,7 @@ final case class AdaptiveLoadBalancingPool(
   extends Pool {
 
   def this(config: Config, dynamicAccess: DynamicAccess) =
-    this(nrOfInstances = config.getInt("nr-of-instances"),
+    this(nrOfInstances = ClusterRouterSettingsBase.getMaxTotalNrOfInstances(config),
       metricsSelector = MetricsSelector.fromConfig(config, dynamicAccess),
       usePoolDispatcher = config.hasPath("pool-dispatcher"))
 

akka-cluster/src/main/resources/reference.conf

@@ -208,12 +208,20 @@ akka {
 
     # Maximum number of routees that will be deployed on each cluster
     # member node.
-    # Note that nr-of-instances defines total number of routees, but
+    # Note that max-total-nr-of-instances defines total number of routees, but
     # number of routees per node will not be exceeded, i.e. if you
-    # define nr-of-instances = 50 and max-nr-of-instances-per-node = 2
+    # define max-total-nr-of-instances = 50 and max-nr-of-instances-per-node = 2
     # it will deploy 2 routees per new member in the cluster, up to
     # 25 members.
     max-nr-of-instances-per-node = 1
+    
+    # Maximum number of routees that will be deployed, in total
+    # on all nodes. See also description of max-nr-of-instances-per-node.
+    # For backwards compatibility reasons, nr-of-instances
+    # has the same purpose as max-total-nr-of-instances for cluster
+    # aware routers and nr-of-instances (if defined by user) takes
+    # precedence over max-total-nr-of-instances. 
+    max-total-nr-of-instances = 10000
 
     # Defines if routees are allowed to be located on the same node as
     # the head router actor, or only on remote nodes.

akka-cluster/src/main/scala/akka/cluster/ClusterActorRefProvider.scala

@@ -11,6 +11,7 @@ import akka.remote.{ RemoteActorRefProvider, RemoteDeployer }
 import akka.remote.routing.RemoteRouterConfig
 import akka.routing.{ Group, Pool }
 import com.typesafe.config.Config
+import com.typesafe.config.ConfigFactory
 
 /**
  * INTERNAL API
@@ -60,9 +61,19 @@ private[akka] class ClusterActorRefProvider(
  * Deployer of cluster aware routers.
  */
 private[akka] class ClusterDeployer(_settings: ActorSystem.Settings, _pm: DynamicAccess) extends RemoteDeployer(_settings, _pm) {
-  override def parseConfig(path: String, config: Config): Option[Deploy] = {
 
-    super.parseConfig(path, config) match {
+  override def parseConfig(path: String, config: Config): Option[Deploy] = {
+    // config is the user supplied section, no defaults
+    // amend it to use max-total-nr-of-instances as nr-of-instances if cluster.enabled and
+    // user has not specified nr-of-instances
+    val config2 =
+      if (config.hasPath("cluster.enabled") && config.getBoolean("cluster.enabled") && !config.hasPath("nr-of-instances")) {
+        val maxTotalNrOfInstances = config.withFallback(default).getInt("cluster.max-total-nr-of-instances")
+        ConfigFactory.parseString("nr-of-instances=" + maxTotalNrOfInstances)
+          .withFallback(config)
+      } else config
+
+    super.parseConfig(path, config2) match {
       case d @ Some(deploy) ⇒
         if (deploy.config.getBoolean("cluster.enabled")) {
           if (deploy.scope != NoScopeGiven)

akka-cluster/src/main/scala/akka/cluster/routing/AdaptiveLoadBalancing.scala

@@ -139,7 +139,7 @@ final case class AdaptiveLoadBalancingPool(
   extends Pool {
 
   def this(config: Config, dynamicAccess: DynamicAccess) =
-    this(nrOfInstances = config.getInt("nr-of-instances"),
+    this(nrOfInstances = ClusterRouterSettingsBase.getMaxTotalNrOfInstances(config),
       metricsSelector = MetricsSelector.fromConfig(config, dynamicAccess),
       usePoolDispatcher = config.hasPath("pool-dispatcher"))
 

akka-cluster/src/main/scala/akka/cluster/routing/ClusterRouterConfig.scala

@@ -32,7 +32,7 @@ import scala.collection.immutable
 object ClusterRouterGroupSettings {
   def fromConfig(config: Config): ClusterRouterGroupSettings =
     ClusterRouterGroupSettings(
-      totalInstances = config.getInt("nr-of-instances"),
+      totalInstances = ClusterRouterSettingsBase.getMaxTotalNrOfInstances(config),
       routeesPaths = immutableSeq(config.getStringList("routees.paths")),
       allowLocalRoutees = config.getBoolean("cluster.allow-local-routees"),
       useRole = ClusterRouterSettingsBase.useRoleOption(config.getString("cluster.use-role")))
@@ -69,7 +69,7 @@ final case class ClusterRouterGroupSettings(
 object ClusterRouterPoolSettings {
   def fromConfig(config: Config): ClusterRouterPoolSettings =
     ClusterRouterPoolSettings(
-      totalInstances = config.getInt("nr-of-instances"),
+      totalInstances = ClusterRouterSettingsBase.getMaxTotalNrOfInstances(config),
       maxInstancesPerNode = config.getInt("cluster.max-nr-of-instances-per-node"),
       allowLocalRoutees = config.getBoolean("cluster.allow-local-routees"),
       useRole = ClusterRouterSettingsBase.useRoleOption(config.getString("cluster.use-role")))
@@ -105,6 +105,18 @@ private[akka] object ClusterRouterSettingsBase {
     case null | "" ⇒ None
     case _         ⇒ Some(role)
   }
+
+  /**
+   * For backwards compatibility reasons, nr-of-instances
+   * has the same purpose as max-total-nr-of-instances for cluster
+   * aware routers and nr-of-instances (if defined by user) takes
+   * precedence over max-total-nr-of-instances.
+   */
+  def getMaxTotalNrOfInstances(config: Config): Int =
+    config.getInt("nr-of-instances") match {
+      case 1 | 0 ⇒ config.getInt("cluster.max-nr-of-instances-per-node")
+      case other ⇒ other
+    }
 }
 
 /**

akka-cluster/src/multi-jvm/scala/akka/cluster/StressSpec.scala

@@ -141,7 +141,6 @@ private[cluster] object StressMultiJvmSpec extends MultiNodeConfig {
     akka.actor.deployment {
       /master-node-1/workers {
         router = round-robin-pool
-        nr-of-instances = 100
         cluster {
           enabled = on
           max-nr-of-instances-per-node = 1
@@ -150,7 +149,6 @@ private[cluster] object StressMultiJvmSpec extends MultiNodeConfig {
       }
       /master-node-2/workers {
         router = round-robin-group
-        nr-of-instances = 100
         routees.paths = ["/user/worker"]
         cluster {
           enabled = on
@@ -159,7 +157,6 @@ private[cluster] object StressMultiJvmSpec extends MultiNodeConfig {
       }
       /master-node-3/workers = {
         router = adaptive-pool
-        nr-of-instances = 100
         cluster {
           enabled = on
           max-nr-of-instances-per-node = 1

akka-cluster/src/multi-jvm/scala/akka/cluster/routing/AdaptiveLoadBalancingRouterSpec.scala

@@ -70,10 +70,10 @@ object AdaptiveLoadBalancingRouterMultiJvmSpec extends MultiNodeConfig {
         /router4 = {
           router = adaptive-pool
           metrics-selector = "akka.cluster.routing.TestCustomMetricsSelector"
-          nr-of-instances = 10
           cluster {
             enabled = on
             max-nr-of-instances-per-node = 2
+            max-total-nr-of-instances = 10
           }
         }
       }

akka-cluster/src/multi-jvm/scala/akka/cluster/routing/ClusterConsistentHashingRouterSpec.scala

@@ -40,10 +40,10 @@ object ClusterConsistentHashingRouterMultiJvmSpec extends MultiNodeConfig {
     withFallback(ConfigFactory.parseString(s"""
       common-router-settings = {
         router = consistent-hashing-pool
-        nr-of-instances = 10
         cluster {
           enabled = on
           max-nr-of-instances-per-node = 2
+          max-total-nr-of-instances = 10
         }
       }
 

akka-cluster/src/multi-jvm/scala/akka/cluster/routing/ClusterRoundRobinSpec.scala

@@ -52,33 +52,33 @@ object ClusterRoundRobinMultiJvmSpec extends MultiNodeConfig {
       akka.actor.deployment {
         /router1 {
           router = round-robin-pool
-          nr-of-instances = 10
           cluster {
             enabled = on
             max-nr-of-instances-per-node = 2
+            max-total-nr-of-instances = 10
           }
         }
         /router3 {
           router = round-robin-pool
-          nr-of-instances = 10
           cluster {
             enabled = on
             max-nr-of-instances-per-node = 1
+            max-total-nr-of-instances = 10
             allow-local-routees = off
           }
         }
         /router4 {
           router = round-robin-group
-          nr-of-instances = 10
           routees.paths = ["/user/myserviceA", "/user/myserviceB"]
           cluster.enabled = on
+          cluster.max-total-nr-of-instances = 10
         }
         /router5 {
           router = round-robin-pool
-          nr-of-instances = 10
           cluster {
             enabled = on
             use-role = a
+            max-total-nr-of-instances = 10
           }
         }
       }

akka-cluster/src/test/scala/akka/cluster/ClusterDeployerSpec.scala

@@ -18,18 +18,18 @@ object ClusterDeployerSpec {
       akka.actor.deployment {
         /user/service1 {
           router = round-robin-pool
-          nr-of-instances = 20
           cluster.enabled = on
           cluster.max-nr-of-instances-per-node = 3
+          cluster.max-total-nr-of-instances = 20
           cluster.allow-local-routees = off
         }
         /user/service2 {
           dispatcher = mydispatcher
           mailbox = mymailbox
           router = round-robin-group
-          nr-of-instances = 20
           routees.paths = ["/user/myservice"]
           cluster.enabled = on
+          cluster.max-total-nr-of-instances = 20
           cluster.allow-local-routees = off
         }
       }

akka-docs/rst/java/cluster-usage.rst

@@ -467,8 +467,9 @@ actors to which the messages will be forwarded to by the router.
 Messages will be forwarded to the routees using :ref:`ActorSelection <actorSelection-java>`, so the same delivery semantics should be expected.
 It is possible to limit the lookup of routees to member nodes tagged with a certain role by specifying ``use-role``.
 
-``nr-of-instances`` defines total number of routees in the cluster. Setting ``nr-of-instances`` 
+``max-total-nr-of-instances`` defines total number of routees in the cluster. By default ``max-total-nr-of-instances`` 
-to a high value will result in new routees added to the router when nodes join the cluster.
+is set to a high value (10000) that will result in new routees added to the router when nodes join the cluster.
+Set it to a lower value if you want to limit total number of routees.
 
 The same type of router could also have been defined in code:
 
@@ -528,10 +529,10 @@ the configuration for a router looks like this:
 It is possible to limit the deployment of routees to member nodes tagged with a certain role by
 specifying ``use-role``.
 
-``nr-of-instances`` defines total number of routees in the cluster, but the number of routees
+``max-total-nr-of-instances`` defines total number of routees in the cluster, but the number of routees
-per node, ``max-nr-of-instances-per-node``, will not be exceeded. Setting ``nr-of-instances``
+per node, ``max-nr-of-instances-per-node``, will not be exceeded. By default ``max-total-nr-of-instances`` 
-to a high value will result in creating and deploying additional routees when new nodes join
+is set to a high value (10000) that will result in new routees added to the router when nodes join the cluster.
-the cluster.
+Set it to a lower value if you want to limit total number of routees.
 
 The same type of router could also have been defined in code:
 

akka-docs/rst/project/migration-guide-2.3.x-2.4.x.rst

@@ -155,6 +155,21 @@ In order to make cluster routers smarter about when they can start local routees
 In case you have implemented a custom Group you will have to update the method's signature,
 however the implementation can remain the same if you don't need to rely on an ActorSystem in your logic.
 
+Cluster aware router max-total-nr-of-instances
+==============================================
+
+In 2.3.x the deployment configuration property ``nr-of-instances`` was used for
+cluster aware routers to specify total number of routees in the cluster.
+This was confusing, especially since the default value is 1.
+
+In 2.4.x there is a new deployement property ``cluster.max-total-nr-of-instances`` that 
+defines total number of routees in the cluster. By default ``max-total-nr-of-instances`` 
+is set to a high value (10000) that will result in new routees added to the router when nodes join the cluster.
+Set it to a lower value if you want to limit total number of routees.
+
+For backwards compatibility reasons ``nr-of-instances`` is still used if defined by user,
+i.e. if defined it takes precedence over ``max-total-nr-of-instances``.
+
 Logger names use full class name 
 ================================
 Previously, few places in akka used "simple" logger names, such as ``Cluster`` or ``Remoting``.

akka-docs/rst/scala/cluster-usage.rst

@@ -464,8 +464,9 @@ actors to which the messages will be forwarded to by the router.
 Messages will be forwarded to the routees using :ref:`ActorSelection <actorSelection-scala>`, so the same delivery semantics should be expected.
 It is possible to limit the lookup of routees to member nodes tagged with a certain role by specifying ``use-role``.
 
-``nr-of-instances`` defines total number of routees in the cluster. Setting ``nr-of-instances`` 
+``max-total-nr-of-instances`` defines total number of routees in the cluster. By default ``max-total-nr-of-instances`` 
-to a high value will result in new routees added to the router when nodes join the cluster.
+is set to a high value (10000) that will result in new routees added to the router when nodes join the cluster.
+Set it to a lower value if you want to limit total number of routees.
 
 The same type of router could also have been defined in code:
 
@@ -523,10 +524,10 @@ the configuration for a router looks like this:
 It is possible to limit the deployment of routees to member nodes tagged with a certain role by
 specifying ``use-role``.
 
-``nr-of-instances`` defines total number of routees in the cluster, but the number of routees
+``max-total-nr-of-instances`` defines total number of routees in the cluster, but the number of routees
-per node, ``max-nr-of-instances-per-node``, will not be exceeded. Setting ``nr-of-instances``
+per node, ``max-nr-of-instances-per-node``, will not be exceeded. By default ``max-total-nr-of-instances`` 
-to a high value will result in creating and deploying additional routees when new nodes join
+is set to a high value (10000) that will result in new routees added to the router when nodes join the cluster.
-the cluster.
+Set it to a lower value if you want to limit total number of routees.
 
 The same type of router could also have been defined in code:
 

akka-samples/akka-sample-cluster-java/src/main/resources/factorial.conf

@@ -22,7 +22,6 @@ akka.actor.deployment {
     # metrics-selector = cpu
     metrics-selector = mix
     #
-    nr-of-instances = 100
     routees.paths = ["/user/factorialBackend"]
     cluster {
       enabled = on

akka-samples/akka-sample-cluster-java/src/main/resources/stats1.conf

@@ -4,7 +4,6 @@ include "application"
 akka.actor.deployment {
   /statsService/workerRouter {
     router = consistent-hashing-group
-    nr-of-instances = 100
     routees.paths = ["/user/statsWorker"]
     cluster {
       enabled = on

akka-samples/akka-sample-cluster-java/src/main/resources/stats2.conf

@@ -4,7 +4,6 @@ include "application"
 akka.actor.deployment {
   /statsService/singleton/workerRouter {
     router = consistent-hashing-pool
-    nr-of-instances = 100
     cluster {
       enabled = on
       max-nr-of-instances-per-node = 3

akka-samples/akka-sample-cluster-java/src/multi-jvm/scala/sample/cluster/stats/StatsSampleSingleMasterSpec.scala

@@ -56,7 +56,6 @@ object StatsSampleSingleMasterSpecConfig extends MultiNodeConfig {
     akka.actor.deployment {
       /statsService/singleton/workerRouter {
           router = consistent-hashing-pool
-          nr-of-instances = 100
           cluster {
             enabled = on
             max-nr-of-instances-per-node = 3

akka-samples/akka-sample-cluster-java/src/multi-jvm/scala/sample/cluster/stats/StatsSampleSpec.scala

@@ -51,7 +51,6 @@ object StatsSampleSpecConfig extends MultiNodeConfig {
     akka.actor.deployment {
       /statsService/workerRouter {
           router = consistent-hashing-group
-          nr-of-instances = 100
           routees.paths = ["/user/statsWorker"]
           cluster {
             enabled = on

akka-samples/akka-sample-cluster-scala/src/main/resources/factorial.conf

@@ -22,7 +22,6 @@ akka.actor.deployment {
     # metrics-selector = cpu
     metrics-selector = mix
     #
-    nr-of-instances = 100
     routees.paths = ["/user/factorialBackend"]
     cluster {
       enabled = on

akka-samples/akka-sample-cluster-scala/src/main/resources/stats1.conf

@@ -4,7 +4,6 @@ include "application"
 akka.actor.deployment {
   /statsService/workerRouter {
     router = consistent-hashing-group
-    nr-of-instances = 100
     routees.paths = ["/user/statsWorker"]
     cluster {
       enabled = on

akka-samples/akka-sample-cluster-scala/src/main/resources/stats2.conf

@@ -4,7 +4,6 @@ include "application"
 akka.actor.deployment {
   /statsService/singleton/workerRouter {
     router = consistent-hashing-pool
-    nr-of-instances = 100
     cluster {
       enabled = on
       max-nr-of-instances-per-node = 3

akka-samples/akka-sample-cluster-scala/src/multi-jvm/scala/sample/cluster/stats/StatsSampleSingleMasterSpec.scala

@@ -55,7 +55,6 @@ object StatsSampleSingleMasterSpecConfig extends MultiNodeConfig {
     akka.actor.deployment {
       /statsService/singleton/workerRouter {
           router = consistent-hashing-pool
-          nr-of-instances = 100
           cluster {
             enabled = on
             max-nr-of-instances-per-node = 3

akka-samples/akka-sample-cluster-scala/src/multi-jvm/scala/sample/cluster/stats/StatsSampleSpec.scala

@@ -47,7 +47,6 @@ object StatsSampleSpecConfig extends MultiNodeConfig {
     akka.actor.deployment {
       /statsService/workerRouter {
           router = consistent-hashing-group
-          nr-of-instances = 100
           routees.paths = ["/user/statsWorker"]
           cluster {
             enabled = on