document remember entities limitation, #22327

* and set default for max-delta-elements

akka-cluster-sharding/src/main/resources/reference.conf

@@ -127,6 +127,12 @@ akka.cluster.sharding {
     # minCap parameter to MajorityWrite and MajorityRead consistency level.
     majority-min-cap = 5
     durable.keys = ["shard-*"]
+    
+    # When using many entities with "remember entities" the Gossip message
+    # can become to large if including to many in same message. Limit to
+    # the same number as the number of ORSet per shard.
+    akka.cluster.sharding.distributed-data.max-delta-elements = 5
+    
   }
 
   # The id of the dispatcher to use for ClusterSharding actors.

akka-cluster-sharding/src/main/scala/akka/cluster/sharding/Shard.scala

@@ -476,8 +476,9 @@ private[akka] class DDataShard(
   implicit private val node = Cluster(context.system)
 
   // The default maximum-frame-size is 256 KiB with Artery.
-  // ORSet with 40000 elements has a size of ~ 200000 bytes.
-  // By splitting the elements over 5 keys we can safely support 200000 entities per shard.
+  // When using entity identifiers with 36 character strings (e.g. UUID.randomUUID).
+  // By splitting the elements over 5 keys we can support 10000 entities per shard.
+  // The Gossip message size of 5 ORSet with 2000 ids is around 200 KiB.
   // This is by intention not configurable because it's important to have the same
   // configuration on each node.
   private val numberOfKeys = 5

akka-docs/rst/java/cluster-sharding.rst

@@ -296,6 +296,10 @@ using a ``Passivate``.
 Note that the state of the entities themselves will not be restored unless they have been made persistent,
 e.g. with :ref:`persistence-java`.
 
+The performance cost of ``rememberEntities`` is rather high when starting/stopping entities and when 
+shards are rebalanced. This cost increases with number of entities per shard and we currently don't
+recommend using it with more than 10000 entities per shard. 
+
 Supervision
 -----------
 

akka-docs/rst/scala/cluster-sharding.rst

@@ -300,6 +300,10 @@ using a ``Passivate``.
 Note that the state of the entities themselves will not be restored unless they have been made persistent,
 e.g. with :ref:`persistence-scala`.
 
+The performance cost of ``rememberEntities`` is rather high when starting/stopping entities and when 
+shards are rebalanced. This cost increases with number of entities per shard and we currently don't
+recommend using it with more than 10000 entities per shard.
+
 Supervision
 -----------
 

akka-stream/src/main/scala/akka/stream/stage/GraphStage.scala

@@ -10,14 +10,14 @@ import java.util.concurrent.locks.ReentrantLock
 
 import akka.actor._
 import akka.annotation.ApiMayChange
-import akka.japi.function.{Effect, Procedure}
+import akka.japi.function.{ Effect, Procedure }
 import akka.stream._
 import akka.stream.impl.StreamLayout.Module
-import akka.stream.impl.fusing.{GraphInterpreter, GraphStageModule, SubSink, SubSource}
+import akka.stream.impl.fusing.{ GraphInterpreter, GraphStageModule, SubSink, SubSource }
 import akka.stream.impl.ReactiveStreamsCompliance
 
 import scala.collection.mutable.ArrayBuffer
-import scala.collection.{immutable, mutable}
+import scala.collection.{ immutable, mutable }
 import scala.concurrent.duration.FiniteDuration
 import akka.stream.actor.ActorSubscriberMessage