!clu #2307 Allow transition from unreachable to reachable

* Replace unreachable Set with Reachability table
* Unreachable members stay in member Set
* Downing a live member was moved it to the unreachable Set,
  and then removed from there by the leader. That will not
  work when flipping back to reachable, so a Down member must
  be detected as unreachable before beeing removed. Similar
  to Exiting. Member shuts down itself if it sees itself as
  Down.
* Flip back to reachable when failure detector monitors it as
  available again
* ReachableMember event
* Can't ignore gossip from aggregated unreachable (see SurviveNetworkInstabilitySpec)
* Make use of ReachableMember event in cluster router
* End heartbeat when acknowledged, EndHeartbeatAck
* Remove nr-of-end-heartbeats from conf
* Full reachability info in JMX cluster status
* Don't use interval after unreachable for AccrualFailureDetector history
* Add QuarantinedEvent to remoting, used for Reachability.Terminated
* Prune reachability table when all reachable
* Update documentation
* Performance testing and optimizations

akka-cluster/src/main/scala/akka/cluster/routing/ClusterRouterConfig.scala

@@ -274,7 +274,7 @@ private[akka] class ClusterRouterActor(override val supervisorStrategy: Supervis
   // re-subscribe when restart
   override def preStart(): Unit = {
     cluster.subscribe(self, classOf[MemberEvent])
-    cluster.subscribe(self, classOf[UnreachableMember])
+    cluster.subscribe(self, classOf[ReachabilityEvent])
   }
   override def postStop(): Unit = cluster.unsubscribe(self)
 
@@ -289,6 +289,13 @@ private[akka] class ClusterRouterActor(override val supervisorStrategy: Supervis
 
   def fullAddress(actorRef: ActorRef): Address = routeeProvider.fullAddress(actorRef)
 
+  def registerRoutees(member: Member) = {
+    routeeProvider.nodes += member.address
+    // createRoutees will create routees based on
+    // totalInstances and maxInstancesPerNode
+    routeeProvider.createRoutees()
+  }
+
   def unregisterRoutees(member: Member) = {
     val address = member.address
     routeeProvider.nodes -= address
@@ -309,17 +316,18 @@ private[akka] class ClusterRouterActor(override val supervisorStrategy: Supervis
       routeeProvider.createRoutees()
 
     case m: MemberEvent if routeeProvider.isAvailable(m.member) ⇒
-      routeeProvider.nodes += m.member.address
-      // createRoutees will create routees based on
-      // totalInstances and maxInstancesPerNode
-      routeeProvider.createRoutees()
+      registerRoutees(m.member)
 
     case other: MemberEvent ⇒
       // other events means that it is no longer interesting, such as
-      // MemberJoined, MemberLeft, MemberExited, MemberRemoved
+      // MemberExited, MemberRemoved
       unregisterRoutees(other.member)
 
     case UnreachableMember(m) ⇒
       unregisterRoutees(m)
+
+    case ReachableMember(m) ⇒
+      if (routeeProvider.isAvailable(m))
+        registerRoutees(m)
   }
 }