Merge branch 'master' into wip-2162-membership-change-listener-spec-jboner

Signed-off-by: Jonas Bonér <jonas@jonasboner.com>

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderDowningNodeThatIsUnreachableSpec.scala

@@ -17,7 +17,7 @@ object LeaderDowningNodeThatIsUnreachableMultiJvmSpec extends MultiNodeConfig {
   val third = role("third")
   val fourth = role("fourth")
 
-  commonConfig(debugConfig(on = false).
+  commonConfig(debugConfig(on = true).
     withFallback(ConfigFactory.parseString("""
       akka.cluster {
         auto-down = on

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala

@@ -49,6 +49,7 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
         cluster.join(firstAddress)
         awaitUpConvergence(numberOfMembers = roles.size)
         cluster.isLeader must be(myself == roles.head)
+        assertLeaderIn(roles)
       }
       testConductor.enter("after")
     }
@@ -58,6 +59,7 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
       currentRoles.size must be >= (2)
       val leader = currentRoles.head
       val aUser = currentRoles.last
+      val remainingRoles = currentRoles.tail
 
       myself match {
 
@@ -78,13 +80,14 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
           cluster.down(leaderAddress)
           testConductor.enter("after-down", "completed")
 
-        case _ if currentRoles.tail.contains(myself) ⇒
+        case _ if remainingRoles.contains(myself) ⇒
           // remaining cluster nodes, not shutdown
           testConductor.enter("before-shutdown", "after-shutdown", "after-down")
 
           awaitUpConvergence(currentRoles.size - 1)
-          val nextExpectedLeader = currentRoles.tail.head
+          val nextExpectedLeader = remainingRoles.head
           cluster.isLeader must be(myself == nextExpectedLeader)
+          assertLeaderIn(remainingRoles)
 
           testConductor.enter("completed")
 

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

@@ -42,15 +42,20 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
     expectedAddresses.sorted.zipWithIndex.foreach { case (a, i) ⇒ members(i).address must be(a) }
   }
 
+  def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(myself)) {
+    assertLeaderIn(nodesInCluster)
+  }
+
   /**
    * Assert that the cluster has elected the correct leader
    * out of all nodes in the cluster. First
    * member in the cluster ring is expected leader.
    */
-  def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(myself)) {
+  def assertLeaderIn(nodesInCluster: Seq[RoleName]): Unit = if (nodesInCluster.contains(myself)) {
     nodesInCluster.length must not be (0)
     val expectedLeader = roleOfLeader(nodesInCluster)
     cluster.isLeader must be(ifNode(expectedLeader)(true)(false))
+    cluster.status must (be(MemberStatus.Up) or be(MemberStatus.Leaving))
   }
 
   /**

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingAndExitingSpec.scala

@@ -20,8 +20,8 @@ object NodeLeavingAndExitingMultiJvmSpec extends MultiNodeConfig {
     debugConfig(on = false)
     .withFallback(ConfigFactory.parseString("""
         akka.cluster {
-          leader-actions-frequency           = 5000 ms  # increase the leader action task frequency
-          unreachable-nodes-reaper-frequency = 30000 ms # turn "off" reaping to unreachable node set
+          leader-actions-frequency           = 5 s  # increase the leader action task frequency to make sure we get a chance to test the LEAVING state
+          unreachable-nodes-reaper-frequency = 30 s # turn "off" reaping to unreachable node set
         }
       """)
     .withFallback(MultiNodeClusterSpec.clusterConfig)))
@@ -64,6 +64,8 @@ abstract class NodeLeavingAndExitingSpec extends MultiNodeSpec(NodeLeavingAndExi
       runOn(first, third) {
 
         // 1. Verify that 'second' node is set to LEAVING
+        //   We have set the 'leader-actions-frequency' to 5 seconds to make sure that we get a
+        //   chance to test the LEAVING state before the leader moves the node to EXITING
         awaitCond(cluster.latestGossip.members.exists(_.status == MemberStatus.Leaving)) // wait on LEAVING
         val hasLeft = cluster.latestGossip.members.find(_.status == MemberStatus.Leaving) // verify node that left
         hasLeft must be('defined)

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeLeavingSpec.scala

@@ -18,8 +18,8 @@ object NodeLeavingMultiJvmSpec extends MultiNodeConfig {
   commonConfig(
     debugConfig(on = false)
     .withFallback(ConfigFactory.parseString("""
-        akka.cluster.leader-actions-frequency           = 5000 ms
-        akka.cluster.unreachable-nodes-reaper-frequency = 30000 ms # turn "off" reaping to unreachable node set
+        akka.cluster.leader-actions-frequency           = 5 s
+        akka.cluster.unreachable-nodes-reaper-frequency = 30 s # turn "off" reaping to unreachable node set
       """))
     .withFallback(MultiNodeClusterSpec.clusterConfig))
 }

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeShutdownSpec.scala

@@ -51,16 +51,17 @@ abstract class NodeShutdownSpec extends MultiNodeSpec(NodeShutdownMultiJvmSpec)
       }
       awaitUpConvergence(numberOfMembers = 2)
       cluster.isSingletonCluster must be(false)
+      assertLeader(first, second)
     }
 
-    "become singleton cluster when one node is shutdown" in {
+    "become singleton cluster when one node is shutdown" taggedAs LongRunningTest in {
       runOn(first) {
         val secondAddress = node(second).address
-        testConductor.shutdown(first, 0)
-        testConductor.removeNode(first)
+        testConductor.shutdown(second, 0)
+        testConductor.removeNode(second)
         awaitUpConvergence(numberOfMembers = 1, canNotBePartOfMemberRing = Seq(secondAddress), 30.seconds)
         cluster.isSingletonCluster must be(true)
-        cluster.isLeader must be(true)
+        assertLeader(first)
       }
 
     }