Test leader member states, see 2157

* The only allowed member states for a leader are up or leaving
* Added above check in assertLeader so that we always verify that
* More usage of assertLeader in the tests

akka-cluster/src/multi-jvm/scala/akka/cluster/LeaderElectionSpec.scala

@@ -49,6 +49,7 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
         cluster.join(firstAddress)
         awaitUpConvergence(numberOfMembers = roles.size)
         cluster.isLeader must be(myself == roles.head)
+        assertLeaderIn(roles)
       }
       testConductor.enter("after")
     }
@@ -58,6 +59,7 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
       currentRoles.size must be >= (2)
       val leader = currentRoles.head
       val aUser = currentRoles.last
+      val remainingRoles = currentRoles.tail
 
       myself match {
 
@@ -78,13 +80,14 @@ abstract class LeaderElectionSpec extends MultiNodeSpec(LeaderElectionMultiJvmSp
           cluster.down(leaderAddress)
           testConductor.enter("after-down", "completed")
 
-        case _ if currentRoles.tail.contains(myself) ⇒
+        case _ if remainingRoles.contains(myself) ⇒
           // remaining cluster nodes, not shutdown
           testConductor.enter("before-shutdown", "after-shutdown", "after-down")
 
           awaitUpConvergence(currentRoles.size - 1)
-          val nextExpectedLeader = currentRoles.tail.head
+          val nextExpectedLeader = remainingRoles.head
           cluster.isLeader must be(myself == nextExpectedLeader)
+          assertLeaderIn(remainingRoles)
 
           testConductor.enter("completed")
 

akka-cluster/src/multi-jvm/scala/akka/cluster/MultiNodeClusterSpec.scala

@@ -42,15 +42,20 @@ trait MultiNodeClusterSpec { self: MultiNodeSpec ⇒
     expectedAddresses.sorted.zipWithIndex.foreach { case (a, i) ⇒ members(i).address must be(a) }
   }
 
+  def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(myself)) {
+    assertLeaderIn(nodesInCluster)
+  }
+
   /**
    * Assert that the cluster has elected the correct leader
    * out of all nodes in the cluster. First
    * member in the cluster ring is expected leader.
    */
-  def assertLeader(nodesInCluster: RoleName*): Unit = if (nodesInCluster.contains(myself)) {
+  def assertLeaderIn(nodesInCluster: Seq[RoleName]): Unit = if (nodesInCluster.contains(myself)) {
     nodesInCluster.length must not be (0)
     val expectedLeader = roleOfLeader(nodesInCluster)
     cluster.isLeader must be(ifNode(expectedLeader)(true)(false))
+    cluster.status must (be(MemberStatus.Up) or be(MemberStatus.Leaving))
   }
 
   /**

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeShutdownSpec.scala

@@ -51,6 +51,7 @@ abstract class NodeShutdownSpec extends MultiNodeSpec(NodeShutdownMultiJvmSpec)
       }
       awaitUpConvergence(numberOfMembers = 2)
       cluster.isSingletonCluster must be(false)
+      assertLeader(first, second)
     }
 
     "become singleton cluster when one node is shutdown" taggedAs LongRunningTest in {
@@ -60,7 +61,7 @@ abstract class NodeShutdownSpec extends MultiNodeSpec(NodeShutdownMultiJvmSpec)
         testConductor.removeNode(second)
         awaitUpConvergence(numberOfMembers = 1, canNotBePartOfMemberRing = Seq(secondAddress), 30.seconds)
         cluster.isSingletonCluster must be(true)
-        cluster.isLeader must be(true)
+        assertLeader(first)
       }
 
     }

akka-cluster/src/multi-jvm/scala/akka/cluster/NodeStartupSpec.scala

@@ -38,7 +38,7 @@ abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) wi
       runOn(first) {
         awaitCond(cluster.isSingletonCluster)
         awaitUpConvergence(numberOfMembers = 1)
-        cluster.isLeader must be(true)
+        assertLeader(first)
       }
     }
   }
@@ -57,6 +57,7 @@ abstract class NodeStartupSpec extends MultiNodeSpec(NodeStartupMultiJvmSpec) wi
       }
       cluster.latestGossip.members.size must be(2)
       awaitCond(cluster.convergence.isDefined)
+      assertLeader(first, second)
     }
   }