From d19855542b1b30cda98b41aace5e94e16a6bf40a Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Thu, 11 Mar 2021 13:29:20 +0100 Subject: [PATCH] configuration for SBR lease release #30090 (#30092) --- .../RandomizedBrainResolverIntegrationSpec.scala | 1 + .../sbr/SplitBrainResolverIntegrationSpec.scala | 1 + .../issue-30090-sbr-release-lease.excludes | 4 ++++ akka-cluster/src/main/resources/reference.conf | 3 +++ .../scala/akka/cluster/sbr/DowningStrategy.scala | 1 + .../akka/cluster/sbr/SplitBrainResolver.scala | 7 ++++++- .../cluster/sbr/SplitBrainResolverProvider.scala | 8 +++++++- .../cluster/sbr/SplitBrainResolverSettings.scala | 8 +++++++- .../cluster/sbr/LeaseMajority5NodeSpec.scala | 1 + .../akka/cluster/sbr/LeaseMajoritySpec.scala | 16 ++++++++-------- .../cluster/sbr/SplitBrainResolverSpec.scala | 16 ++++++++++++++-- 11 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 akka-cluster/src/main/mima-filters/2.6.13.backwards.excludes/issue-30090-sbr-release-lease.excludes diff --git a/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/RandomizedBrainResolverIntegrationSpec.scala b/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/RandomizedBrainResolverIntegrationSpec.scala index a030a460f3..13a74c4de7 100644 --- a/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/RandomizedBrainResolverIntegrationSpec.scala +++ b/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/RandomizedBrainResolverIntegrationSpec.scala @@ -55,6 +55,7 @@ object RandomizedSplitBrainResolverIntegrationSpec extends MultiNodeConfig { active-strategy = lease-majority lease-majority { lease-implementation = test-lease + release-after = 20s } } diff --git a/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/SplitBrainResolverIntegrationSpec.scala b/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/SplitBrainResolverIntegrationSpec.scala index fa31ad02a9..05b6ca55c5 100644 --- a/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/SplitBrainResolverIntegrationSpec.scala +++ b/akka-cluster-sharding/src/multi-jvm/scala/akka/cluster/sbr/SplitBrainResolverIntegrationSpec.scala @@ -396,6 +396,7 @@ class SplitBrainResolverIntegrationSpec lease-majority { lease-implementation = test-lease acquire-lease-delay-for-minority = 3s + release-after = 20s } } test-lease { diff --git a/akka-cluster/src/main/mima-filters/2.6.13.backwards.excludes/issue-30090-sbr-release-lease.excludes b/akka-cluster/src/main/mima-filters/2.6.13.backwards.excludes/issue-30090-sbr-release-lease.excludes new file mode 100644 index 0000000000..eb384976e4 --- /dev/null +++ b/akka-cluster/src/main/mima-filters/2.6.13.backwards.excludes/issue-30090-sbr-release-lease.excludes @@ -0,0 +1,4 @@ +# #30090 adding releaseAfter to LeaseMajority +ProblemFilters.exclude[Problem]("akka.cluster.sbr.LeaseMajoritySettings$") +ProblemFilters.exclude[Problem]("akka.cluster.sbr.LeaseMajoritySettings.*") +ProblemFilters.exclude[DirectMissingMethodProblem]("akka.cluster.sbr.LeaseMajority.this") diff --git a/akka-cluster/src/main/resources/reference.conf b/akka-cluster/src/main/resources/reference.conf index b78e5d1904..6fa0e94586 100644 --- a/akka-cluster/src/main/resources/reference.conf +++ b/akka-cluster/src/main/resources/reference.conf @@ -495,6 +495,9 @@ akka.cluster.split-brain-resolver.lease-majority { # as an best effort to try to keep the majority side. acquire-lease-delay-for-minority = 2s + # Release the lease after this duration. + release-after = 40s + # If the 'role' is defined the majority/minority is based only on members with that 'role'. role = "" } diff --git a/akka-cluster/src/main/scala/akka/cluster/sbr/DowningStrategy.scala b/akka-cluster/src/main/scala/akka/cluster/sbr/DowningStrategy.scala index 39003a1cfd..2e66f0d637 100644 --- a/akka-cluster/src/main/scala/akka/cluster/sbr/DowningStrategy.scala +++ b/akka-cluster/src/main/scala/akka/cluster/sbr/DowningStrategy.scala @@ -591,6 +591,7 @@ import akka.coordination.lease.scaladsl.Lease override val role: Option[String], _lease: Lease, acquireLeaseDelayForMinority: FiniteDuration, + val releaseAfter: FiniteDuration, selfUniqueAddress: UniqueAddress) extends DowningStrategy(selfDc, selfUniqueAddress) { import DowningStrategy._ diff --git a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolver.scala b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolver.scala index 681dbd8f53..f7a42c5023 100644 --- a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolver.scala +++ b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolver.scala @@ -9,6 +9,7 @@ import java.time.temporal.ChronoUnit import scala.concurrent.ExecutionContext import scala.concurrent.duration._ + import akka.actor.Actor import akka.actor.Address import akka.actor.ExtendedActorSystem @@ -161,7 +162,11 @@ import akka.remote.artery.ThisActorSystemQuarantinedEvent def downAllWhenUnstable: FiniteDuration = settings.DownAllWhenUnstable - private val releaseLeaseAfter = stableAfter * 2 + private def releaseLeaseAfter: FiniteDuration = strategy match { + case lm: LeaseMajority => lm.releaseAfter + case other => + throw new IllegalStateException(s"Unexpected use of releaseLeaseAfter for strategy [${other.getClass.getName}]") + } def tickInterval: FiniteDuration = 1.second diff --git a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverProvider.scala b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverProvider.scala index 92a1cc3dcf..852be5101f 100644 --- a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverProvider.scala +++ b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverProvider.scala @@ -55,7 +55,13 @@ final class SplitBrainResolverProvider(system: ActorSystem) extends DowningProvi val leaseOwnerName = cluster.selfUniqueAddress.address.hostPort val leaseName = s.safeLeaseName(system.name) val lease = LeaseProvider(system).getLease(leaseName, s.leaseImplementation, leaseOwnerName) - new LeaseMajority(selfDc, s.role, lease, s.acquireLeaseDelayForMinority, cluster.selfUniqueAddress) + new LeaseMajority( + selfDc, + s.role, + lease, + s.acquireLeaseDelayForMinority, + s.releaseAfter, + cluster.selfUniqueAddress) } Some(SplitBrainResolver.props(settings.DowningStableAfter, strategy)) diff --git a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverSettings.scala b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverSettings.scala index 1544f6a3d9..2be327685b 100644 --- a/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverSettings.scala +++ b/akka-cluster/src/main/scala/akka/cluster/sbr/SplitBrainResolverSettings.scala @@ -100,7 +100,12 @@ import akka.util.Helpers.Requiring case "" => None case name => Some(name) } - LeaseMajoritySettings(leaseImplementation, acquireLeaseDelayForMinority, role(c), leaseName) + + val releaseAfter = + FiniteDuration(c.getDuration("release-after").toMillis, TimeUnit.MILLISECONDS) + + LeaseMajoritySettings(leaseImplementation, acquireLeaseDelayForMinority, releaseAfter, role(c), leaseName) + } private def strategyConfig(strategyName: String): Config = cc.getConfig(strategyName) @@ -128,6 +133,7 @@ import akka.util.Helpers.Requiring @InternalApi private[sbr] final case class LeaseMajoritySettings( leaseImplementation: String, acquireLeaseDelayForMinority: FiniteDuration, + releaseAfter: FiniteDuration, role: Option[String], leaseName: Option[String]) { def safeLeaseName(systemName: String) = leaseName.getOrElse(s"$systemName-akka-sbr") diff --git a/akka-cluster/src/multi-jvm/scala/akka/cluster/sbr/LeaseMajority5NodeSpec.scala b/akka-cluster/src/multi-jvm/scala/akka/cluster/sbr/LeaseMajority5NodeSpec.scala index bc4649fbf7..2a66754ac2 100644 --- a/akka-cluster/src/multi-jvm/scala/akka/cluster/sbr/LeaseMajority5NodeSpec.scala +++ b/akka-cluster/src/multi-jvm/scala/akka/cluster/sbr/LeaseMajority5NodeSpec.scala @@ -41,6 +41,7 @@ object LeaseMajority5NodeSpec extends MultiNodeConfig { lease-majority { lease-implementation = test-lease acquire-lease-delay-for-minority = 1s + release-after = 3s } } diff --git a/akka-cluster/src/test/scala/akka/cluster/sbr/LeaseMajoritySpec.scala b/akka-cluster/src/test/scala/akka/cluster/sbr/LeaseMajoritySpec.scala index b7931553f9..5e5ce85d3b 100644 --- a/akka-cluster/src/test/scala/akka/cluster/sbr/LeaseMajoritySpec.scala +++ b/akka-cluster/src/test/scala/akka/cluster/sbr/LeaseMajoritySpec.scala @@ -13,17 +13,17 @@ class LeaseMajoritySpec extends AkkaSpec() with Eventually { val default = ConfigFactory .parseString( """ - | akka.cluster.split-brain-resolver.lease-majority.lease-implementation = "akka.coordination.lease.kubernetes" - |""".stripMargin) + akka.cluster.split-brain-resolver.lease-majority.lease-implementation = "akka.coordination.lease.kubernetes" + """) .withFallback(ConfigFactory.load()) val blank = ConfigFactory.parseString(""" - |akka.cluster.split-brain-resolver.lease-majority { - | lease-name = " " - |}""".stripMargin).withFallback(default) + akka.cluster.split-brain-resolver.lease-majority { + lease-name = " " + }""").withFallback(default) val named = ConfigFactory.parseString(""" - |akka.cluster.split-brain-resolver.lease-majority { - | lease-name = "shopping-cart-akka-sbr" - |}""".stripMargin).withFallback(default) + akka.cluster.split-brain-resolver.lease-majority { + lease-name = "shopping-cart-akka-sbr" + }""").withFallback(default) "Split Brain Resolver Lease Majority provider" must { diff --git a/akka-cluster/src/test/scala/akka/cluster/sbr/SplitBrainResolverSpec.scala b/akka-cluster/src/test/scala/akka/cluster/sbr/SplitBrainResolverSpec.scala index 5e0ea51244..9d4af88a50 100644 --- a/akka-cluster/src/test/scala/akka/cluster/sbr/SplitBrainResolverSpec.scala +++ b/akka-cluster/src/test/scala/akka/cluster/sbr/SplitBrainResolverSpec.scala @@ -827,7 +827,13 @@ class SplitBrainResolverSpec val acquireLeaseDelayForMinority: FiniteDuration = 2.seconds override def createStrategy() = - new LeaseMajority(selfDc, role, testLease, acquireLeaseDelayForMinority, selfUniqueAddress) + new LeaseMajority( + selfDc, + role, + testLease, + acquireLeaseDelayForMinority, + releaseAfter = 10.seconds, + selfUniqueAddress) } "decide AcquireLeaseAndDownUnreachable, and DownReachable as reverse decision" in { @@ -1115,7 +1121,13 @@ class SplitBrainResolverSpec tickInterval: FiniteDuration = Duration.Zero) extends Setup( stableAfter, - new LeaseMajority(selfDc, role, testLease, acquireLeaseDelayForMinority = 20.millis, selfUniqueAddress), + new LeaseMajority( + selfDc, + role, + testLease, + acquireLeaseDelayForMinority = 20.millis, + releaseAfter = 10.seconds, + selfUniqueAddress), selfUniqueAddress, downAllWhenUnstable, tickInterval)