Run timing sensitive tests in separate job, #30716 (#31149)

* better to exlude TimingTests and only run them in separate
  job so that we see more easily when those fail in CI
  (and lower prio to fix them in CI)
* BoundedBlockingQueueSpec has many timing sensitive things,
  hardened it and marked as TimingTest

.github/workflows/build-test-prValidation.yml

@@ -60,7 +60,7 @@ jobs:
           -Dakka.test.multi-in-test=false \
           -Dakka.test.timefactor=2 \
           -Dakka.actor.testkit.typed.timefactor=2 \
-          -Dakka.test.tags.exclude=gh-exclude \
+          -Dakka.test.tags.exclude=gh-exclude,timing \
           -Dakka.cluster.assert=on \
           -Dsbt.override.build.repos=false \
           -Dakka.test.multi-node=false \

.github/workflows/multi-node.yml

@@ -53,7 +53,7 @@ jobs:
           sbt -jvm-opts .jvmopts-ci \
             -Dakka.test.timefactor=2 \
             -Dakka.actor.testkit.typed.timefactor=2 \
-            -Dakka.test.tags.exclude=gh-exclude \
+            -Dakka.test.tags.exclude=gh-exclude,timing \
             -Dakka.cluster.assert=on \
             -Dsbt.override.build.repos=false \
             -Dakka.test.multi-node=true \
@@ -138,11 +138,11 @@ jobs:
           sbt -jvm-opts .jvmopts-ci \
             -Dakka.test.timefactor=2 \
             -Dakka.actor.testkit.typed.timefactor=2 \
-            -Dakka.test.tags.exclude=gh-exclude \
+            -Dakka.test.tags.exclude=gh-exclude,timing \
             -Dakka.cluster.assert=on \
             -Dakka.remote.artery.transport=aeron-udp \
             -Dsbt.override.build.repos=false \
-            -Dakka.test.tags.exclude=gh-exclude \
+            -Dakka.test.tags.exclude=gh-exclude,timing \
             -Dakka.test.multi-node=true \
             -Dakka.test.multi-node.targetDirName=${PWD}/target/${{ github.run_id }} \
             -Dakka.test.multi-node.java=${JAVA_HOME}/bin/java \

.github/workflows/nightly-builds.yml

@@ -34,7 +34,7 @@ jobs:
             -Dakka.cluster.assert=on \
             -Dakka.test.timefactor=2 \
             -Dakka.actor.testkit.typed.timefactor=2 \
-            -Dakka.test.tags.exclude=gh-exclude \
+            -Dakka.test.tags.exclude=gh-exclude,timing \
             -Dakka.log.timestamps=true \
             -Dmultinode.XX:MetaspaceSize=128M \
             -Dmultinode.Xms256M \
@@ -100,7 +100,7 @@ jobs:
           -Dakka.remote.artery.enabled=off \
           -Dakka.test.timefactor=2 \
           -Dakka.actor.testkit.typed.timefactor=2 \
-          -Dakka.test.tags.exclude=gh-exclude \
+          -Dakka.test.tags.exclude=gh-exclude,timing \
           -Dakka.cluster.assert=on \
           -Dakka.test.names.exclude=akka.cluster.Stress \
           -Dmultinode.XX:MetaspaceSize=128M \
@@ -151,7 +151,7 @@ jobs:
             -Dakka.log.timestamps=true \
             -Dakka.test.timefactor=2 \
             -Dakka.actor.testkit.typed.timefactor=2 \
-            -Dakka.test.tags.exclude=gh-exclude \
+            -Dakka.test.tags.exclude=gh-exclude,timing \
             -Dakka.test.multi-in-test=false \
             -Dmultinode.XX:MetaspaceSize=128M \
             -Dmultinode.Xms256M \

.github/workflows/scala3-build.yml

@@ -52,7 +52,7 @@ jobs:
           -Dakka.test.timefactor=2 \
           -Dakka.actor.testkit.typed.timefactor=2 \
           -Dakka.test.multi-in-test=false \
-          -Dakka.test.tags.exclude=gh-exclude \
+          -Dakka.test.tags.exclude=gh-exclude,timing \
           -Dmultinode.XX:MetaspaceSize=128M \
           -Dmultinode.Xms256M \
           -Dmultinode.Xmx256M \

.github/workflows/timing-tests.yml

@@ -0,0 +1,66 @@
+name: Timing sensitive tests
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+
+  akka-timing-sensitive-tests:
+    name: Akka Tests taggedAs TimingTest
+    runs-on: ubuntu-20.04
+    if: github.repository == 'akka/akka'
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: adopt@1.11
+
+      - name: Cache Coursier cache
+        uses: coursier/cache-action@v6.2
+
+      - name: sbt test
+        run: |-
+          sbt -jvm-opts .jvmopts-ci \
+            -Djava.security.egd=file:/dev/./urandom \
+            -Dakka.cluster.assert=on \
+            -Dakka.test.timefactor=2 \
+            -Dakka.actor.testkit.typed.timefactor=2 \
+            -Dakka.test.tags.only=timing \
+            -Dakka.log.timestamps=true \
+            -Dmultinode.XX:MetaspaceSize=128M \
+            -Dmultinode.Xms256M \
+            -Dmultinode.Xmx256M \
+            -Dmultinode.Xlog:gc \
+            -Dmultinode.XX:+AlwaysActAsServerClassMachine \
+            clean test
+
+      - name: Test Reports
+        # Makes it easier to spot failures instead of looking at the logs.
+        if: ${{ failure() }}
+        uses: marcospereira/action-surefire-report@v1
+        with:
+          report_paths: '**/target/test-reports/TEST-*.xml'
+          fail_if_no_tests: false
+          skip_publishing: true
+
+      - name: Email on failure
+        if: ${{ failure() }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{secrets.MAIL_USERNAME}}
+          password: ${{secrets.MAIL_PASSWORD}}
+          subject: "Failed: ${{ github.workflow }} / ${{ github.job }}"
+          to: akka.official@gmail.com
+          from: Akka CI (GHActions)
+          body: |
+            Job ${{ github.job }} in workflow ${{ github.workflow }} of ${{github.repository}} failed!
+            https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}

akka-actor-tests/src/test/scala/akka/util/BoundedBlockingQueueSpec.scala

@@ -109,17 +109,22 @@ class BoundedBlockingQueueSpec
       events should not contain offer("2")
     }
 
-    "block until the backing queue has space" in {
+    "block until the backing queue has space" taggedAs TimingTest in {
       val TestContext(queue, events, _, _, _, _) = newBoundedBlockingQueue(1)
       queue.offer("a")
 
-      val f = Future(queue.put("b"))
-
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        queue.take()
+      val latch = new CountDownLatch(1)
+      val f = Future {
+        latch.countDown()
+        queue.put("b")
       }
 
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.take() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      queue.take()
+
       Await.result(f, 3.seconds)
       (events should contain).inOrder(offer("a"), poll, offer("b"))
     }
@@ -128,16 +133,21 @@ class BoundedBlockingQueueSpec
       val TestContext(queue, events, _, notFull, lock, _) = newBoundedBlockingQueue(1)
       queue.offer("a")
 
+      val latch = new CountDownLatch(1)
       // Blocks until another thread signals `notFull`
-      val f = Future(queue.put("b"))
-
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        lock.lockInterruptibly()
-        notFull.signal()
-        lock.unlock()
+      val f = Future {
+        latch.countDown()
+        queue.put("b")
       }
 
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.put() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      lock.lockInterruptibly()
+      notFull.signal()
+      lock.unlock()
+
       mustBlockFor(100.milliseconds, f)
       events.toList should containInSequence(offer("a"), awaitNotFull, signalNotFull, getSize, awaitNotFull)
       events shouldNot contain(offer("b"))
@@ -160,42 +170,59 @@ class BoundedBlockingQueueSpec
       events should contain(signalNotFull)
     }
 
-    "block when the queue is empty" in {
+    "block when the queue is empty" taggedAs TimingTest in {
       val TestContext(queue, events, _, _, _, _) = newBoundedBlockingQueue(1)
 
+      val latch = new CountDownLatch(1)
       mustBlockFor(100.milliseconds) {
+        latch.countDown()
         queue.take()
       }
+
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.take() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
       events should contain(awaitNotEmpty)
       events should not contain (poll)
     }
 
-    "block until the backing queue is non-empty" in {
+    "block until the backing queue is non-empty" taggedAs TimingTest in {
       val TestContext(queue, events, _, _, _, _) = newBoundedBlockingQueue(1)
 
-      val f = Future(queue.take())
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        queue.put("a")
+      val latch = new CountDownLatch(1)
+      val f = Future {
+        latch.countDown()
+        queue.take()
       }
 
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.take() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      queue.put("a")
+
       Await.ready(f, 3.seconds)
       (events should contain).inOrder(awaitNotEmpty, offer("a"), poll)
     }
 
-    "check the backing queue size before polling" in {
+    "check the backing queue size before polling" taggedAs TimingTest in {
       val TestContext(queue, events, notEmpty, _, lock, _) = newBoundedBlockingQueue(1)
 
+      val latch = new CountDownLatch(1)
       // Blocks until another thread signals `notEmpty`
-      val f = Future(queue.take())
+      val f = Future {
+        latch.countDown()
+        queue.take()
+      }
 
       // Cause `notFull` signal, but don't fill the queue
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        lock.lockInterruptibly()
-        notEmpty.signal()
-        lock.unlock()
-      }
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.take() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      lock.lockInterruptibly()
+      notEmpty.signal()
+      lock.unlock()
 
       // `f` should still block since the queue is still empty
       mustBlockFor(100.milliseconds, f)
@@ -246,17 +273,24 @@ class BoundedBlockingQueueSpec
       events should contain(signalNotEmpty)
     }
 
-    "block for at least the timeout if the queue is full" in {
+    "block for at least the timeout if the queue is full" taggedAs TimingTest in {
       val TestContext(queue, events, _, notFull, _, _) = newBoundedBlockingQueue(1)
       queue.put("Hello")
 
       notFull.manualTimeControl(true)
 
-      val f = Future(queue.offer("World", 100, TimeUnit.MILLISECONDS))
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        notFull.advanceTime(99.milliseconds)
+      val latch = new CountDownLatch(1)
+      val f = Future {
+        latch.countDown()
+        queue.offer("World", 100, TimeUnit.MILLISECONDS)
       }
+
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.offer() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      notFull.advanceTime(99.milliseconds)
+
       mustBlockFor(100.milliseconds, f)
       events shouldNot contain(offer("World"))
     }
@@ -268,35 +302,49 @@ class BoundedBlockingQueueSpec
       events shouldNot contain(offer("World"))
     }
 
-    "block for less than the timeout when the queue becomes not full" in {
+    "block for less than the timeout when the queue becomes not full" taggedAs TimingTest in {
 
       val TestContext(queue, events, _, notFull, _, _) = newBoundedBlockingQueue(1)
       queue.put("Hello")
 
       notFull.manualTimeControl(true)
-      val f = Future(queue.offer("World", 100, TimeUnit.MILLISECONDS))
-      notFull.advanceTime(99.milliseconds)
-      after(50.milliseconds) {
-        f.isCompleted should be(false)
-        queue.take()
+
+      val latch = new CountDownLatch(1)
+      val f = Future {
+        latch.countDown()
+        queue.offer("World", 100, TimeUnit.MILLISECONDS)
       }
+      notFull.advanceTime(99.milliseconds)
+
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.offer() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      queue.take()
+
       Await.result(f, 3.seconds) should equal(true)
       (events should contain).inOrder(awaitNotFull, signalNotFull, offer("World"))
     }
 
-    "check the backing queue size before offering" in {
+    "check the backing queue size before offering" taggedAs TimingTest in {
       val TestContext(queue, events, _, notFull, lock, _) = newBoundedBlockingQueue(1)
       queue.put("Hello")
+
+      val latch = new CountDownLatch(1)
       // Blocks until another thread signals `notFull`
-      val f = Future(queue.offer("World", 1000, TimeUnit.DAYS))
+      val f = Future {
+        latch.countDown()
+        queue.offer("World", 1000, TimeUnit.DAYS)
+      }
 
       // Cause `notFull` signal, but don't fill the queue
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        lock.lockInterruptibly()
-        notFull.signal()
-        lock.unlock()
-      }
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.offer() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      lock.lockInterruptibly()
+      notFull.signal()
+      lock.unlock()
 
       // `f` should still block since the queue is still empty
       mustBlockFor(100.milliseconds, f)
@@ -348,16 +396,22 @@ class BoundedBlockingQueueSpec
       events should contain(signalNotFull)
     }
 
-    "block for at least the timeout if the queue is empty" in {
+    "block for at least the timeout if the queue is empty" taggedAs TimingTest in {
       val TestContext(queue, events, notEmpty, _, _, _) = newBoundedBlockingQueue(1)
       notEmpty.manualTimeControl(true)
 
-      val f = Future(queue.poll(100, TimeUnit.MILLISECONDS))
-
-      after(10.milliseconds) {
-        f.isCompleted should be(false)
-        notEmpty.advanceTime(99.milliseconds)
+      val latch = new CountDownLatch(1)
+      val f = Future {
+        latch.countDown()
+        queue.poll(100, TimeUnit.MILLISECONDS)
       }
+
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.poll() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      notEmpty.advanceTime(99.milliseconds)
+
       mustBlockFor(100.milliseconds, f)
       events should contain(awaitNotEmpty)
     }
@@ -375,23 +429,24 @@ class BoundedBlockingQueueSpec
       queue.poll(100, TimeUnit.MILLISECONDS) should equal(null)
     }
 
-    "block for less than the timeout when the queue becomes non-empty" in {
+    "block for less than the timeout when the queue becomes non-empty" taggedAs TimingTest in {
       val TestContext(queue, events, notEmpty, _, _, _) = newBoundedBlockingQueue(1)
 
       notEmpty.manualTimeControl(true)
 
-      val polled = new CountDownLatch(1)
+      val latch = new CountDownLatch(1)
       val f = Future {
-        polled.countDown()
+        latch.countDown()
         queue.poll(100, TimeUnit.MILLISECONDS)
       }
 
       notEmpty.advanceTime(99.milliseconds)
-      polled.await(3, TimeUnit.SECONDS)
-      after(50.milliseconds) {
-        f.isCompleted should be(false)
-        queue.put("Hello")
-      }
+      latch.await(3, TimeUnit.SECONDS)
+      // queue.poll() must happen first
+      Thread.sleep(50) // this is why this test is tagged as TimingTest
+      f.isCompleted should be(false)
+      queue.put("Hello")
+
       Await.result(f, 3.seconds) should equal("Hello")
       (events should contain).inOrder(awaitNotEmpty, signalNotEmpty, poll)
     }