name: Multi node test

on:
  schedule:
    - cron: '0 2 * * 1,3,5'
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  # Only run once for latest commit per ref and cancel other (previous) runs.
  group: ci-multi-node-${{ github.ref }}
  cancel-in-progress: true

jobs:
  run-multi-node-tests:
    name: Multi Node Test
    runs-on: ubuntu-20.04
    if: github.repository == 'apache/incubator-pekko'
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install Kubectl
        run: |
          sudo snap install kubectl --classic
      - uses: google-github-actions/setup-gcloud@v0.2
        with:
          service_account_key: ${{ secrets.GKE_SA_KEY }}
          project_id: ${{ secrets.GKE_PROJECT }}

      - name: Create the cluster
        run: |-
          gcloud config set compute/region us-central1
          gcloud config set compute/zone us-central1-c
          ./kubernetes/create-cluster-gke.sh "pekko-multi-node-${GITHUB_RUN_ID}"

      - name: Setup Pods
        run: |
          # Start 10 pods. At most 10 MultiJvmNode (org.apache.pekko.cluster.StressSpec is currently disabled).
          ./kubernetes/setup.sh 10 multi-node-test.hosts tcp

      - name: Setup Java 11
        uses: actions/setup-java@v3
        with:
          distribution: temurin
          java-version: 11

      - name: Cache Coursier cache
        uses: coursier/cache-action@v6.4.0

      - name: Enable jvm-opts
        run: cp .jvmopts-ci .jvmopts

      - name: Multi node test
        run: |
          cat multi-node-test.hosts
          sbt \
            -Dpekko.test.timefactor=2 \
            -Dpekko.actor.testkit.typed.timefactor=2 \
            -Dpekko.test.tags.exclude=gh-exclude,timing \
            -Dpekko.cluster.assert=on \
            -Dsbt.override.build.repos=false \
            -Dpekko.test.multi-node=true \
            -Dpekko.test.multi-node.targetDirName=${PWD}/target/${{ github.run_id }} \
            -Dpekko.test.multi-node.java=${JAVA_HOME}/bin/java \
            -Dmultinode.XX:MetaspaceSize=128M \
            -Dmultinode.Xms512M \
            -Dmultinode.Xmx512M \
            -Dmultinode.Xlog:gc \
            -Dmultinode.XX:+AlwaysActAsServerClassMachine \
            -Daeron.dir=/opt/volumes/media-driver \
            -Dmultinode.Daeron.dir=/opt/volumes/media-driver \
            -Daeron.term.buffer.length=33554432 \
            -Dmultinode.Daeron.term.buffer.length=33554432 \
            multiNodeTest

      # comment out email actions until we have an email address to use (and we need to get INFRA to whitelist dawidd6/action-send-mail)
      #- name: Email on failure
      #  if: ${{ failure() }}
      #  uses: dawidd6/action-send-mail@v3
      #  with:
      #    server_address: smtp.gmail.com
      #    server_port: 465
          # Using port 465 already sets `secure: true`
      #    secure: true
      #    username: ${{secrets.MAIL_USERNAME}}
      #    password: ${{secrets.MAIL_PASSWORD}}
      #    subject: Multi node test (Akka)
      #    to: akka.official@gmail.com
      #    from: Akka CI (GHActions)
      #    body: |
      #      Multi node test of ${{github.repository}} failed!
      #      https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      
      - name: Cleanup the environment
        if: ${{ always() }}
        shell: bash {0}
        run: |
          gcloud container clusters delete "pekko-multi-node-${GITHUB_RUN_ID}" --quiet

  run-multi-node-aeron-tests:
    name: Multi Node Test with Artery Aeron UDP transport
    runs-on: ubuntu-20.04
    if: github.repository == 'apache/incubator-pekko'
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Install Kubectl
        run: |
          sudo snap install kubectl --classic
      - uses: google-github-actions/setup-gcloud@v0.2
        with:
          service_account_key: ${{ secrets.GKE_SA_KEY }}
          project_id: ${{ secrets.GKE_PROJECT }}

      - name: Create the cluster
        run: |-
          gcloud config set compute/region us-central1
          gcloud config set compute/zone us-central1-c
          ./kubernetes/create-cluster-gke.sh "pekko-artery-aeron-cluster-${GITHUB_RUN_ID}"

      - name: Setup Pods
        run: |
          # Start 10 pods. At most 10 MultiJvmNode (org.apache.pekko.cluster.StressSpec is currently disabled).
          ./kubernetes/setup.sh 10 multi-node-test.hosts udp

      - name: Setup Java 11
        uses: actions/setup-java@v3
        with:
          distribution: temurin
          java-version: 11

      - name: Cache Coursier cache
        uses: coursier/cache-action@v6.4.0

      - name: Enable jvm-opts
        run: cp .jvmopts-ci .jvmopts

      - name: Multi node test with Artery Aeron UDP
        run: |
          cat multi-node-test.hosts
          sbt \
            -Dpekko.test.timefactor=2 \
            -Dpekko.actor.testkit.typed.timefactor=2 \
            -Dpekko.cluster.assert=on \
            -Dpekko.remote.artery.transport=aeron-udp \
            -Dsbt.override.build.repos=false \
            -Dpekko.test.tags.exclude=gh-exclude,gh-exclude-aeron,timing \
            -Dpekko.test.multi-node=true \
            -Dpekko.test.multi-node.targetDirName=${PWD}/target/${{ github.run_id }} \
            -Dpekko.test.multi-node.java=${JAVA_HOME}/bin/java \
            -Dmultinode.XX:MetaspaceSize=128M \
            -Dmultinode.Xms512M \
            -Dmultinode.Xmx512M \
            -Dmultinode.Xlog:gc \
            -Dmultinode.XX:+AlwaysActAsServerClassMachine \
            -Daeron.dir=/opt/volumes/media-driver \
            -Dmultinode.Daeron.dir=/opt/volumes/media-driver \
            -Daeron.term.buffer.length=33554432 \
            -Dmultinode.Daeron.term.buffer.length=33554432 \
            multiNodeTest

      #- name: Email on failure
      #  if: ${{ failure() }}
      #  uses: dawidd6/action-send-mail@v3
      #  with:
      #    server_address: smtp.gmail.com
      #    server_port: 465
          # Using port 465 already sets `secure: true`
      #    secure: true
      #    username: ${{secrets.MAIL_USERNAME}}
      #    password: ${{secrets.MAIL_PASSWORD}}
      #    subject: Multi node test with Aeron (Akka)
      #    to: akka.official@gmail.com
      #    from: Akka CI (GHActions)
      #    body: |
      #      Multi node test of ${{github.repository}} failed!
      #      https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      
      - name: Cleanup the environment
        if: ${{ always() }}
        shell: bash {0}
        run: |
          gcloud container clusters delete "pekko-artery-aeron-cluster-${GITHUB_RUN_ID}" --quiet