nightly-interop.yml

# Nightly live cross-language interop: for each of the 7 brokers, one producer SDK writes to a
# real broker container and a consumer SDK in a different language reads it back through the
# published packages (PyPI / npm / Maven Central / NuGet / Go proxy). Per-job records are merged
# by scripts/interop-merge.sh into interop-results.json and attached to the
# `nightly` prerelease. Extra evidence only — it is not a condition for "Compatible" (ADR-0032).
# The `r1-scenarios` matrix adds the five R1 behaviour scenarios (Redis delay, kill -9 recovery,
# legacy late ack, Laravel shared queue, Kafka cross-language retry) and the post-R1 Pulsar
# exhaustion scenario (Go producer, Java and Node consumers, ADR-0036); see README.md.
# The ADR-0037 scenarios (version gate on Pulsar and Artemis, poison on Artemis and SQS) are matrix entries with
# `published: "1"`: they run against the ADR-0037 releases pinned in the probe manifests (babelqueue-pulsar 1.3.0,
# babelqueue-artemis 1.1.0, babelqueue-sqs 1.4.0, BabelQueue.Artemis 1.2.0, BabelQueue.Sqs 1.4.0, @babelqueue/pulsar 1.4.0,
# @babelqueue/artemis 1.3.0, @babelqueue/sqs 1.4.0) and fail the job like any other scenario.
# The `otel` matrix (R1-otel Faz 13) adds the cross-hop OpenTelemetry check for the same 7 brokers: a producer
# in one SDK and a consumer in another write their spans to files (otel-interop/), and scripts/otel-verify.sh
# asserts that each consumer span is the child of the producer span of the same message (the W3C traceparent
# crossed the broker out of band, ADR-0028) and that both carry the ADR-0038 shape with messaging.system = the
# broker. Pairs: redis python->go, rabbitmq python->node (@babelqueue/rabbitmq), sqs python->go, pulsar
# java->dotnet, kafka java->go, artemis java->python, azureservicebus python->go. Each job records
# otel-<broker>.json; scripts/interop-merge.sh folds them into the `otel` cell of every broker in
# interop-results.json. The probes install the SDK releases pinned in otel-interop/'s manifests. The span shape,
# `system` and the transports' traceparent wiring exist only from the R1-otel releases (Go v1.19.0 / otel v0.4.0,
# PyPI 1.25.0, npm core 1.14.0, Maven core 1.15.0, NuGet Core 1.14.0 and their transports), which the pins
# now target. scripts/otel-verify.sh with BQ_SDK_ROOT still runs the same check against in-repo SDK sources.
name: nightly-interop

on:
  schedule:
    - cron: "17 3 * * *"
  workflow_dispatch:

concurrency:
  group: nightly-interop
  cancel-in-progress: false

permissions:
  contents: read

defaults:
  run:
    shell: bash

jobs:
  redis:
    name: redis (python -> go)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - name: Start Redis
        working-directory: redis-orders
        run: docker compose up -d --wait --wait-timeout 120
      - name: Producer (Python)
        working-directory: redis-orders/producer-python
        run: |
          python -m venv .venv
          .venv/bin/pip install -r requirements.txt
          .venv/bin/python produce.py
      - name: Consumer (Go)
        working-directory: redis-orders/consumer-go
        run: |
          go run . 2>&1 | tee "$RUNNER_TEMP/consumer.log"
          grep -q "processed 3 message" "$RUNNER_TEMP/consumer.log"
      - name: Broker logs
        if: failure()
        working-directory: redis-orders
        run: docker compose logs --tail=100
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: redis
          producer: python
          consumer: go
          job-status: ${{ job.status }}

  rabbitmq:
    name: rabbitmq (python -> node)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-node@v7
        with:
          node-version: "20"
      - name: Start RabbitMQ
        working-directory: rabbitmq-orders
        run: docker compose up -d --wait --wait-timeout 180
      - name: Producer (Python)
        working-directory: rabbitmq-orders/producer-python
        run: |
          python -m venv .venv
          .venv/bin/pip install -r requirements.txt
          .venv/bin/python produce.py
      - name: Consumer (Node)
        working-directory: rabbitmq-orders/consumer-node
        run: |
          npm install --no-audit --no-fund
          node consume.mjs 2>&1 | tee "$RUNNER_TEMP/consumer.log"
          grep -q "handled 4 message" "$RUNNER_TEMP/consumer.log"
      - name: Broker logs
        if: failure()
        working-directory: rabbitmq-orders
        run: docker compose logs --tail=100
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: rabbitmq
          producer: python
          consumer: node
          job-status: ${{ job.status }}

  sqs:
    name: sqs (python -> go, LocalStack)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      AWS_REGION: eu-central-1
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      SQS_ENDPOINT: http://localhost:4566
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - name: Start LocalStack (SQS)
        run: |
          docker run -d --name localstack -p 4566:4566 \
            -e SERVICES=sqs -e SQS_ENDPOINT_STRATEGY=path \
            localstack/localstack:3.8.1
          for _ in $(seq 1 60); do
            if curl -sf http://localhost:4566/_localstack/health | grep -q '"sqs": "\(available\|running\)"'; then
              echo "LocalStack SQS is ready"
              exit 0
            fi
            sleep 2
          done
          echo "LocalStack did not become ready" >&2
          exit 1
      - name: Create the orders queue
        run: aws --endpoint-url "$SQS_ENDPOINT" sqs create-queue --queue-name orders
      - name: Producer (Python)
        working-directory: sqs-orders/producer-python
        run: |
          python -m venv .venv
          .venv/bin/pip install -r requirements.txt
          .venv/bin/python produce.py
      - name: Consumer (Go)
        working-directory: sqs-orders/consumer-go
        run: |
          go run . 2>&1 | tee "$RUNNER_TEMP/consumer.log"
          grep -q "processed 3 message" "$RUNNER_TEMP/consumer.log"
      - name: Broker logs
        if: failure()
        run: docker logs --tail=100 localstack
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: sqs
          producer: python
          consumer: go
          job-status: ${{ job.status }}

  pulsar:
    name: pulsar (java -> dotnet)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-java@v6
        with:
          distribution: temurin
          java-version: "17"
          cache: maven
          cache-dependency-path: pulsar-orders/producer-java/pom.xml
      - uses: actions/setup-dotnet@v6
        with:
          dotnet-version: "8.0.x"
      - name: Start Pulsar standalone
        working-directory: pulsar-orders
        run: docker compose up -d --wait --wait-timeout 300
      - name: Build producer and consumer
        run: |
          mvn -q -B -f pulsar-orders/producer-java/pom.xml compile
          dotnet build pulsar-orders/consumer-dotnet --nologo -v quiet
      - name: Consumer first, then producer
        working-directory: pulsar-orders
        run: |
          # A Shared subscription only receives messages sent after it exists, so the .NET
          # consumer must be subscribed before the Java producer publishes.
          log="$RUNNER_TEMP/consumer.log"
          CONSUME_SECONDS=60 dotnet run --no-build --project consumer-dotnet > "$log" 2>&1 &
          consumer_pid=$!
          for _ in $(seq 1 60); do
            if docker compose exec -T pulsar bin/pulsar-admin topics subscriptions persistent://public/default/orders 2>/dev/null | grep -qx babelqueue; then
              break
            fi
            sleep 2
          done
          producer_rc=0
          mvn -q -B -f producer-java/pom.xml exec:java || producer_rc=$?
          # Without a producer there is nothing to wait for; stop the consumer early.
          if [ "$producer_rc" -ne 0 ]; then kill "$consumer_pid" 2>/dev/null || true; fi
          consumer_rc=0
          wait "$consumer_pid" || consumer_rc=$?
          # Always show the consumer output, also when it failed (connection errors etc.).
          cat "$log"
          test "$producer_rc" -eq 0
          test "$consumer_rc" -eq 0
          test "$(grep -c '^\[dotnet\] orders:' "$log")" -eq 4
      - name: Broker logs
        if: failure()
        working-directory: pulsar-orders
        run: docker compose logs --tail=100
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: pulsar
          producer: java
          consumer: dotnet
          job-status: ${{ job.status }}

  kafka:
    name: kafka (java -> go)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-java@v6
        with:
          distribution: temurin
          java-version: "17"
          cache: maven
          cache-dependency-path: kafka-orders/producer-java/pom.xml
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - name: Start Redpanda and create the topic
        working-directory: kafka-orders
        run: |
          docker compose up -d --wait --wait-timeout 180
          docker compose exec -T redpanda rpk topic create orders
      - name: Producer (Java)
        run: |
          mvn -q -B -f kafka-orders/producer-java/pom.xml compile
          mvn -q -B -f kafka-orders/producer-java/pom.xml exec:java
      - name: Consumer (Go)
        working-directory: kafka-orders/consumer-go
        run: |
          CONSUME_SECONDS=20 go run . 2>&1 | tee "$RUNNER_TEMP/consumer.log"
          test "$(grep -c '^\[go\] orders:' "$RUNNER_TEMP/consumer.log")" -eq 4
      - name: Broker logs
        if: failure()
        working-directory: kafka-orders
        run: docker compose logs --tail=100
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: kafka
          producer: java
          consumer: go
          job-status: ${{ job.status }}

  artemis:
    name: artemis (java/JMS -> python/AMQP 1.0)
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-java@v6
        with:
          distribution: temurin
          java-version: "17"
          cache: maven
          cache-dependency-path: artemis-orders/producer-java/pom.xml
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - name: Start Artemis
        working-directory: artemis-orders
        run: |
          docker compose up -d --wait --wait-timeout 180
          # §7 makes `orders` an anycast queue. The Python AMQP 1.0 attach carries no `queue`
          # capability, so on an unknown address Artemis would auto-create it MULTICAST and the
          # JMS producer's anycast send would be refused. Provision the queue up front.
          docker compose exec -T artemis /var/lib/artemis-instance/bin/artemis queue create \
            --user artemis --password artemis --name orders --address orders \
            --anycast --durable --auto-create-address --silent
      - name: Build producer and install consumer
        run: |
          mvn -q -B -f artemis-orders/producer-java/pom.xml compile
          python -m venv artemis-orders/consumer-python/.venv
          artemis-orders/consumer-python/.venv/bin/pip install "babelqueue[artemis]"
      - name: Consumer first, then producer
        working-directory: artemis-orders
        run: |
          # The consumer starts before the Java (JMS) producer sends. PYTHONUNBUFFERED makes its log lines
          # visible immediately; CONSUME_TIMEOUT keeps it listening long enough to outlast the
          # Maven start-up of the producer (it exits as soon as 4 messages are handled).
          log="$RUNNER_TEMP/consumer.log"
          PYTHONUNBUFFERED=1 CONSUME_TIMEOUT=90 \
            consumer-python/.venv/bin/python consumer-python/consume.py > "$log" 2>&1 &
          consumer_pid=$!
          # The pre-provisioned durable queue holds anything sent before the attach completes.
          producer_rc=0
          mvn -q -B -f producer-java/pom.xml exec:java || producer_rc=$?
          if [ "$producer_rc" -ne 0 ]; then kill "$consumer_pid" 2>/dev/null || true; fi
          consumer_rc=0
          wait "$consumer_pid" || consumer_rc=$?
          cat "$log"
          test "$producer_rc" -eq 0
          test "$consumer_rc" -eq 0
          grep -q "consumed 4 message" "$log"
      - name: Broker logs
        if: failure()
        working-directory: artemis-orders
        run: docker compose logs --tail=100
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: artemis
          producer: java
          consumer: python
          job-status: ${{ job.status }}

  azureservicebus:
    name: azureservicebus (python -> go, emulator)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - name: Start the Service Bus emulator
        working-directory: asb-orders
        run: |
          docker compose up -d
          # The emulator needs its SQL backend first. Wait until its health endpoint reports up;
          # the producer then runs exactly once (a retry after a partial send would duplicate
          # messages and break the "processed 3" assertion).
          for _ in $(seq 1 60); do
            if curl -sf http://localhost:5300/health >/dev/null; then
              echo "emulator health endpoint is up"
              sleep 5
              exit 0
            fi
            sleep 3
          done
          echo "Service Bus emulator did not become healthy" >&2
          docker compose logs --tail=150 >&2
          exit 1
      - name: Producer (Python)
        working-directory: asb-orders/producer-python
        run: |
          python -m venv .venv
          .venv/bin/pip install -r requirements.txt
          .venv/bin/python produce.py
      - name: Consumer (Go)
        working-directory: asb-orders/consumer-go
        run: |
          go run . 2>&1 | tee "$RUNNER_TEMP/consumer.log"
          grep -q "processed 3 message" "$RUNNER_TEMP/consumer.log"
      - name: Broker logs
        if: failure()
        working-directory: asb-orders
        run: docker compose logs --tail=150
      - name: Record result
        if: always()
        uses: ./.github/actions/record-result
        with:
          broker: azureservicebus
          producer: python
          consumer: go
          job-status: ${{ job.status }}

  # R1 scenarios (Faz 10): Redis delay / kill -9 recovery / late ack / Laravel shared queue and the
  # Kafka cross-language retry, plus the post-R1 Pulsar exhaustion scenario (pulsar-exhaustion: a Go
  # producer and DLQ reader, Java and Node consumers whose handler always throws). They run against
  # the published SDK versions pinned in each probe's manifest (ADR-0037: babelqueue 1.24.0, babelqueue-pulsar
  # 1.3.0, @babelqueue/pulsar 1.4.0, @babelqueue/redis 1.4.0, Go core v1.18.0 with redis and pulsar v1.4.0,
  # babelqueue/laravel 1.10.0, plus the artemis and sqs pins above); a probe pin that is not on its registry turns the job red. They are not part of the 7-broker
  # interop-results.json (scripts/interop-merge.sh is unchanged); the publish job fails the run when
  # any of them fails.
  r1-scenarios:
    name: r1 (${{ matrix.scenario }})
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        include:
          - scenario: redis-delay
            script: scripts/redis-delay.sh
            compose: redis-orders
          - scenario: redis-recovery
            script: scripts/redis-recovery.sh
            compose: redis-orders
          - scenario: redis-late-ack
            script: scripts/redis-late-ack.sh
            compose: redis-orders
          - scenario: redis-laravel
            script: scripts/redis-laravel-shared.sh
            compose: redis-orders
          - scenario: kafka-retry
            script: scripts/kafka-retry.sh
            compose: kafka-orders
          - scenario: pulsar-exhaustion
            script: scripts/pulsar-exhaustion.sh
            compose: pulsar-orders
          - scenario: version-gate-pulsar
            script: scripts/version-gate.sh
            compose: pulsar-orders
            brokers: pulsar
            published: "1"
          - scenario: version-gate-artemis
            script: scripts/version-gate.sh
            compose: artemis-orders
            brokers: artemis
            published: "1"
          - scenario: poison-dlq-artemis
            script: scripts/poison-dlq.sh
            compose: artemis-orders
            brokers: artemis
            published: "1"
          - scenario: poison-dlq-sqs
            script: scripts/poison-dlq.sh
            compose: sqs-orders
            brokers: sqs
            published: "1"
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - uses: actions/setup-node@v7
        with:
          node-version: "20"
      - uses: actions/setup-java@v6
        with:
          distribution: temurin
          java-version: "17"
      - uses: actions/setup-dotnet@v6
        with:
          dotnet-version: "8.0.x"
      - name: Toolchain versions
        run: |
          php --version
          composer --version
          docker compose version
      - name: Run scenario
        env:
          GATE_BROKERS: ${{ matrix.brokers }}
          POISON_BROKERS: ${{ matrix.brokers }}
          BQ_ALLOW_PUBLISHED: ${{ matrix.published }}
        run: ${{ matrix.script }}
      - name: Broker logs
        if: failure()
        working-directory: ${{ matrix.compose }}
        run: docker compose logs --tail=100

  # R1-otel Faz 13: cross-hop span parent/child on every broker (see the header). Not part of the 7-broker status
  # column of interop-results.json; the publish job reports it in the `otel` cell and fails the run when one is red.
  otel:
    name: otel (${{ matrix.broker }})
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        include:
          - broker: redis
            compose: redis-orders
          - broker: rabbitmq
            compose: rabbitmq-orders
          - broker: sqs
            compose: sqs-orders
          - broker: pulsar
            compose: pulsar-orders
          - broker: kafka
            compose: kafka-orders
          - broker: artemis
            compose: artemis-orders
          - broker: azureservicebus
            compose: asb-orders
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"
      - uses: actions/setup-go@v7
        with:
          go-version: "1.26"
          cache: false
      - uses: actions/setup-node@v7
        with:
          node-version: "20"
      - uses: actions/setup-java@v6
        with:
          distribution: temurin
          java-version: "17"
      - uses: actions/setup-dotnet@v6
        with:
          dotnet-version: "8.0.x"
      - name: Cross-hop span check
        env:
          # The runner is thrown away; keep the broker up so the "Broker logs" step below can read its logs.
          KEEP_BROKERS: "1"
          OTEL_RECORD_DIR: ${{ runner.temp }}/otel-result
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: scripts/otel-verify.sh ${{ matrix.broker }}
      - name: Broker logs
        if: failure()
        working-directory: ${{ matrix.compose }}
        run: docker compose logs --tail=100
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: otel-result-${{ matrix.broker }}
          path: ${{ runner.temp }}/otel-result/otel-${{ matrix.broker }}.json
          if-no-files-found: warn

  publish:
    name: merge results and publish to the nightly release
    runs-on: ubuntu-latest
    needs: [redis, rabbitmq, sqs, pulsar, kafka, artemis, azureservicebus, r1-scenarios, otel]
    if: ${{ !cancelled() }}
    timeout-minutes: 10
    permissions:
      contents: write
    env:
      GH_TOKEN: ${{ github.token }}
      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/download-artifact@v4
        with:
          pattern: "*result-*"
          path: results
      - name: Merge results
        run: scripts/interop-merge.sh results out
      - name: Job summary
        run: |
          {
            echo "## Live interop: $(jq -r '[.brokers[] | select(.status == "passing")] | length' out/interop-results.json)/$(jq -r '.brokers | length' out/interop-results.json) passing"
            echo
            echo "| broker | producer | consumer | status | otel (cross-hop span) |"
            echo "| :--- | :--- | :--- | :--- | :--- |"
            jq -r '.brokers[] | "| \(.broker) | \(.producer) | \(.consumer) | \(.status) | \(.otel.status) (\(.otel.producer) -> \(.otel.consumer)) |"' out/interop-results.json
          } >> "$GITHUB_STEP_SUMMARY"
      - name: Upload to the nightly prerelease
        run: |
          # Create the prerelease only when it is missing; an existing `nightly` tag is never moved.
          if ! gh release view nightly --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
            gh release create nightly --repo "$GITHUB_REPOSITORY" --prerelease \
              --target "$GITHUB_SHA" \
              --title "Nightly live interop" \
              --notes "Rolling prerelease holding the latest nightly live-interop results (interop-results.json). Assets are replaced on every run; the tag is never moved."
          fi
          gh release upload nightly out/interop-results.json --repo "$GITHUB_REPOSITORY" --clobber
      - name: Fail unless 7/7 pass
        run: jq -e '[.brokers[] | select(.status != "passing")] | length == 0' out/interop-results.json >/dev/null
      - name: Fail unless the OTel cross-hop checks pass
        if: ${{ always() }}
        run: |
          jq -e '[.brokers[] | select(.otel.status != "passing")] | length == 0' out/interop-results.json >/dev/null \
            || { jq -r '.brokers[] | select(.otel.status != "passing") | "otel \(.broker): \(.otel.status)"' out/interop-results.json >&2; exit 1; }
      - name: Fail unless the R1 scenarios pass
        if: ${{ always() }}
        env:
          R1_RESULT: ${{ needs.r1-scenarios.result }}
        run: |
          if [ "$R1_RESULT" != "success" ]; then
            echo "r1-scenarios finished with: $R1_RESULT" >&2
            exit 1
          fi