flamenet-messenger-ios/.gitlab-ci.yml

# T4: the Mac is in the merge path. Runs on the project's own macOS runner
# (tag mac-local), the unit suite that used to run only when someone
# remembered to. UI tests stay manual: they need a local relay and a booted
# pair of simulators.
stages: [test]

unit-tests:
  stage: test
  tags: [mac-local]
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
  variables:
    GIT_DEPTH: "1"
    # The obfs4 transport is a submodule, and the runner skips those by
    # default. Without this the job fails at package resolution with
    # "the package manifest at Packages/Obfs4/Package.swift cannot be
    # accessed", which reads as a missing file rather than a missing clone.
    GIT_SUBMODULE_STRATEGY: recursive
    # And this, because the submodule is a PRIVATE repo. It rewrites the
    # submodule's URL to use `CI_JOB_TOKEN`, which is the only credential this
    # job has. It also requires flamenet-elligator2 to allow this project in its
    # CI job-token allowlist, a setting on the *other* project, so a failure
    # here looks like a permissions problem with no local cause.
    GIT_SUBMODULE_FORCE_HTTPS: "true"
    GIT_SUBMODULE_DEPTH: "1"
  script:
    - export DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer
    # `Scripts/generate.sh`, never bare `xcodegen generate`. `.gitignore`
    # excludes the whole `.xcodeproj`, and SwiftPM keeps `Package.resolved`
    # INSIDE it, so the pin git can actually see lives in `Scripts/`, and only
    # that script copies it into the generated workspace. Running xcodegen
    # directly here meant CI resolved every dependency fresh against the
    # network while local builds were correctly pinned, which is why this only
    # ever broke on CI. It broke when upstream MOVED the
    # `151.0.0` tag of stasel/WebRTC, a prebuilt XCFramework, from
    # 19aa8c1f to 3e43fa9b. The pin is what noticed.
    # The protocol vectors are copied into `Tests/Vectors`, so check the
    # copy is current. They are copied rather than resolved from a sibling
    # checkout because that is exactly how Android lost twenty conformance tests
    # its engine looks for `../../../flamenet-e2e/...`, which exists on a
    # developer's machine and never in a build directory, so its suite failed
    # here from the day it was written without anyone seeing
    # it. A copy in the test bundle cannot be absent. What a copy CAN do is go
    # stale, so this is the other half of that bargain.
    #
    # Pinned, not tip. flamenet-relay#56 is the write-up of what tip costs: a
    # merge in flamenet-e2e turned that repo's `main` red in a pipeline nobody
    # was watching and blocked six MRs, and was misdiagnosed as runner
    # starvation. The commit lives in `.e2e-ref`; `.e2e-ref.md` says how to move
    # it.
    #
    # A pin ALONE is worse than tip, which is what the first version of
    # this got wrong. Pinning silently means the vectors can move and
    # `ProtocolVectorTests` goes on checking last month's protocol, reporting
    # green, the very failure that suite exists to prevent, reintroduced one
    # level up. `vectors-head` below is the other half.
    - FLAMENET_E2E_PIN=$(cat .e2e-ref)
    - rm -rf ci-vectors
    - git clone -q https://gitlab.com/paulhitt/flamenet-e2e.git ci-vectors
    - git -C ci-vectors checkout -q "$FLAMENET_E2E_PIN"
    - |
      for f in protocol-v1 pqx3dh-v2 account-kdf-v1 backup-v1 backup-payload-v1 content-v1; do
        if ! diff -q "ci-vectors/test/vectors/$f.json" "Tests/Vectors/$f.json" >/dev/null; then
          echo "GATE: Tests/Vectors/$f.json differs from flamenet-e2e@$FLAMENET_E2E_PIN."
          echo "GATE: the vectors moved. Read what changed, then refresh the copy:"
          echo "GATE:   cp ci-vectors/test/vectors/*.json Tests/Vectors/"
          diff "ci-vectors/test/vectors/$f.json" "Tests/Vectors/$f.json" | head -20
          exit 1
        fi
      done
      # Inside the block, not a bare `- echo "GATE: ..."` list item. An
      # unquoted `: ` in a plain scalar makes YAML read the item as a MAPPING,
      # and the whole file becomes invalid, which GitLab reports as a pipeline
      # that "failed" with zero jobs and no yaml_errors on the pipeline object.
      # Nothing in that says "your CI file is malformed". `ci/lint` does.
      echo "GATE: protocol vectors match flamenet-e2e@$FLAMENET_E2E_PIN"
    - Scripts/generate.sh
    # Keep the build output out of Spotlight.
    #
    # `ci-build` reaches ~700 MB and is rewritten every job. Indexing it drives
    # this host, which is the mac-local runner and therefore in the merge path
    # for both app repos, to a load average in the hundreds: `mds_stores` and
    # `mdworker_shared` sit at the top of CPU while no `xcodebuild` process is
    # near it. That makes every job slower and the machine unpleasant to use
    # while CI runs.
    #
    # The marker has to be written HERE rather than committed, because the
    # directory it belongs in is created by the build and removed between runs.
    - mkdir -p ci-build && touch ci-build/.metadata_never_index
    - |
      # `|| true`, deliberately: the log is the gate, not this pipeline's
      # exit status, and the checks below need to run so they can print the
      # tail of the log when something goes wrong.
      set -o pipefail
      xcodebuild -project FlamenetMessenger.xcodeproj -scheme FlamenetMessenger \
        -sdk iphonesimulator -destination "platform=iOS Simulator,name=iPhone 17" \
        -derivedDataPath ci-build test -only-testing:FlamenetMessengerTests 2>&1 \
        | tee ci-test.log \
        | grep -E "Executed|TEST (SUCCEEDED|FAILED)" || true
    - |
      # Assert on the LOG, never on grep's exit status. This job used to
      # end in `| grep -E "Executed|TEST (SUCCEEDED|FAILED)" || exit 1`, and a
      # pipeline exits with the status of its LAST command. A failing test run
      # prints " TEST FAILED ", grep matched it, grep exited 0, and the
      # job went green on a red suite. Every pass this job ever reported was
      # unverified. It only ever went red when the build died before printing
      # anything at all, which is failing for the wrong reason.
      grep -q '\*\* TEST SUCCEEDED \*\*' ci-test.log || {
        echo "GATE: xcodebuild did not report '** TEST SUCCEEDED **'"
        tail -40 ci-test.log; exit 1; }
      # `[1-9][0-9]*`, so a suite that ran nothing cannot pass. A tree that
      # silently stops building tests otherwise reports the same green as one
      # that runs them all.
      grep -qE 'Executed [1-9][0-9]* tests, with 0 failures' ci-test.log || {
        echo "GATE: no run of at least one test with zero failures"
        tail -40 ci-test.log; exit 1; }
      # A floor, because "at least one test" is not a gate. Twice now a
      # green pipeline here has had to be checked by hand against main's count
      # ios!133 and ios!134 both printed a truncated log with no sign of the
      # tests that had just been added. A suite that silently dropped from 741
      # to 12 passes the check above and looks identical in the log.
      #
      # Set below today's count, not equal to it: this must not need editing
      # every time somebody adds a test. Raise it when it stops meaning
      # anything.
      COUNT=$(grep -oE 'Executed [0-9]+ tests, with 0 failures' ci-test.log \
              | tail -1 | grep -oE '[0-9]+' | head -1)
      if [ "${COUNT:-0}" -lt 700 ]; then
        echo "GATE: only ${COUNT:-0} tests ran; the suite is not all there"
        tail -40 ci-test.log; exit 1
      fi
      echo "GATE: Executed $COUNT tests, with 0 failures"
  artifacts:
    when: on_failure
    paths: [ci-test.log]
    expire_in: 1 week
  timeout: 30m

# The pin's conscience. Compares the vectors this repo carries against
# flamenet-e2e's actual HEAD, and is allowed to fail. Without it, `.e2e-ref` is
# permission to fall behind forever with a green board.
#
# Deliberately does NOT build the app. The question it answers, "did the
# published vectors move?", needs a clone and a diff, and nothing else. A job
# that took twenty minutes to say "you are two commits behind" would be turned
# off within a week.
#
# A red run here means an update is waiting. Take it per `.e2e-ref.md`.
vectors-head:
  stage: test
  tags: [mac-local]
  allow_failure: true
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
  variables:
    GIT_DEPTH: "1"
  script:
    - rm -rf ci-vectors-head
    - git clone --depth 1 https://gitlab.com/paulhitt/flamenet-e2e.git ci-vectors-head
    - |
      echo "pinned:  $(cat .e2e-ref)"
      echo "at HEAD: $(git -C ci-vectors-head rev-parse HEAD)"
      behind=0
      for f in protocol-v1 pqx3dh-v2 account-kdf-v1 backup-v1 backup-payload-v1 content-v1; do
        if ! diff -q "ci-vectors-head/test/vectors/$f.json" "Tests/Vectors/$f.json" >/dev/null; then
          echo "ADVISORY: $f.json has moved at HEAD:"
          diff "ci-vectors-head/test/vectors/$f.json" "Tests/Vectors/$f.json" | head -30
          behind=1
        fi
      done
      if [ "$behind" = "1" ]; then
        echo "ADVISORY: the published vectors have changed. This is not a failure of"
        echo "ADVISORY: this branch; it means an update is waiting. See .e2e-ref.md."
        exit 1
      fi
      echo "ADVISORY: vectors are current with flamenet-e2e HEAD"
  timeout: 10m