# GitLab CI can create duplicate pipelines in certain cases when one event that
# triggers a pipeline (e.g. a push) causes another such event (e.g. the
# resulting update to an MR). This block prevents that in common cases.
workflow:
  rules:
    # If pushing to a branch that has an open MR associated with it, don't
    # launch the on-push pipeline.
    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"'
      when: never
    - when: always

# Define the potential stages that can be run in a GitLab CI pipeline job
stages:
  - setup
  - lint
  - test
  - docs
  - benchmark
  - test-release
  - release
  - handler

variables:
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pypoetry"
  POETRY_VIRTUALENVS_IN_PROJECT: "true"
  # Force nox to produce colorful logs:
  FORCE_COLOR: "true"
  # Enable feature flags
  # https://docs.gitlab.com/runner/configuration/feature-flags.html
  FF_SCRIPT_SECTIONS: "true"
  FF_USE_FASTZIP: "true"

.base:
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  before_script:
    - java -version
    - python --version
    - poetry self show
    # Force git to use 8 characters as the short-hash length. This avoids an
    # issue[0] with poetry-dynamic-versioning where it doesn't generate a
    # consistent name for the package generated in the CI, using either 7 or 8
    # characters across different jobs depending on how much of the repository
    # is pulled down onto the CI runner for each job.
    # [0] https://github.com/mtkennerly/dunamai/issues/89
    - git config core.abbrev 8
    # Fail if the Poetry lock file is out of date.
    - poetry lock --check
    # Set up SSH config so that the runner can `pip install` out of GitLab
    # repositories.
    - mkdir -p ~/.ssh/
    - cp $GITLAB_SSH_KNOWN_HOSTS ~/.ssh/known_hosts
    - chmod 600 "$CI_SSH_KEY"
    - cp -pv "$CI_SSH_KEY" ~/.ssh/id_ed25519
    - poetry install --no-root --only scripting
    - source .venv/bin/activate
  artifacts:
    when: always
    expire_in: 1 week
  cache:
    # Cache the pip cache. While the cache could be persisted across changes to
    # the Poetry lock file, clearing it when that changes provides a good way to
    # keep the cache from growing too large due to old packages.
    - key:
        files: ["poetry.lock"]
      paths: [".cache/pip"]
  tags: [aws-small]
  interruptible: true
  after_script:
    # Just to be safe, remove the key after the job finishes
    - rm -v ~/.ssh/id_ed25519

.analytics_base:
  extends: .base
  before_script:
    - java -version
    - python --version
    # Set up SSH config so that the runner can `pip install` out of GitLab
    # repositories.
    - mkdir -p ~/.ssh/
    - cp $GITLAB_SSH_KNOWN_HOSTS ~/.ssh/known_hosts
    - chmod 600 "$CI_SSH_KEY"
    - cp -pv "$CI_SSH_KEY" ~/.ssh/id_ed25519
    - export CORE_WHEEL_DIR="$(pwd)/dist"
    - git clone $analytics_git ../analytics
    - cd ../analytics
    - poetry install --no-root --only scripting
    - source .venv/bin/activate
    - nox -s build

# "Fake" job to prevent dev pipelines from being interrupted on new commits.
# See gitlab-org/gitlab#34221
prevent_dev_interrupt:
  stage: setup
  script:
    - echo "No-op job to prevent dev pipelines from being interrupted"
  variables:
    GIT_STRATEGY: none
  timeout: 1m
  rules:
    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"

package:
  extends: .base
  stage: setup
  needs: []
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  # make a docker daemon available for cibuildwheel to use
  services:
    - name: docker:dind
      entrypoint: ["env", "-u", "DOCKER_HOST"]
      command: ["dockerd-entrypoint.sh"]
  variables:
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_DRIVER: overlay2
    # See https://github.com/docker-library/docker/pull/166
    DOCKER_TLS_CERTDIR: ""
  script:
    - curl -sSL https://get.docker.com/ | sh
    - nox -s build
  artifacts:
    paths: ["dist/"]
  timeout: 15m
  interruptible: true
  tags: [saas-linux-large-amd64]

# Requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY variables to be set.
# We use an access key from the core-ci AWS user, which has permissions to list
# and modify all objects in the tumult.core-wheel-cache bucket.
get_mac_wheels:
  extends: .base
  stage: setup
  needs: []
  artifacts:
    paths: ["dist/"]
    expire_in: 1 week
  script:
    - nox --no-venv -s get_mac_wheels
  rules:
    # Start this job with a short delay to allow the CircleCI pipeline
    # to be created
    - if: "$CI_COMMIT_TAG"
      when: delayed
      start_in: 30 seconds
    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
      when: delayed
      start_in: 30 seconds
    - when: never

lint:
  extends: .base
  stage: lint
  needs: ["package"]
  script:
    - nox -t lint -- --check
  timeout: 20m
  rules:
    - if: "$CI_COMMIT_TAG"
      when: never
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - when: on_success

sanity_checks:
  extends: lint
  stage: lint
  script:
    - nox -t sanity-checks

test:
  extends: .base
  stage: test
  needs: ["package"]
  script:
    - nox -s test_smoketest test_fast
  coverage: '/^TOTAL.+?(\d+\%)$/'
  artifacts:
    paths: ["coverage/"]
    reports:
      coverage_report:
        coverage_format: cobertura
        path: "coverage.xml"
      junit: "junit*.xml"
    expose_as: "Coverage"
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_TAG'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
    - when: manual
      allow_failure: true
  tags: [aws-c6a.xlarge]

test_docs:
  extends: test
  script:
    - nox -s test_doctest

test_slow:
  extends: test
  script:
    - nox -s test_slow
  timeout: 23h
  rules:
    - if: '$CI_COMMIT_TAG'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - when: manual
      allow_failure: true

test_dep_matrix:
  extends: .base
  stage: test
  needs: ["package"]
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.10", "3.11", "3.12"]
        TEST_TYPE: ["oldest", "newest"]
      - PYTHON_VERSION: ["3.10"]
        TEST_TYPE: ["pyspark3.4"]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python${PYTHON_VERSION}
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_COMMIT_TAG'
      when: never
    - when: manual
      allow_failure: true
  script:
    - nox -s "test_dependency_matrix(${PYTHON_VERSION}-${TEST_TYPE})"
  cache:
    - key:
        prefix: "$PYTHON_VERSION-$TEST_TYPE"
        files: ["poetry.lock"]
      paths: [".cache/pip"]
  timeout: 3h
  tags: [aws-c6a.xlarge]
  interruptible: true

.analytics_tests:
  extends: .analytics_base
  variables:
    analytics_git: "git@gitlab.com:tumult-labs/analytics.git"
  stage: test
  needs:
    - job: package
      artifacts: true
    - job: test_slow
      artifacts: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      allow_failure: true
  tags: [aws-c6a.xlarge]

test_analytics:
  extends: .analytics_tests
  script:
    - nox -s test_smoketest test_fast

test_analytics_docs:
  extends: .analytics_tests
  script:
    - nox -s test_doctest

.analytics_pro_tests:
  extends: .analytics_base
  variables:
    analytics_git: "git@gitlab.com:tumult-labs/analytics-ee.git"
  stage: test
  needs:
    - job: package
      artifacts: true
    - job: test_slow
      artifacts: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      allow_failure: true
  tags: [aws-c6a.xlarge]

test_analytics_pro:
  extends: .analytics_pro_tests
  script:
    - nox -s test_smoketest test_fast

test_analytics_pro_docs:
  extends: .analytics_pro_tests
  script:
    - nox -s test_doctest

# Requires the following CI variables to be set:
# * NIGHTLY_HANDLER_TOKEN, a GitLab project access token with the read_api permission
# * NIGHTLY_SLACK_WEBHOOK_URL, a Slack incoming webhook for posting to the appropriate
#   channel
nightly_handler:
  stage: handler
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  script:
    - source .pipeline_handlers
    - nightly_handler
  variables:
    GIT_DEPTH: 1
  timeout: 2m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: always
  interruptible: false

docs:
  extends: .base
  stage: docs
  needs: ["package"]
  script:
    - nox -t docs
    # Remove Sphinx build cache before saving, as it is large and isn't needed anymore.
    - rm -r public/.doctrees
  artifacts:
    paths:
      - public/
    expose_as: "Documentation"
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: "$CI_COMMIT_TAG"
    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
    - when: manual
      allow_failure: true

test_release_linux_x86:
  extends: .base
  stage: test-release
  needs: ["package"]
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.10", "3.11", "3.12"]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python${PYTHON_VERSION}
  script:
    - nox -s release_smoketest release_test
  coverage: '/^TOTAL.+?(\d+\%)$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: "coverage.xml"
      junit: "junit*.xml"
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: "$CI_COMMIT_TAG"
    - when: manual
      allow_failure: true
  tags: [aws-c6a.xlarge]

test_release_macos_arm:
  stage: test-release
  needs: [get_mac_wheels]
  parallel:
    matrix:
      - IMAGE: ["macos-14-xcode-15"]
  image: ${IMAGE}
  before_script:
    - python --version
    # Setup Java 11
    - brew install openjdk@11
    - export PATH="/opt/homebrew/opt/openjdk@11/bin:$PATH"
    - java -version
  script:
    # Set up SSH config so that the runner can `pip install` out of GitLab
    # repositories.
    - mkdir -p ~/.ssh/
    - cp $GITLAB_SSH_KNOWN_HOSTS ~/.ssh/known_hosts
    - chmod 600 "$CI_SSH_KEY"
    - cp -pv "$CI_SSH_KEY" ~/.ssh/id_ed25519
    - pip install poetry
    - export PATH="/Users/gitlab/.local/bin:$PATH"
    - poetry self add "poetry-dynamic-versioning[plugin]"
    - poetry install --no-root --only scripting
    - source .venv/bin/activate
    - nox -s release_smoketest release_test
  coverage: '/^TOTAL.+?(\d+\%)$/'
  artifacts:
    paths: ["coverage/"]
    reports:
      coverage_report:
        coverage_format: cobertura
        path: "coverage.xml"
      junit: "junit*.xml"
    expose_as: "Coverage"
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: "$CI_COMMIT_TAG"
  tags: [saas-macos-medium-m1]

test_release_macos_x86:
  stage: test-release
  needs: [get_mac_wheels]
  parallel:
    matrix:
      - IMAGE: ["macos-14-xcode-15"]
  image: ${IMAGE}
  before_script:
    - softwareupdate --install-rosetta --agree-to-license
    # Install Homebrew for x86
    - arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
    - which -a brew
    # Ensure Homebrew is set up correctly
    - export PATH="/usr/local/bin:$PATH"
    - arch -x86_64 brew update
    # Setup non-ARM Python
    - arch -x86_64 brew install python@3.10
    - export PATH="/usr/local/opt/python@3.10/libexec/bin:$PATH"
    - python --version
    # Setup Java 11
    - arch -x86_64 brew install openjdk@11
    - export PATH="/usr/local/opt/openjdk@11/bin:$PATH"
    - java -version
  script:
    # Set up SSH config so that the runner can `pip install` out of GitLab
    # repositories.
    - mkdir -p ~/.ssh/
    - cp $GITLAB_SSH_KNOWN_HOSTS ~/.ssh/known_hosts
    - chmod 600 "$CI_SSH_KEY"
    - cp -pv "$CI_SSH_KEY" ~/.ssh/id_ed25519
    - curl -sSL https://install.python-poetry.org | python3.10 -
    - export PATH="/Users/gitlab/.local/bin:$PATH"
    - poetry self add "poetry-dynamic-versioning[plugin]"
    - poetry install --no-root --only scripting
    - source .venv/bin/activate
    - nox -s release_smoketest release_test
  coverage: '/^TOTAL.+?(\d+\%)$/'
  artifacts:
    paths: ["coverage/"]
    reports:
      coverage_report:
        coverage_format: cobertura
        path: "coverage.xml"
      junit: "junit*.xml"
    expose_as: "Coverage"
  timeout: 1h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: "$CI_COMMIT_TAG"
  tags: [saas-macos-medium-m1]

audit:
  extends: .base
  stage: test-release
  needs: ["package"]
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.10", "3.11", "3.12"]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python${PYTHON_VERSION}
  script:
    - nox -s "audit(python=\"${PYTHON_VERSION}\")"
  timeout: 15m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: "$CI_COMMIT_TAG"
    - when: manual
      allow_failure: true

publish:
  stage: release
  dependencies: [get_mac_wheels, package]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  script:
    - poetry publish -u __token__ -p "${RELEASE_PYPI_TOKEN}"
  timeout: 3m
  rules:
    - if: "$CI_COMMIT_TAG"
  interruptible: false

# This job requires the DOCS_TOKEN_USER and DOCS_TOKEN variables to be set. They
# should be the name and token of a project access token from the
# tumult-labs/ops/docs.tmlt.dev project with the read_repository and
# write_repository permissions. The bot user for this token must have permission
# to push the protected `prod` branch.
publish_docs:
  stage: release
  dependencies: [docs]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  variables:
    GIT_STRATEGY: none
  before_script:
    - docs_credentials="_:${DOCS_TOKEN:?Token for publishing to docs repo is not set}"
    - |
      if [[ -z "$CI_COMMIT_TAG" ]]; then
        version="$CI_COMMIT_BRANCH"
        docs_version="$version"
      else
        version="$CI_COMMIT_TAG"
        # Check if this tag is a pre-release.
        if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta|rc)\.[0-9]+$ ]]; then
          echo "Version $version is a pre-release, skipping docs publication."
          exit 0
        fi
        # Convert X.Y.Z semantic version to vX.Y for docs.
        docs_version="v$(echo $version | sed -E 's/^([[:digit:]]+\.[[:digit:]]+).*/\1/')"
      fi
    - echo "Publishing version $version as $docs_version..."
  script:
    - git clone https://$docs_credentials@gitlab.com/tumult-labs/ops/docs.tmlt.dev.git
    # Remove old docs published as this version, if any
    - rm -rf docs.tmlt.dev/public/core/$docs_version
    - mv public/ docs.tmlt.dev/public/core/$docs_version
    - cd docs.tmlt.dev
    - git config user.name "gitlab"
    - git config user.email "gitlab@tmlt.io"
    - git add public
    - git diff --staged --stat
    - git commit -m "[auto] Publish docs for $version ($CI_COMMIT_SHORT_SHA)" --author "$CI_COMMIT_AUTHOR"
    - git push
  timeout: 3m
  interruptible: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: "$CI_COMMIT_TAG"

benchmark_nightly:
  extends: .base
  stage: benchmark
  needs: ["package"]
  script:
    - nox -s benchmark
  artifacts:
    paths:
      - benchmark_output/
    expose_as: "Benchmarks"
  timeout: 3h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: "$CI_COMMIT_TAG"
      when: never
    - when: manual
      allow_failure: true
  tags: [aws-c6a.xlarge]

benchmark_dependency_matrix:
  extends: .base
  stage: benchmark
  needs: ["package"]
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.10", "3.11", "3.12"]
        TEST_TYPE: ["oldest", "newest"]
      - PYTHON_VERSION: ["3.10"]
        TEST_TYPE: ["pyspark3.4"]
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python${PYTHON_VERSION}
  script:
    - nox -s "benchmark_multi_deps(${PYTHON_VERSION}-${TEST_TYPE})"
  artifacts:
    paths:
      - benchmark_output/
    expose_as: "Benchmarks"
  timeout: 3h
  rules: # Can only be triggered manually
    - if: "$CI_COMMIT_TAG"
      when: never
    - when: manual
      allow_failure: true
  tags: [aws-c6a.xlarge]

# Requires the following CI variables to be set:
# * NIGHTLY_HANDLER_TOKEN, a GitLab project access token with the read_api permission
# * NIGHTLY_SLACK_WEBHOOK_URL, a Slack incoming webhook for posting to the appropriate
#   channel
nightly_handler:
  stage: handler
  image: registry.gitlab.com/tumult-labs/ops/ci/linux:python3.10
  script:
    - poetry install --no-root --only ci-tools
    - source .venv/bin/activate
    - python .ci/pipeline-handler.py nightly
        --allow-failure test_analytics
        --allow-failure test_analytics_docs
        --allow-failure test_analytics_pro
        --allow-failure test_analytics_pro_docs
  variables:
    GIT_DEPTH: 1
  timeout: 2m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: always
  interruptible: false

# Requires the following CI variables to be set:
# * VERSION, the semantic version number to be given to the release. This would
#   typically be specified when launching the manual job.
# * RELEASER_TOKEN_USERNAME and RELEASER_TOKEN, a GitLab project access token
#   (and an associated username) with the api permission and the ability to push
#   tags.
trigger_release:
  extends: .base
  stage: handler
  before_script:
    - !reference [.base, before_script]
    - releaser_credentials="_:${RELEASER_TOKEN?Token for releasing is not set}"
    - git config user.name "gitlab"
    - git config user.email "gitlab@tmlt.io"
  script:
    - nox -s prepare_release
    - git checkout -B "release/$VERSION"
    - git add -u
    - git diff --staged --stat
    - git commit -m "[auto] Prepare release $VERSION" --author "$CI_COMMIT_AUTHOR" --allow-empty
    - git tag "$VERSION"
    - nox -s post_release
    - git add -u
    - git diff --staged --stat
    - git commit -m "[auto] Post-release $VERSION" --author "$CI_COMMIT_AUTHOR" --allow-empty
    - git remote add origin-https "https://$releaser_credentials@gitlab.com/$CI_PROJECT_PATH.git"
    - git push origin-https "release/$VERSION" --push-option ci.skip
    - git push origin-https "$VERSION"
    - |
      cat > body.json <<EOF
      {
        "source_branch": "release/$VERSION",
        "target_branch": "dev",
        "title": "Merge release $VERSION",
        "assignee_id": $GITLAB_USER_ID,
        "remove_source_branch": true,
        "squash": false,
        "approvals_before_merge": 1
      }
      EOF
    - cat body.json
    - |
      http_status=$(
        curl -XPOST -s -o response.json -w "%{http_code}" \
            "$CI_API_V4_URL/projects/$CI_PROJECT_ID/merge_requests" \
            --header "PRIVATE-TOKEN: $RELEASER_TOKEN" \
            -H "Content-Type: application/json" -d "@body.json"
      )
      echo "Response body:"
      cat response.json | jq
      if [[ $http_status -ne 201 ]]; then
        echo "Got unexpected HTTP status $http_status, exiting..."
        exit 1
      fi
  timeout: 5m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: manual
  interruptible: false

# Requires the following CI variables to be set:
# * RELEASE_SLACK_WEBHOOK_URL, a Slack incoming webhook for posting to the appropriate
#   channel
release_handler:
  stage: handler
  script:
    - source .ci/pipeline-handlers.bash
    - release_handler
  timeout: 2m
  rules:
    - if: "$CI_COMMIT_TAG"
  interruptible: false
