Daily scheduled Optimization Tests #112
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled Optimization Tests | |
| run-name: "${{ github.event.schedule == '0 2 * * 0' && 'Weekly' || 'Daily' }} scheduled Optimization Tests" | |
| on: | |
| # We create two different (overlapping) schedules - a daily schedule and a weekly schedule | |
| # We detect the schedule in the job definition, and run different tests depending on the schedule | |
| # If you change any of the 'cron' strings below, make sure to also update them in the 'optimizations-tests' definition below | |
| schedule: | |
| # Run daily at 2 AM UTC | |
| - cron: "0 2 * * *" | |
| # Run weekly at 2 AM UTC | |
| - cron: "0 2 * * 0" | |
| workflow_dispatch: # Allow manual triggering | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| FORCE_COLOR: 1 | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} | |
| TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/gcp_jwt_key.json | |
| WEEKLY_TESTS: "test_slow_optimization_openai_rft" | |
| jobs: | |
| optimization-tests: | |
| if: github.repository == 'tensorzero/tensorzero' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| - name: Write GCP JWT key to file | |
| env: | |
| GCP_JWT_KEY: ${{ secrets.GCP_JWT_KEY }} | |
| run: echo "$GCP_JWT_KEY" > $GITHUB_WORKSPACE/gcp_jwt_key.json | |
| - name: Install Rust | |
| run: | | |
| for attempt in 1 2 3; do | |
| if rustup toolchain install stable && rustup default stable; then | |
| break | |
| fi | |
| if [ $attempt -eq 3 ]; then | |
| echo "Failed to install Rust toolchain after 3 attempts" | |
| exit 1 | |
| fi | |
| sleep $((10 * attempt)) | |
| done | |
| shell: bash | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 | |
| with: | |
| cache-provider: "buildjet" | |
| shared-key: "optimization-test-cache" | |
| - name: Install cargo-nextest | |
| uses: taiki-e/install-action@d12e869b89167df346dd0ff65da342d1fb1202fb | |
| with: | |
| tool: cargo-nextest | |
| - name: Launch ClickHouse container for optimization tests | |
| run: docker compose -f tensorzero-optimizers/tests/docker-compose.yml up --wait | |
| - name: Run daily optimization tests | |
| if: github.event.schedule == '0 2 * * *' || github.event_name == 'workflow_dispatch' | |
| run: TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests cargo test-optimization --nocapture --no-fail-fast -- --skip ${{ env.WEEKLY_TESTS }} | |
| - name: Run weekly optimization tests | |
| if: github.event.schedule == '0 2 * * 0' | |
| run: TENSORZERO_CLICKHOUSE_URL=http://chuser:chpassword@localhost:8123/tensorzero_e2e_tests cargo test-optimization --nocapture --no-fail-fast ${{ env.WEEKLY_TESTS }} | |
| - name: Print ClickHouse container logs | |
| if: always() | |
| continue-on-error: true | |
| run: docker compose -f tensorzero-optimizers/tests/docker-compose.yml logs -t | |
| - name: Log test completion | |
| if: always() | |
| run: | | |
| echo "Optimization tests completed at $(date)" | |
| echo "Test result: ${{ job.status }}" | |
| - name: Post to Slack on failure | |
| if: failure() | |
| uses: slackapi/slack-github-action@v2.1.1 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| channel: C09DM0RGDFG | |
| text: Live optimization tests failed for run <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}> |