From 71d05fc5bf1c3f7c9d4f0929b913de431a1a575b Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Thu, 6 Nov 2025 21:28:09 -0700 Subject: [PATCH 1/5] ci: debugging for compilation issues --- .github/workflows/turborepo-test.yml | 210 ++++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 1 deletion(-) diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index 51e70a4c4a23d..3c16763574bf0 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -109,7 +109,6 @@ jobs: needs: - find-changes runs-on: ${{ matrix.os.runner }} - timeout-minutes: 45 if: ${{ needs.find-changes.outputs.rest == 'true' }} strategy: fail-fast: false @@ -160,7 +159,21 @@ jobs: - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.6 + - name: Enable Rust verbose logging + run: | + echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + shell: bash + + - name: Log Rust environment info + run: | + echo "=== Rust Toolchain Information ===" + rustc --version --verbose + cargo --version --verbose + rustup show + shell: bash + - name: Integration Tests + timeout-minutes: 45 run: | if [ -z "${RUSTC_WRAPPER}" ]; then unset RUSTC_WRAPPER @@ -176,6 +189,78 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Capture state on timeout + if: cancelled() + run: | + echo "==========================================" + echo "INTEGRATION TEST STEP TIMED OUT" + echo "==========================================" + echo "Time: $(date)" + echo "" + echo "=== Active Rust/Cargo Processes (likely hung) ===" + ps aux | grep -E "[r]ustc|[c]argo" || echo "No rustc/cargo processes found" + echo "" + echo "=== Top CPU Consumers ===" + if [ "$RUNNER_OS" == "Linux" ]; then + ps aux --sort=-%cpu | head -15 + elif [ "$RUNNER_OS" == "macOS" ]; then + ps aux -r | head -15 + fi + echo "" + echo "(ICE files will be checked in next step)" + shell: bash + + - name: Check for and display ICE files + if: always() + run: | + echo "=== Searching for Rust ICE (Internal Compiler Error) files ===" + ICE_FOUND=0 + + # Search in common locations + if [ "$RUNNER_OS" == "Windows" ]; then + # Windows paths + SEARCH_PATHS=( + "." + "$TEMP" + "$TMP" + "$USERPROFILE/.rustup/tmp" + ) + else + # Unix paths + SEARCH_PATHS=( + "." + "/tmp" + "$HOME/.rustup/tmp" + "./target" + ) + fi + + for search_path in "${SEARCH_PATHS[@]}"; do + if [ -d "$search_path" ]; then + echo "Searching in: $search_path" + while IFS= read -r ice_file; do + if [ -f "$ice_file" ]; then + ICE_FOUND=1 + echo "" + echo "========================================" + echo "ICE FILE FOUND: $ice_file" + echo "========================================" + cat "$ice_file" + echo "" + echo "========================================" + echo "" + fi + done < <(find "$search_path" -maxdepth 3 -name "rustc-ice-*.txt" -type f 2>/dev/null || true) + fi + done + + if [ $ICE_FOUND -eq 0 ]; then + echo "No ICE files found." + else + echo "WARNING: Rust Internal Compiler Errors detected! See above for details." + fi + shell: bash + rust_lint: name: Rust lints needs: @@ -290,6 +375,33 @@ jobs: - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.6 + - name: Enable Rust verbose logging + run: | + echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + shell: bash + + - name: Log Rust environment info + run: | + echo "=== Rust Toolchain Information ===" + rustc --version --verbose + cargo --version --verbose + rustup show + echo "=== System Resources ===" + if [ "$RUNNER_OS" == "Linux" ]; then + echo "CPUs: $(nproc)" + free -h + df -h + elif [ "$RUNNER_OS" == "macOS" ]; then + echo "CPUs: $(sysctl -n hw.ncpu)" + vm_stat + df -h + elif [ "$RUNNER_OS" == "Windows" ]; then + echo "CPUs: $NUMBER_OF_PROCESSORS" + wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value + wmic logicaldisk get size,freespace,caption + fi + shell: bash + - name: Run tests timeout-minutes: 120 # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues @@ -312,6 +424,102 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Capture state on timeout + if: cancelled() + run: | + echo "==========================================" + echo "RUST TEST STEP TIMED OUT" + echo "==========================================" + echo "Time: $(date)" + echo "" + echo "=== Active Rust/Cargo Processes (likely hung) ===" + ps aux | grep -E "[r]ustc|[c]argo" || echo "No rustc/cargo processes found" + echo "" + echo "=== Top CPU Consumers ===" + if [ "$RUNNER_OS" == "Linux" ]; then + ps aux --sort=-%cpu | head -15 + elif [ "$RUNNER_OS" == "macOS" ]; then + ps aux -r | head -15 + fi + echo "" + echo "(ICE files will be checked in next step)" + shell: bash + + - name: Check for and display ICE files + if: always() + run: | + echo "=== Searching for Rust ICE (Internal Compiler Error) files ===" + ICE_FOUND=0 + + # Search in common locations + if [ "$RUNNER_OS" == "Windows" ]; then + # Windows paths + SEARCH_PATHS=( + "." + "$TEMP" + "$TMP" + "$USERPROFILE/.rustup/tmp" + ) + else + # Unix paths + SEARCH_PATHS=( + "." + "/tmp" + "$HOME/.rustup/tmp" + "./target" + ) + fi + + for search_path in "${SEARCH_PATHS[@]}"; do + if [ -d "$search_path" ]; then + echo "Searching in: $search_path" + while IFS= read -r ice_file; do + if [ -f "$ice_file" ]; then + ICE_FOUND=1 + echo "" + echo "========================================" + echo "ICE FILE FOUND: $ice_file" + echo "========================================" + cat "$ice_file" + echo "" + echo "========================================" + echo "" + fi + done < <(find "$search_path" -maxdepth 3 -name "rustc-ice-*.txt" -type f 2>/dev/null || true) + fi + done + + if [ $ICE_FOUND -eq 0 ]; then + echo "No ICE files found." + else + echo "WARNING: Rust Internal Compiler Errors detected! See above for details." + fi + shell: bash + + - name: Display final system state + if: always() + run: | + echo "=== Final System State ===" + if [ "$RUNNER_OS" == "Linux" ]; then + echo "Memory:" + free -h + echo "Disk:" + df -h + echo "Top processes by CPU:" + ps aux --sort=-%cpu | head -10 || true + elif [ "$RUNNER_OS" == "macOS" ]; then + echo "Memory:" + vm_stat + echo "Disk:" + df -h + echo "Top processes by CPU:" + ps aux -r | head -10 || true + elif [ "$RUNNER_OS" == "Windows" ]; then + wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value + wmic logicaldisk get size,freespace,caption + fi + shell: bash + basic-example: name: "`basic` example" timeout-minutes: 40 From 0455a29c29aefab4a04ad501699c6529638cf233 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Thu, 6 Nov 2025 21:36:46 -0700 Subject: [PATCH 2/5] WIP cb2d7 --- .github/workflows/turborepo-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index 3c16763574bf0..6a51dda735b80 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -397,8 +397,8 @@ jobs: df -h elif [ "$RUNNER_OS" == "Windows" ]; then echo "CPUs: $NUMBER_OF_PROCESSORS" - wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value - wmic logicaldisk get size,freespace,caption + wmic.exe OS get FreePhysicalMemory,TotalVisibleMemorySize /Value + wmic.exe logicaldisk get size,freespace,caption fi shell: bash @@ -515,8 +515,8 @@ jobs: echo "Top processes by CPU:" ps aux -r | head -10 || true elif [ "$RUNNER_OS" == "Windows" ]; then - wmic OS get FreePhysicalMemory,TotalVisibleMemorySize /Value - wmic logicaldisk get size,freespace,caption + wmic.exe OS get FreePhysicalMemory,TotalVisibleMemorySize /Value + wmic.exe logicaldisk get size,freespace,caption fi shell: bash From ae47b69bb081e2d40bf83ac85776b48fe33ee9c5 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 7 Nov 2025 05:16:25 -0700 Subject: [PATCH 3/5] WIP 5906b --- .github/workflows/turborepo-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index 6a51dda735b80..a2a527d1eae22 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -397,8 +397,8 @@ jobs: df -h elif [ "$RUNNER_OS" == "Windows" ]; then echo "CPUs: $NUMBER_OF_PROCESSORS" - wmic.exe OS get FreePhysicalMemory,TotalVisibleMemorySize /Value - wmic.exe logicaldisk get size,freespace,caption + powershell "Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List" + powershell "Get-CimInstance Win32_LogicalDisk | Select-Object DeviceID,Size,FreeSpace | Format-Table -AutoSize" fi shell: bash @@ -515,8 +515,8 @@ jobs: echo "Top processes by CPU:" ps aux -r | head -10 || true elif [ "$RUNNER_OS" == "Windows" ]; then - wmic.exe OS get FreePhysicalMemory,TotalVisibleMemorySize /Value - wmic.exe logicaldisk get size,freespace,caption + powershell "Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List" + powershell "Get-CimInstance Win32_LogicalDisk | Select-Object DeviceID,Size,FreeSpace | Format-Table -AutoSize" fi shell: bash From 3cfbbd07b8908c2ae7f73c5a55311170885d854e Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 7 Nov 2025 05:40:23 -0700 Subject: [PATCH 4/5] WIP 4a82f --- .github/workflows/turborepo-test.yml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index a2a527d1eae22..a075c31968017 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -162,6 +162,8 @@ jobs: - name: Enable Rust verbose logging run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + echo "CARGO_LOG=cargo::core::compiler::fingerprint=info" >> $GITHUB_ENV + echo "RUST_LOG=info" >> $GITHUB_ENV shell: bash - name: Log Rust environment info @@ -170,6 +172,8 @@ jobs: rustc --version --verbose cargo --version --verbose rustup show + echo "=== sccache stats (before) ===" + sccache --show-stats || echo "sccache not available or failed" shell: bash - name: Integration Tests @@ -189,6 +193,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Show sccache stats + if: always() + run: | + echo "=== sccache stats (after) ===" + sccache --show-stats || echo "sccache not available or failed" + shell: bash + - name: Capture state on timeout if: cancelled() run: | @@ -378,6 +389,8 @@ jobs: - name: Enable Rust verbose logging run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + echo "CARGO_LOG=cargo::core::compiler::fingerprint=info" >> $GITHUB_ENV + echo "RUST_LOG=info" >> $GITHUB_ENV shell: bash - name: Log Rust environment info @@ -391,15 +404,19 @@ jobs: echo "CPUs: $(nproc)" free -h df -h + echo "File descriptors: $(ulimit -n)" elif [ "$RUNNER_OS" == "macOS" ]; then echo "CPUs: $(sysctl -n hw.ncpu)" vm_stat df -h + echo "File descriptors: $(ulimit -n)" elif [ "$RUNNER_OS" == "Windows" ]; then echo "CPUs: $NUMBER_OF_PROCESSORS" powershell "Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List" powershell "Get-CimInstance Win32_LogicalDisk | Select-Object DeviceID,Size,FreeSpace | Format-Table -AutoSize" fi + echo "=== sccache stats (before) ===" + sccache --show-stats || echo "sccache not available or failed" shell: bash - name: Run tests @@ -410,9 +427,9 @@ jobs: unset RUSTC_WRAPPER fi if [ "$RUNNER_OS" == "Windows" ]; then - cargo test --workspace --exclude turborepo-napi + cargo test --workspace --exclude turborepo-napi -vv else - cargo test --workspace + cargo test --workspace -vv fi shell: bash env: @@ -424,6 +441,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Show sccache stats + if: always() + run: | + echo "=== sccache stats (after) ===" + sccache --show-stats || echo "sccache not available or failed" + shell: bash + - name: Capture state on timeout if: cancelled() run: | From 5261d104568b6b6d699a82c672f1c8185f45021c Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 7 Nov 2025 06:03:43 -0700 Subject: [PATCH 5/5] WIP 55153 --- .github/workflows/turborepo-test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml index a075c31968017..1cf015d6ed5dd 100644 --- a/.github/workflows/turborepo-test.yml +++ b/.github/workflows/turborepo-test.yml @@ -162,7 +162,6 @@ jobs: - name: Enable Rust verbose logging run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV - echo "CARGO_LOG=cargo::core::compiler::fingerprint=info" >> $GITHUB_ENV echo "RUST_LOG=info" >> $GITHUB_ENV shell: bash @@ -192,6 +191,10 @@ jobs: CARGO_INCREMENTAL: 0 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # sccache timeout configuration to prevent hangs + SCCACHE_IDLE_TIMEOUT: 0 + SCCACHE_REQUEST_TIMEOUT: 30 + SCCACHE_ERROR_LOG: error - name: Show sccache stats if: always() @@ -389,7 +392,6 @@ jobs: - name: Enable Rust verbose logging run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV - echo "CARGO_LOG=cargo::core::compiler::fingerprint=info" >> $GITHUB_ENV echo "RUST_LOG=info" >> $GITHUB_ENV shell: bash @@ -427,9 +429,9 @@ jobs: unset RUSTC_WRAPPER fi if [ "$RUNNER_OS" == "Windows" ]; then - cargo test --workspace --exclude turborepo-napi -vv + cargo test --workspace --exclude turborepo-napi else - cargo test --workspace -vv + cargo test --workspace fi shell: bash env: @@ -440,6 +442,10 @@ jobs: CARGO_INCREMENTAL: 0 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # sccache timeout configuration to prevent hangs + SCCACHE_IDLE_TIMEOUT: 0 + SCCACHE_REQUEST_TIMEOUT: 30 + SCCACHE_ERROR_LOG: error - name: Show sccache stats if: always()