opendp · Maegereg · Sep 12, 2025 · Aug 31, 2025 · Sep 10, 2025 · Sep 10, 2025
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Set up runner
-        uses: opendp/tumult-tools/actions/setup@0f3d49599e5824a9f407a6e2063990c1e0d4c2e8
+        uses: opendp/tumult-tools/actions/setup@eabe1054863f0916a0087ad180fd83719049c094
       - run: uv run nox -s build
       - name: Archive packaged library
         uses: actions/upload-artifact@v4
@@ -50,7 +50,7 @@ jobs:
       - name: Checkout code repository
         uses: actions/checkout@v4
       - name: Set up runner
-        uses: opendp/tumult-tools/actions/setup@0f3d49599e5824a9f407a6e2063990c1e0d4c2e8
+        uses: opendp/tumult-tools/actions/setup@eabe1054863f0916a0087ad180fd83719049c094
       - name: Download dist
         uses: actions/download-artifact@v4
         with:
@@ -66,20 +66,46 @@ jobs:
       - name: Checkout code repository
         uses: actions/checkout@v4
       - name: Set up runner
-        uses: opendp/tumult-tools/actions/setup@0f3d49599e5824a9f407a6e2063990c1e0d4c2e8
+        uses: opendp/tumult-tools/actions/setup@eabe1054863f0916a0087ad180fd83719049c094
       - name: Download dist
         uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist
       - run: uv run nox -t benchmark
 
+  Dependency-Matrix:
+    if: github.repository == 'opendp/tumult-analytics'
+    strategy:
+      fail-fast: false
+      matrix:
+        # Note: mac runners are rather expensive (10x multiplier) so we don't use them here.
+        os: [ubuntu-latest]
+        dependencies: [oldest, newest]
+        python: ["3.9", "3.12"]
+    runs-on: ${{ matrix.os }}
+    needs: Package
+    steps:
+      - name: Checkout code repository
+        uses: actions/checkout@v4
+      - name: Set up runner
+        uses: opendp/tumult-tools/actions/setup@eabe1054863f0916a0087ad180fd83719049c094
+      - name: Download dist
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - run: uv run nox -s "test_dependency_matrix(${{matrix.python}}-${{matrix.dependencies}})"
+        env: 
+          SPARK_LOCAL_HOSTNAME: localhost
+
   Publish-To-PyPI:
     if: github.repository == 'opendp/tumult-analytics'
     runs-on: ubuntu-latest
     needs:
       - Test-Slow
       - Benchmark
+      - Dependency-Matrix
     environment:
       name: pypi
       url: https://pypi.org/p/tmlt-analytics
@@ -106,7 +132,7 @@ jobs:
       - name: Checkout code repository
         uses: actions/checkout@v4
       - name: Set up runner
-        uses: opendp/tumult-tools/actions/setup@0f3d49599e5824a9f407a6e2063990c1e0d4c2e8
+        uses: opendp/tumult-tools/actions/setup@eabe1054863f0916a0087ad180fd83719049c094
       - name: Download dist
         uses: actions/download-artifact@v4
         with:
@@ -124,3 +150,4 @@ jobs:
           docs-repository-token: ${{ secrets.DOCS_REPO_PAT }}
           docs-path: docs/analytics
           version: ${{ format('v{0}.{1}', env.MAJOR_VERSION, env.MINOR_VERSION) }}
+
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -12,6 +12,8 @@ Unreleased
 
 Changed
 ~~~~~~~
+- Dropped support for pyspark <3.5.0 after discovering that it does not work on Macs (this may not work for older versions as well).
+
 
 .. _v0.20.2:
 

diff --git a/noxfile.py b/noxfile.py
@@ -5,11 +5,10 @@
 API reference.
 """
 
-import os
 from pathlib import Path
 
 import nox
-from tmlt.nox_utils import SessionManager
+from tmlt.nox_utils import DependencyConfiguration, SessionManager
 
 nox.options.default_venv_backend = "uv|virtualenv"
 
@@ -29,31 +28,18 @@
 """For test suites where we track coverage (i.e. the fast tests and the full
 test suite), fail if test coverage falls below this percentage."""
 
-DEPENDENCY_MATRIX = {
-    name: {
-        # The Python minor version to run with
-        "python": python,
-        # All other entries take PEP440 version specifiers for the package named in
-        # the key -- see https://peps.python.org/pep-0440/#version-specifiers
-        "pyspark[sql]": pyspark,
-        "sympy": sympy,
-        "pandas": pandas,
-        "tmlt.core": core,
-    }
-    for (name, python, pyspark, sympy, pandas, core) in [
-        # fmt: off
-        ("3.9-oldest",     "3.9",  "==3.3.1", "==1.8", "==1.4.0", "==0.18.0"),
-        ("3.9-pyspark3.4", "3.9",  "==3.4.0", "==1.9", "==1.5.3", ">=0.18.0"),
-        ("3.9-newest",     "3.9",  "==3.5.1", "==1.9", "==1.5.3", ">=0.18.0"),
-        ("3.10-oldest",    "3.10", "==3.3.1", "==1.8", "==1.4.0", "==0.18.0"),
-        ("3.10-newest",    "3.10", "==3.5.1", "==1.9", "==1.5.3", ">=0.18.0"),
-        ("3.11-oldest",    "3.11", "==3.4.0", "==1.8", "==1.5.0", "==0.18.0"),
-        ("3.11-newest",    "3.11", "==3.5.1", "==1.9", "==1.5.3", ">=0.18.0"),
-        ("3.12-oldest",    "3.12", "==3.5.0", "==1.8", "==2.2.0", "==0.18.0"),
-        ("3.12-newest",    "3.12", "==3.5.1", "==1.9", "==2.2.3", ">=0.18.0"),
-        # fmt: on
-    ]
-}
+DEPENDENCY_MATRIX = [
+    #fmt: off
+    DependencyConfiguration(id="3.9-oldest",     python="3.9",  packages={"pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "tmlt.core": "==0.18.0"}),
+    DependencyConfiguration(id="3.9-newest",     python="3.9",  packages={"pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "tmlt.core": ">=0.18.0"}),
+    DependencyConfiguration(id="3.10-oldest",    python="3.10", packages={"pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "tmlt.core": "==0.18.0"}),
+    DependencyConfiguration(id="3.10-newest",    python="3.10", packages={"pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "tmlt.core": ">=0.18.0"}),
+    DependencyConfiguration(id="3.11-oldest",    python="3.11", packages={"pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.5.0", "tmlt.core": "==0.18.0"}),
+    DependencyConfiguration(id="3.11-newest",    python="3.11", packages={"pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "tmlt.core": ">=0.18.0"}),
+    DependencyConfiguration(id="3.12-oldest",    python="3.12", packages={"pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==2.2.0", "tmlt.core": "==0.18.0"}),
+    DependencyConfiguration(id="3.12-newest",    python="3.12", packages={"pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==2.2.3", "tmlt.core": ">=0.18.0"}),
+    #fmt: on
+]
 
 AUDIT_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
 AUDIT_SUPPRESSIONS = [
@@ -120,3 +106,5 @@
 sm.audit()
 
 sm.make_release()
+
+sm.test_dependency_matrix(DEPENDENCY_MATRIX)
diff --git a/pyproject.toml b/pyproject.toml
@@ -35,9 +35,7 @@ dependencies = [
     "pandas >=1.4.0,<2 ; python_version < '3.11'",
     "pandas >=1.5.0,<2 ; python_version == '3.11'",
     "pandas >=2.2.0,<3 ; python_version >= '3.12'",
-    "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11'",
-    "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11'",
-    "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12'",
+    "pyspark[sql] >=3.5.0,<3.6",
     "sympy >=1.8,<1.13",
     "typeguard >=4.0.0,<5",
     "typing-extensions >=4.1.0,<5",

diff --git a/uv.lock b/uv.lock
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,8 @@ Unreleased @@
     Changed
     ~~~~~~~
+    - Dropped support for pyspark <3.5.0 after discovering that it does not work on Macs (this may not work for older versions as well).
     .. _v0.20.2:
@@ Expand Down @@