From e139716e331501265dc4ae19831d56b51981a38c Mon Sep 17 00:00:00 2001 From: dasm Date: Fri, 12 Sep 2025 16:23:48 -0700 Subject: [PATCH 1/4] Update nox-utils. --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 8a42394..579a49c 100644 --- a/uv.lock +++ b/uv.lock @@ -2478,8 +2478,8 @@ test = [ [[package]] name = "tmlt-nox-utils" -version = "0.0.0.post15+60b02cab" -source = { git = "https://github.com/opendp/tumult-tools.git?subdirectory=nox-utils#60b02cabf3b5bd0d20fa51c4598c92622f2c65e9" } +version = "0.0.0.post25+eabe1054" +source = { git = "https://github.com/opendp/tumult-tools.git?subdirectory=nox-utils#eabe1054863f0916a0087ad180fd83719049c094" } dependencies = [ { name = "gitpython" }, { name = "nox" }, From 7e799bba1d0613ab0c961ea62c1cb5b1f5bf5be2 Mon Sep 17 00:00:00 2001 From: dasm Date: Sat, 13 Sep 2025 21:21:27 -0700 Subject: [PATCH 2/4] Add dependency matrix nos session --- noxfile.py | 161 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 103 insertions(+), 58 deletions(-) diff --git a/noxfile.py b/noxfile.py index dae24b3..820d9b1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,7 +14,7 @@ import nox from nox import session as session -from tmlt.nox_utils import SessionManager +from tmlt.nox_utils import DependencyConfiguration, SessionManager CWD = Path(".").resolve() @@ -33,74 +33,117 @@ """For test suites where we track coverage (i.e. the fast tests and the full test suite), fail if test coverage falls below this percentage.""" -DEPENDENCY_MATRIX = { - name: { - # The Python minor version to run with - "python": python, - # All other entries take PEP440 version specifiers for the package named in - # the key -- see https://peps.python.org/pep-0440/#version-specifiers - "pyspark[sql]": pyspark, - "sympy": sympy, - "pandas": pandas, - "numpy": numpy, - "scipy": scipy, - "randomgen": randomgen, - "pyarrow": pyarrow, - } - for (name, python, pyspark, sympy, pandas, numpy, scipy, randomgen, pyarrow) in [ - # fmt: off - # name - # python pyspark sympy pandas - # numpy scipy randomgen pyarrow - ( - "3.9-oldest", - "3.9", "==3.3.1", "==1.8", "==1.4.0", - "==1.23.2", "==1.6.0", "==1.20.0", "==14.0.1", +DEPENDENCY_MATRIX = [ + DependencyConfiguration( + id="3.9-oldest", python="3.9", + packages={ + "pyspark[sql]": "==3.3.1", + "sympy": "==1.8", + "pandas": "==1.4.0", + "numpy": "==1.23.2", + "scipy": "==1.6.0", + "randomgen": "==1.20.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.9-pyspark3.4", - "3.9", "==3.4.0", "==1.9", "==1.5.3", - "==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.9-pyspark3.4", python="3.9", + packages={ + "pyspark[sql]": "==3.4.0", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.13.1", + "randomgen": "==1.26.0", + "pyarrow": "==16.1.0", + } ), - ( - "3.9-newest", - "3.9", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.9-newest", python="3.9", + packages={ + "pyspark[sql]": "==3.5.1", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.13.1", + "randomgen": "==1.26.0", + "pyarrow": "==16.1.0", + } ), - ( - "3.10-oldest", - "3.10", "==3.1.1", "==1.8", "==1.4.0", - "==1.23.2", "==1.8.0", "==1.23.0", "==14.0.1", + DependencyConfiguration( + id="3.10-oldest", python="3.10", + packages={ + "pyspark[sql]": "==3.1.1", + "sympy": "==1.8", + "pandas": "==1.4.0", + "numpy": "==1.23.2", + "scipy": "==1.8.0", + "randomgen": "==1.23.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.10-newest", - "3.10", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.14.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.10-newest", python="3.10", + packages={ + "pyspark[sql]": "==3.5.1", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.0", + "pyarrow": "==16.1.0", + } ), - ( - "3.11-oldest", - "3.11", "==3.4.0", "==1.8", "==1.5.0", - "==1.23.2", "==1.9.2", "==1.26.0", "==14.0.1", + DependencyConfiguration( + id="3.11-oldest", python="3.11", + packages={ + "pyspark[sql]": "==3.4.0", + "sympy": "==1.8", + "pandas": "==1.5.0", + "numpy": "==1.23.2", + "scipy": "==1.9.2", + "randomgen": "==1.26.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.11-newest", - "3.11", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0", + DependencyConfiguration( + id="3.11-newest", python="3.11", + packages={ + "pyspark[sql]": "==3.5.1", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.1", + "pyarrow": "==16.1.0", + } ), - ( - "3.12-oldest", - "3.12", "==3.5.0", "==1.8", "==2.2.0", - "==1.26.0", "==1.11.2", "==1.26.0", "==14.0.1", + DependencyConfiguration( + id="3.12-oldest", python="3.12", + packages={ + "pyspark[sql]": "==3.5.0", + "sympy": "==1.8", + "pandas": "==2.2.0", + "numpy": "==1.26.0", + "scipy": "==1.11.2", + "randomgen": "==1.26.0", + "pyarrow": "==14.0.1" + }, ), # 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797 - ( - "3.12-newest", - "3.12", "==3.5.1", "==1.9", "==2.2.2", - "==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0", + DependencyConfiguration( + id="3.12-newest", python="3.12", + packages={ + "pyspark[sql]": "==3.5.1", + "sympy": "==1.9", + "pandas": "==2.2.2", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.1", + "pyarrow": "==16.1.0", + } ), - # fmt: on ] -} AUDIT_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] AUDIT_SUPPRESSIONS = [ @@ -157,5 +200,7 @@ sm.make_release() +sm.test_dependency_matrix(dependency_matrix=DEPENDENCY_MATRIX) + for name, timeout in BENCHMARKS: sm.benchmark(Path('benchmark') / f"{name}.py", timeout) From 36cc9da5309bfaba387ed2f2538d85b5f1e2caec Mon Sep 17 00:00:00 2001 From: dasm Date: Sat, 13 Sep 2025 21:23:29 -0700 Subject: [PATCH 3/4] Increase minimum pyspark version to 3.5.0. --- noxfile.py | 26 +++++++------------------- pyproject.toml | 4 +--- uv.lock | 4 +--- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/noxfile.py b/noxfile.py index 820d9b1..b7298a2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -37,7 +37,7 @@ DependencyConfiguration( id="3.9-oldest", python="3.9", packages={ - "pyspark[sql]": "==3.3.1", + "pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "numpy": "==1.23.2", @@ -46,22 +46,10 @@ "pyarrow": "==14.0.1", } ), - DependencyConfiguration( - id="3.9-pyspark3.4", python="3.9", - packages={ - "pyspark[sql]": "==3.4.0", - "sympy": "==1.9", - "pandas": "==1.5.3", - "numpy": "==1.26.4", - "scipy": "==1.13.1", - "randomgen": "==1.26.0", - "pyarrow": "==16.1.0", - } - ), DependencyConfiguration( id="3.9-newest", python="3.9", packages={ - "pyspark[sql]": "==3.5.1", + "pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "numpy": "==1.26.4", @@ -73,7 +61,7 @@ DependencyConfiguration( id="3.10-oldest", python="3.10", packages={ - "pyspark[sql]": "==3.1.1", + "pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "numpy": "==1.23.2", @@ -85,7 +73,7 @@ DependencyConfiguration( id="3.10-newest", python="3.10", packages={ - "pyspark[sql]": "==3.5.1", + "pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "numpy": "==1.26.4", @@ -97,7 +85,7 @@ DependencyConfiguration( id="3.11-oldest", python="3.11", packages={ - "pyspark[sql]": "==3.4.0", + "pyspark[sql]": "==3.5.0", "sympy": "==1.8", "pandas": "==1.5.0", "numpy": "==1.23.2", @@ -109,7 +97,7 @@ DependencyConfiguration( id="3.11-newest", python="3.11", packages={ - "pyspark[sql]": "==3.5.1", + "pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==1.5.3", "numpy": "==1.26.4", @@ -134,7 +122,7 @@ DependencyConfiguration( id="3.12-newest", python="3.12", packages={ - "pyspark[sql]": "==3.5.1", + "pyspark[sql]": "==3.5.6", "sympy": "==1.9", "pandas": "==2.2.2", "numpy": "==1.26.4", diff --git a/pyproject.toml b/pyproject.toml index 7ccf96f..c3d0562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,9 +35,7 @@ dependencies = [ "pandas >=1.5.0,<2 ; python_version == '3.11'", "pandas >=2.2.0,<3 ; python_version >= '3.12'", "pyarrow >=18.0.0,<19", - "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11'", - "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11'", - "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12'", + "pyspark[sql] >=3.5.0,<3.6", "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", diff --git a/uv.lock b/uv.lock index 579a49c..f45c14d 100644 --- a/uv.lock +++ b/uv.lock @@ -2426,9 +2426,7 @@ requires-dist = [ { name = "pandas", marker = "python_full_version >= '3.12'", specifier = ">=2.2.0,<3" }, { name = "parameterized", specifier = ">=0.7.4,<0.8" }, { name = "pyarrow", specifier = ">=18.0.0,<19" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11'", specifier = ">=3.3.1,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*'", specifier = ">=3.4.0,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12'", specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], specifier = ">=3.5.0,<3.6" }, { name = "pytest", specifier = ">=7.3.1,<8" }, { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" }, From 8a6661e09eb7ec412dbad919c7b1e459dd341efb Mon Sep 17 00:00:00 2001 From: dasm Date: Wed, 17 Sep 2025 11:06:36 -0700 Subject: [PATCH 4/4] Fix minimum arm mac dependency versions. --- noxfile.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index b7298a2..b6eb5ea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,6 +10,8 @@ indicate this in their docstrings. """ +import platform +import sys from pathlib import Path import nox @@ -33,6 +35,10 @@ """For test suites where we track coverage (i.e. the fast tests and the full test suite), fail if test coverage falls below this percentage.""" +def is_arm_mac(): + """Returns true if the current system is am arm-based mac.""" + return sys.platform == "darwin" and platform.processor() == "arm" + DEPENDENCY_MATRIX = [ DependencyConfiguration( id="3.9-oldest", python="3.9", @@ -41,8 +47,11 @@ "sympy": "==1.8", "pandas": "==1.4.0", "numpy": "==1.23.2", - "scipy": "==1.6.0", - "randomgen": "==1.20.0", + # Scipy 1.7.3 is the first to include arm mac wheels, but it is + # incompatible with numpy 1.23.2, so we use the next version. + "scipy": "==1.6.0" if not is_arm_mac else "==1.8.0", + # Randomgen 1.23.0 is the first version to include arm mac wheels. + "randomgen": "==1.20.0" if not is_arm_mac() else "==1.23.0", "pyarrow": "==14.0.1", } ),