diff --git a/noxfile.py b/noxfile.py index dae24b3..b6eb5ea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -10,11 +10,13 @@ indicate this in their docstrings. """ +import platform +import sys from pathlib import Path import nox from nox import session as session -from tmlt.nox_utils import SessionManager +from tmlt.nox_utils import DependencyConfiguration, SessionManager CWD = Path(".").resolve() @@ -33,74 +35,112 @@ """For test suites where we track coverage (i.e. the fast tests and the full test suite), fail if test coverage falls below this percentage.""" -DEPENDENCY_MATRIX = { - name: { - # The Python minor version to run with - "python": python, - # All other entries take PEP440 version specifiers for the package named in - # the key -- see https://peps.python.org/pep-0440/#version-specifiers - "pyspark[sql]": pyspark, - "sympy": sympy, - "pandas": pandas, - "numpy": numpy, - "scipy": scipy, - "randomgen": randomgen, - "pyarrow": pyarrow, - } - for (name, python, pyspark, sympy, pandas, numpy, scipy, randomgen, pyarrow) in [ - # fmt: off - # name - # python pyspark sympy pandas - # numpy scipy randomgen pyarrow - ( - "3.9-oldest", - "3.9", "==3.3.1", "==1.8", "==1.4.0", - "==1.23.2", "==1.6.0", "==1.20.0", "==14.0.1", +def is_arm_mac(): + """Returns true if the current system is am arm-based mac.""" + return sys.platform == "darwin" and platform.processor() == "arm" + +DEPENDENCY_MATRIX = [ + DependencyConfiguration( + id="3.9-oldest", python="3.9", + packages={ + "pyspark[sql]": "==3.5.0", + "sympy": "==1.8", + "pandas": "==1.4.0", + "numpy": "==1.23.2", + # Scipy 1.7.3 is the first to include arm mac wheels, but it is + # incompatible with numpy 1.23.2, so we use the next version. + "scipy": "==1.6.0" if not is_arm_mac else "==1.8.0", + # Randomgen 1.23.0 is the first version to include arm mac wheels. + "randomgen": "==1.20.0" if not is_arm_mac() else "==1.23.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.9-pyspark3.4", - "3.9", "==3.4.0", "==1.9", "==1.5.3", - "==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.9-newest", python="3.9", + packages={ + "pyspark[sql]": "==3.5.6", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.13.1", + "randomgen": "==1.26.0", + "pyarrow": "==16.1.0", + } ), - ( - "3.9-newest", - "3.9", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.10-oldest", python="3.10", + packages={ + "pyspark[sql]": "==3.5.0", + "sympy": "==1.8", + "pandas": "==1.4.0", + "numpy": "==1.23.2", + "scipy": "==1.8.0", + "randomgen": "==1.23.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.10-oldest", - "3.10", "==3.1.1", "==1.8", "==1.4.0", - "==1.23.2", "==1.8.0", "==1.23.0", "==14.0.1", + DependencyConfiguration( + id="3.10-newest", python="3.10", + packages={ + "pyspark[sql]": "==3.5.6", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.0", + "pyarrow": "==16.1.0", + } ), - ( - "3.10-newest", - "3.10", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.14.1", "==1.26.0", "==16.1.0", + DependencyConfiguration( + id="3.11-oldest", python="3.11", + packages={ + "pyspark[sql]": "==3.5.0", + "sympy": "==1.8", + "pandas": "==1.5.0", + "numpy": "==1.23.2", + "scipy": "==1.9.2", + "randomgen": "==1.26.0", + "pyarrow": "==14.0.1", + } ), - ( - "3.11-oldest", - "3.11", "==3.4.0", "==1.8", "==1.5.0", - "==1.23.2", "==1.9.2", "==1.26.0", "==14.0.1", + DependencyConfiguration( + id="3.11-newest", python="3.11", + packages={ + "pyspark[sql]": "==3.5.6", + "sympy": "==1.9", + "pandas": "==1.5.3", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.1", + "pyarrow": "==16.1.0", + } ), - ( - "3.11-newest", - "3.11", "==3.5.1", "==1.9", "==1.5.3", - "==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0", - ), - ( - "3.12-oldest", - "3.12", "==3.5.0", "==1.8", "==2.2.0", - "==1.26.0", "==1.11.2", "==1.26.0", "==14.0.1", + DependencyConfiguration( + id="3.12-oldest", python="3.12", + packages={ + "pyspark[sql]": "==3.5.0", + "sympy": "==1.8", + "pandas": "==2.2.0", + "numpy": "==1.26.0", + "scipy": "==1.11.2", + "randomgen": "==1.26.0", + "pyarrow": "==14.0.1" + }, ), # 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797 - ( - "3.12-newest", - "3.12", "==3.5.1", "==1.9", "==2.2.2", - "==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0", + DependencyConfiguration( + id="3.12-newest", python="3.12", + packages={ + "pyspark[sql]": "==3.5.6", + "sympy": "==1.9", + "pandas": "==2.2.2", + "numpy": "==1.26.4", + "scipy": "==1.14.1", + "randomgen": "==1.26.1", + "pyarrow": "==16.1.0", + } ), - # fmt: on ] -} AUDIT_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] AUDIT_SUPPRESSIONS = [ @@ -157,5 +197,7 @@ sm.make_release() +sm.test_dependency_matrix(dependency_matrix=DEPENDENCY_MATRIX) + for name, timeout in BENCHMARKS: sm.benchmark(Path('benchmark') / f"{name}.py", timeout) diff --git a/pyproject.toml b/pyproject.toml index 7ccf96f..c3d0562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,9 +35,7 @@ dependencies = [ "pandas >=1.5.0,<2 ; python_version == '3.11'", "pandas >=2.2.0,<3 ; python_version >= '3.12'", "pyarrow >=18.0.0,<19", - "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11'", - "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11'", - "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12'", + "pyspark[sql] >=3.5.0,<3.6", "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", diff --git a/uv.lock b/uv.lock index 8a42394..f45c14d 100644 --- a/uv.lock +++ b/uv.lock @@ -2426,9 +2426,7 @@ requires-dist = [ { name = "pandas", marker = "python_full_version >= '3.12'", specifier = ">=2.2.0,<3" }, { name = "parameterized", specifier = ">=0.7.4,<0.8" }, { name = "pyarrow", specifier = ">=18.0.0,<19" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11'", specifier = ">=3.3.1,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*'", specifier = ">=3.4.0,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12'", specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], specifier = ">=3.5.0,<3.6" }, { name = "pytest", specifier = ">=7.3.1,<8" }, { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" }, @@ -2478,8 +2476,8 @@ test = [ [[package]] name = "tmlt-nox-utils" -version = "0.0.0.post15+60b02cab" -source = { git = "https://github.com/opendp/tumult-tools.git?subdirectory=nox-utils#60b02cabf3b5bd0d20fa51c4598c92622f2c65e9" } +version = "0.0.0.post25+eabe1054" +source = { git = "https://github.com/opendp/tumult-tools.git?subdirectory=nox-utils#eabe1054863f0916a0087ad180fd83719049c094" } dependencies = [ { name = "gitpython" }, { name = "nox" },