这是indexloc提供的服务，不要输入任何密码
Skip to content

Add dependency matrix nox session #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 18, 2025
Merged

Add dependency matrix nox session #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e139716
Update nox-utils.
Maegereg Sep 12, 2025
7e799bb
Add dependency matrix nos session
Maegereg Sep 14, 2025
36cc9da
Increase minimum pyspark version to 3.5.0.
Maegereg Sep 14, 2025
8a6661e
Fix minimum arm mac dependency versions.
Maegereg Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 101 additions & 59 deletions noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@
indicate this in their docstrings.
"""

import platform
import sys
from pathlib import Path

import nox
from nox import session as session
from tmlt.nox_utils import SessionManager
from tmlt.nox_utils import DependencyConfiguration, SessionManager

CWD = Path(".").resolve()

Expand All @@ -33,74 +35,112 @@
"""For test suites where we track coverage (i.e. the fast tests and the full
test suite), fail if test coverage falls below this percentage."""

DEPENDENCY_MATRIX = {
name: {
# The Python minor version to run with
"python": python,
# All other entries take PEP440 version specifiers for the package named in
# the key -- see https://peps.python.org/pep-0440/#version-specifiers
"pyspark[sql]": pyspark,
"sympy": sympy,
"pandas": pandas,
"numpy": numpy,
"scipy": scipy,
"randomgen": randomgen,
"pyarrow": pyarrow,
}
for (name, python, pyspark, sympy, pandas, numpy, scipy, randomgen, pyarrow) in [
# fmt: off
# name
# python pyspark sympy pandas
# numpy scipy randomgen pyarrow
(
"3.9-oldest",
"3.9", "==3.3.1", "==1.8", "==1.4.0",
"==1.23.2", "==1.6.0", "==1.20.0", "==14.0.1",
def is_arm_mac():
"""Returns true if the current system is am arm-based mac."""
return sys.platform == "darwin" and platform.processor() == "arm"

DEPENDENCY_MATRIX = [
DependencyConfiguration(
id="3.9-oldest", python="3.9",
packages={
"pyspark[sql]": "==3.5.0",
"sympy": "==1.8",
"pandas": "==1.4.0",
"numpy": "==1.23.2",
# Scipy 1.7.3 is the first to include arm mac wheels, but it is
# incompatible with numpy 1.23.2, so we use the next version.
"scipy": "==1.6.0" if not is_arm_mac else "==1.8.0",
# Randomgen 1.23.0 is the first version to include arm mac wheels.
"randomgen": "==1.20.0" if not is_arm_mac() else "==1.23.0",
"pyarrow": "==14.0.1",
}
),
(
"3.9-pyspark3.4",
"3.9", "==3.4.0", "==1.9", "==1.5.3",
"==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0",
DependencyConfiguration(
id="3.9-newest", python="3.9",
packages={
"pyspark[sql]": "==3.5.6",
"sympy": "==1.9",
"pandas": "==1.5.3",
"numpy": "==1.26.4",
"scipy": "==1.13.1",
"randomgen": "==1.26.0",
"pyarrow": "==16.1.0",
}
),
(
"3.9-newest",
"3.9", "==3.5.1", "==1.9", "==1.5.3",
"==1.26.4", "==1.13.1", "==1.26.0", "==16.1.0",
DependencyConfiguration(
id="3.10-oldest", python="3.10",
packages={
"pyspark[sql]": "==3.5.0",
"sympy": "==1.8",
"pandas": "==1.4.0",
"numpy": "==1.23.2",
"scipy": "==1.8.0",
"randomgen": "==1.23.0",
"pyarrow": "==14.0.1",
}
),
(
"3.10-oldest",
"3.10", "==3.1.1", "==1.8", "==1.4.0",
"==1.23.2", "==1.8.0", "==1.23.0", "==14.0.1",
DependencyConfiguration(
id="3.10-newest", python="3.10",
packages={
"pyspark[sql]": "==3.5.6",
"sympy": "==1.9",
"pandas": "==1.5.3",
"numpy": "==1.26.4",
"scipy": "==1.14.1",
"randomgen": "==1.26.0",
"pyarrow": "==16.1.0",
}
),
(
"3.10-newest",
"3.10", "==3.5.1", "==1.9", "==1.5.3",
"==1.26.4", "==1.14.1", "==1.26.0", "==16.1.0",
DependencyConfiguration(
id="3.11-oldest", python="3.11",
packages={
"pyspark[sql]": "==3.5.0",
"sympy": "==1.8",
"pandas": "==1.5.0",
"numpy": "==1.23.2",
"scipy": "==1.9.2",
"randomgen": "==1.26.0",
"pyarrow": "==14.0.1",
}
),
(
"3.11-oldest",
"3.11", "==3.4.0", "==1.8", "==1.5.0",
"==1.23.2", "==1.9.2", "==1.26.0", "==14.0.1",
DependencyConfiguration(
id="3.11-newest", python="3.11",
packages={
"pyspark[sql]": "==3.5.6",
"sympy": "==1.9",
"pandas": "==1.5.3",
"numpy": "==1.26.4",
"scipy": "==1.14.1",
"randomgen": "==1.26.1",
"pyarrow": "==16.1.0",
}
),
(
"3.11-newest",
"3.11", "==3.5.1", "==1.9", "==1.5.3",
"==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0",
),
(
"3.12-oldest",
"3.12", "==3.5.0", "==1.8", "==2.2.0",
"==1.26.0", "==1.11.2", "==1.26.0", "==14.0.1",
DependencyConfiguration(
id="3.12-oldest", python="3.12",
packages={
"pyspark[sql]": "==3.5.0",
"sympy": "==1.8",
"pandas": "==2.2.0",
"numpy": "==1.26.0",
"scipy": "==1.11.2",
"randomgen": "==1.26.0",
"pyarrow": "==14.0.1"
},
),
# 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797
(
"3.12-newest",
"3.12", "==3.5.1", "==1.9", "==2.2.2",
"==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0",
DependencyConfiguration(
id="3.12-newest", python="3.12",
packages={
"pyspark[sql]": "==3.5.6",
"sympy": "==1.9",
"pandas": "==2.2.2",
"numpy": "==1.26.4",
"scipy": "==1.14.1",
"randomgen": "==1.26.1",
"pyarrow": "==16.1.0",
}
),
# fmt: on
]
}

AUDIT_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
AUDIT_SUPPRESSIONS = [
Expand Down Expand Up @@ -157,5 +197,7 @@

sm.make_release()

sm.test_dependency_matrix(dependency_matrix=DEPENDENCY_MATRIX)

for name, timeout in BENCHMARKS:
sm.benchmark(Path('benchmark') / f"{name}.py", timeout)
4 changes: 1 addition & 3 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,7 @@ dependencies = [
"pandas >=1.5.0,<2 ; python_version == '3.11'",
"pandas >=2.2.0,<3 ; python_version >= '3.12'",
"pyarrow >=18.0.0,<19",
"pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11'",
"pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11'",
"pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12'",
"pyspark[sql] >=3.5.0,<3.6",
"randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'",
"randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'",
"randomgen >=1.26.0,<=2 ; python_version >= '3.11'",
Expand Down
8 changes: 3 additions & 5 deletions uv.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.