diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b65d688..24fe292 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,10 @@ For now, the repository remains available, and we encourage users to continue en Unreleased ---------- +Changed +~~~~~~~ +- Dropped support for pyspark <3.5.0 on Macs after discovering that these configurations frequently crash. Older versions of the library may also be affected. + .. _v0.18.2: 0.18.2 - 2025-04-02 diff --git a/noxfile.py b/noxfile.py index b6eb5ea..78a3a30 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,15 +35,19 @@ """For test suites where we track coverage (i.e. the fast tests and the full test suite), fail if test coverage falls below this percentage.""" +def is_mac(): + """Returns true if the current system is a mac.""" + return sys.platform == "darwin" + def is_arm_mac(): """Returns true if the current system is am arm-based mac.""" - return sys.platform == "darwin" and platform.processor() == "arm" + return is_mac() and platform.processor() == "arm" DEPENDENCY_MATRIX = [ DependencyConfiguration( id="3.9-oldest", python="3.9", packages={ - "pyspark[sql]": "==3.5.0", + "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "numpy": "==1.23.2", @@ -70,7 +74,7 @@ def is_arm_mac(): DependencyConfiguration( id="3.10-oldest", python="3.10", packages={ - "pyspark[sql]": "==3.5.0", + "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0", "sympy": "==1.8", "pandas": "==1.4.0", "numpy": "==1.23.2", @@ -94,7 +98,7 @@ def is_arm_mac(): DependencyConfiguration( id="3.11-oldest", python="3.11", packages={ - "pyspark[sql]": "==3.5.0", + "pyspark[sql]": "==3.4.0" if not is_mac() else "==3.5.0", "sympy": "==1.8", "pandas": "==1.5.0", "numpy": "==1.23.2", diff --git a/pyproject.toml b/pyproject.toml index c3d0562..d4e2c19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,10 @@ dependencies = [ "pandas >=1.5.0,<2 ; python_version == '3.11'", "pandas >=2.2.0,<3 ; python_version >= '3.12'", "pyarrow >=18.0.0,<19", - "pyspark[sql] >=3.5.0,<3.6", + "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11' and sys_platform != 'darwin'", + "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11' and sys_platform != 'darwin'", + "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12' and sys_platform != 'darwin'", + "pyspark[sql] >=3.5.0,<3.6 ; sys_platform == 'darwin'", "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", diff --git a/uv.lock b/uv.lock index f45c14d..0973e88 100644 --- a/uv.lock +++ b/uv.lock @@ -2,10 +2,14 @@ version = 1 revision = 2 requires-python = ">=3.9, <3.13" resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] [[package]] @@ -227,7 +231,8 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.10'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, @@ -242,9 +247,12 @@ name = "click" version = "8.2.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, @@ -280,7 +288,8 @@ name = "contourpy" version = "1.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.10'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version < '3.10'" }, @@ -340,9 +349,12 @@ name = "contourpy" version = "1.3.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version >= '3.10'" }, @@ -727,7 +739,8 @@ name = "kiwisolver" version = "1.4.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.10'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/85/4d/2255e1c76304cbd60b48cee302b66d1dde4468dc5b1160e4b7cb43778f2a/kiwisolver-1.4.7.tar.gz", hash = "sha256:9893ff81bd7107f7b685d3017cc6583daadb4fc26e4a888350df530e41980a60", size = 97286, upload-time = "2024-09-04T09:39:44.302Z" } wheels = [ @@ -814,9 +827,12 @@ name = "kiwisolver" version = "1.4.8" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/82/59/7c91426a8ac292e1cdd53a63b6d9439abd573c875c3f92c146767dd33faf/kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e", size = 97538, upload-time = "2024-12-24T18:30:51.519Z" } wheels = [ @@ -959,7 +975,8 @@ name = "matplotlib" version = "3.9.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.10'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "contourpy", version = "1.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -1010,9 +1027,12 @@ name = "matplotlib" version = "3.10.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", ] dependencies = [ { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -1260,9 +1280,12 @@ name = "pandas" version = "1.5.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version < '3.12'" }, @@ -1297,7 +1320,8 @@ name = "pandas" version = "2.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version >= '3.12'" }, @@ -1887,8 +1911,10 @@ name = "randomgen" version = "1.26.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version < '3.11'" }, @@ -1925,8 +1951,10 @@ name = "randomgen" version = "2.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version >= '3.11'" }, @@ -1992,7 +2020,8 @@ name = "scipy" version = "1.13.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.10'", + "python_full_version < '3.10' and sys_platform != 'darwin'", + "python_full_version < '3.10' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version < '3.10'" }, @@ -2030,9 +2059,12 @@ name = "scipy" version = "1.15.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.10.*' and sys_platform != 'darwin'", + "python_full_version == '3.10.*' and sys_platform == 'darwin'", ] dependencies = [ { name = "numpy", marker = "python_full_version >= '3.10'" }, @@ -2426,7 +2458,10 @@ requires-dist = [ { name = "pandas", marker = "python_full_version >= '3.12'", specifier = ">=2.2.0,<3" }, { name = "parameterized", specifier = ">=0.7.4,<0.8" }, { name = "pyarrow", specifier = ">=18.0.0,<19" }, - { name = "pyspark", extras = ["sql"], specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "sys_platform == 'darwin'", specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11' and sys_platform != 'darwin'", specifier = ">=3.3.1,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*' and sys_platform != 'darwin'", specifier = ">=3.4.0,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12' and sys_platform != 'darwin'", specifier = ">=3.5.0,<3.6" }, { name = "pytest", specifier = ">=7.3.1,<8" }, { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" },