opendp · Maegereg · Sep 25, 2025 · Sep 25, 2025 · Sep 25, 2025
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -15,6 +15,10 @@ For now, the repository remains available, and we encourage users to continue en
 Unreleased
 ----------
 
+Changed
+~~~~~~~
+- Dropped support for pyspark <3.5.0 on Macs after discovering that these configurations frequently crash. Older versions of the library may also be affected.
+
 .. _v0.18.2:
 
 0.18.2 - 2025-04-02

diff --git a/noxfile.py b/noxfile.py
@@ -35,15 +35,19 @@
 """For test suites where we track coverage (i.e. the fast tests and the full
 test suite), fail if test coverage falls below this percentage."""
 
+def is_mac():
+    """Returns true if the current system is a mac."""
+    return sys.platform == "darwin"
+
 def is_arm_mac():
     """Returns true if the current system is am arm-based mac."""
-    return sys.platform == "darwin" and platform.processor() == "arm"
+    return is_mac() and platform.processor() == "arm"
 
 DEPENDENCY_MATRIX = [
         DependencyConfiguration(
             id="3.9-oldest", python="3.9",
             packages={
-                "pyspark[sql]": "==3.5.0",  
+                "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0",  
                 "sympy": "==1.8",    
                 "pandas": "==1.4.0",
                 "numpy": "==1.23.2", 
@@ -70,7 +74,7 @@ def is_arm_mac():
         DependencyConfiguration(
             id="3.10-oldest", python="3.10", 
             packages={
-                "pyspark[sql]": "==3.5.0",  
+                "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0",
                 "sympy": "==1.8",    
                 "pandas": "==1.4.0",
                 "numpy": "==1.23.2", 
@@ -94,7 +98,7 @@ def is_arm_mac():
         DependencyConfiguration(
             id="3.11-oldest", python="3.11", 
             packages={
-                "pyspark[sql]": "==3.5.0",
+                "pyspark[sql]": "==3.4.0" if not is_mac() else "==3.5.0",  
                 "sympy": "==1.8",
                 "pandas": "==1.5.0",
                 "numpy": "==1.23.2",

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,7 +35,10 @@ dependencies = [
     "pandas >=1.5.0,<2 ; python_version == '3.11'",
     "pandas >=2.2.0,<3 ; python_version >= '3.12'",
     "pyarrow >=18.0.0,<19",
-    "pyspark[sql] >=3.5.0,<3.6",
+    "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11' and sys_platform != 'darwin'",
+    "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11' and sys_platform != 'darwin'",
+    "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12' and sys_platform != 'darwin'",
+    "pyspark[sql] >=3.5.0,<3.6 ; sys_platform == 'darwin'",
     "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'",
     "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'",
     "randomgen >=1.26.0,<=2 ; python_version >= '3.11'",

diff --git a/uv.lock b/uv.lock