From 59533029edec5079f96b82349c81c16c34c58506 Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Sun, 17 Aug 2025 10:55:57 -0700 Subject: [PATCH 1/7] adds PySpark 4.0.0 --- noxfile.py | 2 +- pyproject.toml | 2 +- uv.lock | 53 ++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index dae24b3..2432e71 100644 --- a/noxfile.py +++ b/noxfile.py @@ -95,7 +95,7 @@ # 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797 ( "3.12-newest", - "3.12", "==3.5.1", "==1.9", "==2.2.2", + "3.12", "==4.0.0", "==1.9", "==2.3.1", "==1.26.4", "==1.14.1", "==1.26.1", "==16.1.0", ), # fmt: on diff --git a/pyproject.toml b/pyproject.toml index 7ccf96f..bc125b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "pyarrow >=18.0.0,<19", "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11'", "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11'", - "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12'", + "pyspark[sql] >=3.5.0,<5 ; python_version >= '3.12'", "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", diff --git a/uv.lock b/uv.lock index 8a42394..36d93ab 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9, <3.13" resolution-markers = [ "python_full_version >= '3.12'", @@ -1509,11 +1509,28 @@ wheels = [ name = "py4j" version = "0.10.9.7" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version < '3.10'", +] sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, ] +[[package]] +name = "py4j" +version = "0.10.9.9" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, +] + [[package]] name = "pyarrow" version = "18.1.0" @@ -1761,17 +1778,40 @@ wheels = [ name = "pyspark" version = "3.5.6" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version < '3.10'", +] dependencies = [ - { name = "py4j" }, + { name = "py4j", version = "0.10.9.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" } [package.optional-dependencies] sql = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version < '3.12'" }, { name = "pandas", version = "1.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "pyarrow", marker = "python_full_version < '3.12'" }, +] + +[[package]] +name = "pyspark" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", +] +dependencies = [ + { name = "py4j", version = "0.10.9.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/0e/5b38d51f1b1c2618cccfbf35093268665af9a3bdb493e5a3ecd991def633/pyspark-4.0.0.tar.gz", hash = "sha256:38db1b4f6095a080d7605e578d775528990e66dc326311d93e94a71cfc24e5a5", size = 434132212, upload-time = "2025-05-23T03:29:33.916Z" } + +[package.optional-dependencies] +sql = [ + { name = "numpy", marker = "python_full_version >= '3.12'" }, { name = "pandas", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pyarrow" }, + { name = "pyarrow", marker = "python_full_version >= '3.12'" }, ] [[package]] @@ -2357,7 +2397,8 @@ dependencies = [ { name = "pandas", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "parameterized" }, { name = "pyarrow" }, - { name = "pyspark", extra = ["sql"] }, + { name = "pyspark", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, extra = ["sql"], marker = "python_full_version < '3.12'" }, + { name = "pyspark", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, extra = ["sql"], marker = "python_full_version >= '3.12'" }, { name = "pytest" }, { name = "randomgen", version = "1.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "randomgen", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -2428,7 +2469,7 @@ requires-dist = [ { name = "pyarrow", specifier = ">=18.0.0,<19" }, { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11'", specifier = ">=3.3.1,<3.6" }, { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*'", specifier = ">=3.4.0,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12'", specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12'", specifier = ">=3.5.0,<5" }, { name = "pytest", specifier = ">=7.3.1,<8" }, { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" }, From 1dedd7c748aeb79d6334b80d05c23eaf95a63af0 Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Thu, 21 Aug 2025 20:01:33 -0400 Subject: [PATCH 2/7] use Python 3.12 to test PySpark 4.0.0 --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index a02597f..fdcfcfd 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9 \ No newline at end of file +3.12 \ No newline at end of file From 11725762e7f32d441fd7c01abf890f557ffba06c Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Fri, 17 Oct 2025 14:04:12 -0400 Subject: [PATCH 3/7] current work save --- noxfile.py | 50 ++++++++++++++-------------- pyproject.toml | 22 ++++++++----- src/tmlt/core/random/rng.py | 1 + uv.lock | 66 ++++++++++++++++++++++++++++--------- 4 files changed, 90 insertions(+), 49 deletions(-) diff --git a/noxfile.py b/noxfile.py index e8a9360..732e68c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -47,27 +47,27 @@ def is_arm_mac(): DEPENDENCY_MATRIX = [ - DependencyConfiguration( - id="3.10-oldest", - python="3.10", - packages={ - "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0", - "sympy": "==1.8", - "pandas": "==1.4.0", - "numpy": "==1.23.2", - "scipy": "==1.8.0", - "randomgen": "==1.23.0", - "pyarrow": "==14.0.1", - }, - ), + #DependencyConfiguration( + # id="3.10-oldest", + # python="3.10", + # packages={ + # "pyspark[sql]": "==3.3.1" if not is_mac() else "==3.5.0", + # "sympy": "==1.8", + # "pandas": "==1.4.0", + # "numpy": "==1.23.2", + # "scipy": "==1.8.0", + # "randomgen": "==1.23.0", + # "pyarrow": "==14.0.1", + # }, + #), DependencyConfiguration( id="3.10-newest", python="3.10", packages={ - "pyspark[sql]": "==3.5.6", - "sympy": "==1.9", - "pandas": "==1.5.3", - "numpy": "==1.26.4", + "pyspark[sql]": "==4.0.1", + "sympy": "==1.12", + "pandas": "==2.3.3", + "numpy": "==2.2.6", "scipy": "==1.14.1", "randomgen": "==1.26.0", "pyarrow": "==16.1.0", @@ -90,10 +90,10 @@ def is_arm_mac(): id="3.11-newest", python="3.11", packages={ - "pyspark[sql]": "==3.5.6", - "sympy": "==1.9", - "pandas": "==1.5.3", - "numpy": "==1.26.4", + "pyspark[sql]": "==4.0.1", + "sympy": "==1.12", + "pandas": "==2.3.3", + "numpy": "==2.2.6", "scipy": "==1.14.1", "randomgen": "==1.26.1", "pyarrow": "==16.1.0", @@ -117,10 +117,10 @@ def is_arm_mac(): id="3.12-newest", python="3.12", packages={ - "pyspark[sql]": "==3.5.6", - "sympy": "==1.9", - "pandas": "==2.2.2", - "numpy": "==1.26.4", + "pyspark[sql]": "==4.0.1", + "sympy": "==1.12", + "pandas": "==2.3.3", + "numpy": "==2.2.6", "scipy": "==1.14.1", "randomgen": "==1.26.1", "pyarrow": "==16.1.0", diff --git a/pyproject.toml b/pyproject.toml index 9803610..ec08c40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,16 +28,16 @@ keywords = [ requires-python = ">=3.10,<3.13" dependencies = [ - "numpy >=1.23.2,<2 ; python_version < '3.12'", - "numpy >=1.26.0,<2 ; python_version >= '3.12'", - "pandas >=1.4.0,<2 ; python_version < '3.11'", - "pandas >=1.5.0,<2 ; python_version == '3.11'", + "numpy >=1.23.2,<=2.2.6 ; python_version < '3.12'", + "numpy >=1.26.0,<=2.2.6 ; python_version >= '3.12'", + "pandas >=1.4.0,<3 ; python_version < '3.11'", + "pandas >=1.5.0,<3 ; python_version == '3.11'", "pandas >=2.2.0,<3 ; python_version >= '3.12'", "pyarrow >=18.0.0,<19", - "pyspark[sql] >=3.3.1,<3.6 ; python_version < '3.11' and sys_platform != 'darwin'", - "pyspark[sql] >=3.4.0,<3.6 ; python_version == '3.11' and sys_platform != 'darwin'", - "pyspark[sql] >=3.5.0,<3.6 ; python_version >= '3.12' and sys_platform != 'darwin'", - "pyspark[sql] >=3.5.0,<3.6 ; sys_platform == 'darwin'", + "pyspark[sql] >=3.3.1,<5 ; python_version < '3.11' and sys_platform != 'darwin'", + "pyspark[sql] >=3.4.0,<5 ; python_version == '3.11' and sys_platform != 'darwin'", + "pyspark[sql] >=3.5.0,<5; python_version >= '3.12' and sys_platform != 'darwin'", + "pyspark[sql] >=3.5.0,<5 ; sys_platform == 'darwin'", "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", @@ -99,6 +99,12 @@ scripting = [ "nox >=2024.03.02", "tmlt.nox_utils" ] +dev = [ + "ruff>=0.5.0", # Added to support numpy migration. +] + +[tool.ruff.lint] +select = ["NPY201"] [tool.uv.sources] # Switch which of these is commented out to test local changes to nox-utils: diff --git a/src/tmlt/core/random/rng.py b/src/tmlt/core/random/rng.py index 1d07264..ff7d2db 100644 --- a/src/tmlt/core/random/rng.py +++ b/src/tmlt/core/random/rng.py @@ -9,6 +9,7 @@ import numpy as np from randomgen.rdrand import RDRAND # pylint: disable=no-name-in-module from randomgen.wrapper import UserBitGenerator # pylint: disable=no-name-in-module +"""Can this be switched to something else to avoid import errors?""" try: _core_privacy_prng = np.random.Generator(RDRAND()) diff --git a/uv.lock b/uv.lock index 1363524..687b601 100644 --- a/uv.lock +++ b/uv.lock @@ -1186,9 +1186,10 @@ name = "py4j" version = "0.10.9.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } wheels = [ @@ -1200,7 +1201,8 @@ name = "py4j" version = "0.10.9.9" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } wheels = [ @@ -1425,9 +1427,10 @@ name = "pyspark" version = "3.5.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ { name = "py4j", version = "0.10.9.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -1446,7 +1449,8 @@ name = "pyspark" version = "4.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.12' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and sys_platform == 'darwin'", ] dependencies = [ { name = "py4j", version = "0.10.9.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -1656,6 +1660,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" }, ] +[[package]] +name = "ruff" +version = "0.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/58/6ca66896635352812de66f71cdf9ff86b3a4f79071ca5730088c0cd0fc8d/ruff-0.14.1.tar.gz", hash = "sha256:1dd86253060c4772867c61791588627320abcb6ed1577a90ef432ee319729b69", size = 5513429, upload-time = "2025-10-16T18:05:41.766Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/39/9cc5ab181478d7a18adc1c1e051a84ee02bec94eb9bdfd35643d7c74ca31/ruff-0.14.1-py3-none-linux_armv6l.whl", hash = "sha256:083bfc1f30f4a391ae09c6f4f99d83074416b471775b59288956f5bc18e82f8b", size = 12445415, upload-time = "2025-10-16T18:04:48.227Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2e/1226961855ccd697255988f5a2474890ac7c5863b080b15bd038df820818/ruff-0.14.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f6fa757cd717f791009f7669fefb09121cc5f7d9bd0ef211371fad68c2b8b224", size = 12784267, upload-time = "2025-10-16T18:04:52.515Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ea/fd9e95863124ed159cd0667ec98449ae461de94acda7101f1acb6066da00/ruff-0.14.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d6191903d39ac156921398e9c86b7354d15e3c93772e7dbf26c9fcae59ceccd5", size = 11781872, upload-time = "2025-10-16T18:04:55.396Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5a/e890f7338ff537dba4589a5e02c51baa63020acfb7c8cbbaea4831562c96/ruff-0.14.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed04f0e04f7a4587244e5c9d7df50e6b5bf2705d75059f409a6421c593a35896", size = 12226558, upload-time = "2025-10-16T18:04:58.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7a/8ab5c3377f5bf31e167b73651841217542bcc7aa1c19e83030835cc25204/ruff-0.14.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5c9e6cf6cd4acae0febbce29497accd3632fe2025c0c583c8b87e8dbdeae5f61", size = 12187898, upload-time = "2025-10-16T18:05:01.455Z" }, + { url = "https://files.pythonhosted.org/packages/48/8d/ba7c33aa55406955fc124e62c8259791c3d42e3075a71710fdff9375134f/ruff-0.14.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fa2458527794ecdfbe45f654e42c61f2503a230545a91af839653a0a93dbc6", size = 12939168, upload-time = "2025-10-16T18:05:04.397Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c2/70783f612b50f66d083380e68cbd1696739d88e9b4f6164230375532c637/ruff-0.14.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:39f1c392244e338b21d42ab29b8a6392a722c5090032eb49bb4d6defcdb34345", size = 14386942, upload-time = "2025-10-16T18:05:07.102Z" }, + { url = "https://files.pythonhosted.org/packages/48/44/cd7abb9c776b66d332119d67f96acf15830d120f5b884598a36d9d3f4d83/ruff-0.14.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7382fa12a26cce1f95070ce450946bec357727aaa428983036362579eadcc5cf", size = 13990622, upload-time = "2025-10-16T18:05:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/4259b696db12ac152fe472764b4f78bbdd9b477afd9bc3a6d53c01300b37/ruff-0.14.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd0bf2be3ae8521e1093a487c4aa3b455882f139787770698530d28ed3fbb37c", size = 13431143, upload-time = "2025-10-16T18:05:13.46Z" }, + { url = "https://files.pythonhosted.org/packages/e0/35/266a80d0eb97bd224b3265b9437bd89dde0dcf4faf299db1212e81824e7e/ruff-0.14.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cabcaa9ccf8089fb4fdb78d17cc0e28241520f50f4c2e88cb6261ed083d85151", size = 13132844, upload-time = "2025-10-16T18:05:16.1Z" }, + { url = "https://files.pythonhosted.org/packages/65/6e/d31ce218acc11a8d91ef208e002a31acf315061a85132f94f3df7a252b18/ruff-0.14.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:747d583400f6125ec11a4c14d1c8474bf75d8b419ad22a111a537ec1a952d192", size = 13401241, upload-time = "2025-10-16T18:05:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b5/dbc4221bf0b03774b3b2f0d47f39e848d30664157c15b965a14d890637d2/ruff-0.14.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5a6e74c0efd78515a1d13acbfe6c90f0f5bd822aa56b4a6d43a9ffb2ae6e56cd", size = 12132476, upload-time = "2025-10-16T18:05:22.163Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/ac99194e790ccd092d6a8b5f341f34b6e597d698e3077c032c502d75ea84/ruff-0.14.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0ea6a864d2fb41a4b6d5b456ed164302a0d96f4daac630aeba829abfb059d020", size = 12139749, upload-time = "2025-10-16T18:05:25.162Z" }, + { url = "https://files.pythonhosted.org/packages/47/26/7df917462c3bb5004e6fdfcc505a49e90bcd8a34c54a051953118c00b53a/ruff-0.14.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:0826b8764f94229604fa255918d1cc45e583e38c21c203248b0bfc9a0e930be5", size = 12544758, upload-time = "2025-10-16T18:05:28.018Z" }, + { url = "https://files.pythonhosted.org/packages/64/d0/81e7f0648e9764ad9b51dd4be5e5dac3fcfff9602428ccbae288a39c2c22/ruff-0.14.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cbc52160465913a1a3f424c81c62ac8096b6a491468e7d872cb9444a860bc33d", size = 13221811, upload-time = "2025-10-16T18:05:30.707Z" }, + { url = "https://files.pythonhosted.org/packages/c3/07/3c45562c67933cc35f6d5df4ca77dabbcd88fddaca0d6b8371693d29fd56/ruff-0.14.1-py3-none-win32.whl", hash = "sha256:e037ea374aaaff4103240ae79168c0945ae3d5ae8db190603de3b4012bd1def6", size = 12319467, upload-time = "2025-10-16T18:05:33.261Z" }, + { url = "https://files.pythonhosted.org/packages/02/88/0ee4ca507d4aa05f67e292d2e5eb0b3e358fbcfe527554a2eda9ac422d6b/ruff-0.14.1-py3-none-win_amd64.whl", hash = "sha256:59d599cdff9c7f925a017f6f2c256c908b094e55967f93f2821b1439928746a1", size = 13401123, upload-time = "2025-10-16T18:05:35.984Z" }, + { url = "https://files.pythonhosted.org/packages/b8/81/4b6387be7014858d924b843530e1b2a8e531846807516e9bea2ee0936bf7/ruff-0.14.1-py3-none-win_arm64.whl", hash = "sha256:e3b443c4c9f16ae850906b8d0a707b2a4c16f8d2f0a7fe65c475c5886665ce44", size = 12436636, upload-time = "2025-10-16T18:05:38.995Z" }, +] + [[package]] name = "scipy" version = "1.15.3" @@ -1994,6 +2024,9 @@ build = [ ci-tools = [ { name = "requests" }, ] +dev = [ + { name = "ruff" }, +] docs = [ { name = "pydata-sphinx-theme" }, { name = "sphinx" }, @@ -2034,17 +2067,17 @@ test = [ [package.metadata] requires-dist = [ - { name = "numpy", marker = "python_full_version < '3.12'", specifier = ">=1.23.2,<2" }, - { name = "numpy", marker = "python_full_version >= '3.12'", specifier = ">=1.26.0,<2" }, - { name = "pandas", marker = "python_full_version < '3.11'", specifier = ">=1.4.0,<2" }, - { name = "pandas", marker = "python_full_version == '3.11.*'", specifier = ">=1.5.0,<2" }, + { name = "numpy", marker = "python_full_version < '3.12'", specifier = ">=1.23.2,<=2.2.6" }, + { name = "numpy", marker = "python_full_version >= '3.12'", specifier = ">=1.26.0,<=2.2.6" }, + { name = "pandas", marker = "python_full_version < '3.11'", specifier = ">=1.4.0,<3" }, + { name = "pandas", marker = "python_full_version == '3.11.*'", specifier = ">=1.5.0,<3" }, { name = "pandas", marker = "python_full_version >= '3.12'", specifier = ">=2.2.0,<3" }, { name = "parameterized", specifier = ">=0.7.4,<0.8" }, { name = "pyarrow", specifier = ">=18.0.0,<19" }, - { name = "pyspark", extras = ["sql"], marker = "sys_platform == 'darwin'", specifier = ">=3.5.0,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11' and sys_platform != 'darwin'", specifier = ">=3.3.1,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*' and sys_platform != 'darwin'", specifier = ">=3.4.0,<3.6" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12' and sys_platform != 'darwin'", specifier = ">=3.5.0,<3.6" }, + { name = "pyspark", extras = ["sql"], marker = "sys_platform == 'darwin'", specifier = ">=3.5.0,<5" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11' and sys_platform != 'darwin'", specifier = ">=3.3.1,<5" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*' and sys_platform != 'darwin'", specifier = ">=3.4.0,<5" }, + { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12' and sys_platform != 'darwin'", specifier = ">=3.5.0,<5" }, { name = "pytest", specifier = ">=7.3.1,<8" }, { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" }, @@ -2063,6 +2096,7 @@ audit = [{ name = "pip-audit", specifier = ">=2.9.0,<3" }] black = [{ name = "black", specifier = ">=23.3,<24" }] build = [{ name = "cibuildwheel", specifier = ">=2,<3" }] ci-tools = [{ name = "requests", specifier = ">=2.31.0,<3" }] +dev = [{ name = "ruff", specifier = ">=0.5.0" }] docs = [ { name = "pydata-sphinx-theme", specifier = ">=0.14.4,<15" }, { name = "sphinx", specifier = ">=7.2.0,<8" }, From eebd3f26a84a04648d947174acefbe1ed0ad8f49 Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Sun, 2 Nov 2025 16:14:03 -0500 Subject: [PATCH 4/7] Can collect Python 3.10 newest dependency tests --- noxfile.py | 6 +- pyproject.toml | 16 +-- test/unit/domains/test_numpy_domains.py | 2 +- test/unit/domains/test_pandas_domains.py | 4 +- test/unit/transformations/test_dictionary.py | 6 +- uv.lock | 125 +++---------------- 6 files changed, 31 insertions(+), 128 deletions(-) diff --git a/noxfile.py b/noxfile.py index 732e68c..03f7f9e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -69,8 +69,10 @@ def is_arm_mac(): "pandas": "==2.3.3", "numpy": "==2.2.6", "scipy": "==1.14.1", - "randomgen": "==1.26.0", + "randomgen": "==2.3.0", "pyarrow": "==16.1.0", + "py4j": "==0.10.9.9", + "tzdata": "==2025.2", }, ), DependencyConfiguration( @@ -80,7 +82,7 @@ def is_arm_mac(): "pyspark[sql]": "==3.4.0" if not is_mac() else "==3.5.0", "sympy": "==1.8", "pandas": "==1.5.0", - "numpy": "==1.23.2", + "numpy": "==1.26.0", "scipy": "==1.9.2", "randomgen": "==1.26.0", "pyarrow": "==14.0.1", diff --git a/pyproject.toml b/pyproject.toml index ec08c40..23fb08b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,19 +28,11 @@ keywords = [ requires-python = ">=3.10,<3.13" dependencies = [ - "numpy >=1.23.2,<=2.2.6 ; python_version < '3.12'", - "numpy >=1.26.0,<=2.2.6 ; python_version >= '3.12'", - "pandas >=1.4.0,<3 ; python_version < '3.11'", - "pandas >=1.5.0,<3 ; python_version == '3.11'", - "pandas >=2.2.0,<3 ; python_version >= '3.12'", + "numpy >=1.26.0,<=2.2.6", + "pandas >=1.5.0,<3", "pyarrow >=18.0.0,<19", - "pyspark[sql] >=3.3.1,<5 ; python_version < '3.11' and sys_platform != 'darwin'", - "pyspark[sql] >=3.4.0,<5 ; python_version == '3.11' and sys_platform != 'darwin'", - "pyspark[sql] >=3.5.0,<5; python_version >= '3.12' and sys_platform != 'darwin'", - "pyspark[sql] >=3.5.0,<5 ; sys_platform == 'darwin'", - "randomgen >=1.20.0,<=1.26.0 ; python_version < '3.10'", - "randomgen >=1.23.0,<=1.26.0 ; python_version == '3.10'", - "randomgen >=1.26.0,<=2 ; python_version >= '3.11'", + "pyspark[sql] >=3.4.0, <5", + "randomgen >=1.26.0,<=2.3.0", "scipy >=1.6.0,<2.0.0 ; python_version < '3.10'", "scipy >=1.8.0,<2.0.0 ; python_version == '3.10'", "scipy >=1.9.2,<2.0.0 ; python_version == '3.11'", diff --git a/test/unit/domains/test_numpy_domains.py b/test/unit/domains/test_numpy_domains.py index 85d454f..dc8b7b4 100644 --- a/test/unit/domains/test_numpy_domains.py +++ b/test/unit/domains/test_numpy_domains.py @@ -802,7 +802,7 @@ def test_validate( "dtype, expected, expectation", [ # pylint: disable=line-too-long - (np.dtype(np.object0), NumpyStringDomain(), does_not_raise()), # type: ignore[attr-defined] + (np.dtype(np.object_), NumpyStringDomain(), does_not_raise()), # type: ignore[attr-defined] # pylint: enable=line-too-long ( np.dtype([("f1", np.int64)]), diff --git a/test/unit/domains/test_pandas_domains.py b/test/unit/domains/test_pandas_domains.py index 851b91f..25d0d95 100644 --- a/test/unit/domains/test_pandas_domains.py +++ b/test/unit/domains/test_pandas_domains.py @@ -266,7 +266,7 @@ def test_validate( PandasSeriesDomain(NumpyIntegerDomain(size=32)), does_not_raise(), ) - for dtype in [np.int8, np.int16, np.int32, np.bool8] # type: ignore[attr-defined] + for dtype in [np.int8, np.int16, np.int32, np.bool_] # type: ignore[attr-defined] ] # pylint: enable=line-too-long + [ @@ -281,7 +281,7 @@ def test_validate( does_not_raise(), ), ( - np.dtype(np.object0), # type: ignore[attr-defined] + np.dtype(np.object_), # type: ignore[attr-defined] PandasSeriesDomain(NumpyStringDomain()), does_not_raise(), ), diff --git a/test/unit/transformations/test_dictionary.py b/test/unit/transformations/test_dictionary.py index 591e401..37d062b 100644 --- a/test/unit/transformations/test_dictionary.py +++ b/test/unit/transformations/test_dictionary.py @@ -458,10 +458,10 @@ def test_properties( [ ( ["A", ("B", "B"), "C"], - {"A": np.int_(20), ("B", "B"): "XYZ", "C": np.float_(10.11)}, + {"A": np.int_(20), ("B", "B"): "XYZ", "C": np.float64(10.11)}, ), (["A"], {"A": np.int_(20)}), - ([("B", "B"), "C"], {("B", "B"): "XYZ", "C": np.float_(10.11)}), + ([("B", "B"), "C"], {("B", "B"): "XYZ", "C": np.float64(10.11)}), ] ) def test_correctness( @@ -488,7 +488,7 @@ def test_correctness( input_domain=input_domain, input_metric=input_metric, keys=keys ) actual = transformation( - {"A": np.int_(20), ("B", "B"): "XYZ", "C": np.float_(10.11)} + {"A": np.int_(20), ("B", "B"): "XYZ", "C": np.float64(10.11)} ) self.assertEqual(actual, expected) diff --git a/uv.lock b/uv.lock index 687b601..2f6ef0c 100644 --- a/uv.lock +++ b/uv.lock @@ -954,50 +954,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "pandas" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*' and sys_platform != 'darwin'", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", - "python_full_version < '3.11' and sys_platform == 'darwin'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version < '3.12'" }, - { name = "python-dateutil", marker = "python_full_version < '3.12'" }, - { name = "pytz", marker = "python_full_version < '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/ee/146cab1ff6d575b54ace8a6a5994048380dc94879b0125b25e62edcb9e52/pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1", size = 5203060, upload-time = "2023-01-19T08:31:39.615Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/cd/34f6b0780301be81be804d7aa71d571457369e6131e2b330af2b0fed1aad/pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406", size = 18619230, upload-time = "2023-01-19T08:29:07.301Z" }, - { url = "https://files.pythonhosted.org/packages/5f/34/b7858bb7d6d6bf4d9df1dde777a11fcf3ff370e1d1b3956e3d0fcca8322c/pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572", size = 11982991, upload-time = "2023-01-19T08:29:15.383Z" }, - { url = "https://files.pythonhosted.org/packages/b8/6c/005bd604994f7cbede4d7bf030614ef49a2213f76bc3d738ecf5b0dcc810/pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996", size = 10927131, upload-time = "2023-01-19T08:29:20.342Z" }, - { url = "https://files.pythonhosted.org/packages/27/c7/35b81ce5f680f2dac55eac14d103245cd8cf656ae4a2ff3be2e69fd1d330/pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354", size = 11368188, upload-time = "2023-01-19T08:29:25.807Z" }, - { url = "https://files.pythonhosted.org/packages/49/e2/79e46612dc25ebc7603dc11c560baa7266c90f9e48537ecf1a02a0dd6bff/pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23", size = 12062104, upload-time = "2023-01-19T08:29:30.695Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cd/f27c2992cbe05a3e39937f73a4be635a9ec149ec3ca4467d8cf039718994/pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328", size = 10362473, upload-time = "2023-01-19T08:29:37.506Z" }, - { url = "https://files.pythonhosted.org/packages/e2/24/a26af514113fd5eca2d8fe41ba4f22f70dfe6afefde4a6beb6a203570935/pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc", size = 18387750, upload-time = "2023-01-19T08:29:43.119Z" }, - { url = "https://files.pythonhosted.org/packages/53/c9/d2f910dace7ef849b626980d0fd033b9cded36568949c8d560c9630ad2e0/pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d", size = 11868668, upload-time = "2023-01-19T08:29:48.733Z" }, - { url = "https://files.pythonhosted.org/packages/b0/be/1843b9aff84b98899663e7cad9f45513dfdd11d69cb5bd85c648aaf6a8d4/pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc", size = 10814036, upload-time = "2023-01-19T08:29:54.886Z" }, - { url = "https://files.pythonhosted.org/packages/63/8d/c2bd356b9d4baf1c5cf8d7e251fb4540e87083072c905430da48c2bb31eb/pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae", size = 11374218, upload-time = "2023-01-19T08:30:00.5Z" }, - { url = "https://files.pythonhosted.org/packages/56/73/3351beeb807dca69fcc3c4966bcccc51552bd01549a9b13c04ab00a43f21/pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6", size = 12017319, upload-time = "2023-01-19T08:30:06.097Z" }, - { url = "https://files.pythonhosted.org/packages/da/6d/1235da14daddaa6e47f74ba0c255358f0ce7a6ee05da8bf8eb49161aa6b5/pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003", size = 10303385, upload-time = "2023-01-19T08:30:11.148Z" }, -] - [[package]] name = "pandas" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12' and sys_platform != 'darwin'", - "python_full_version >= '3.12' and sys_platform == 'darwin'", -] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.12'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.12'" }, - { name = "pytz", marker = "python_full_version >= '3.12'" }, - { name = "tzdata", marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/51/48f713c4c728d7c55ef7444ba5ea027c26998d96d1a40953b346438602fc/pandas-2.3.0.tar.gz", hash = "sha256:34600ab34ebf1131a7613a260a61dbe8b62c188ec0ea4c296da7c9a06b004133", size = 4484490, upload-time = "2025-06-05T03:27:54.133Z" } wheels = [ @@ -1181,29 +1146,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/0e/8601d2331dea0825f3be769688b48a95f387a83c918cca6a8c9cee4b9eb7/py_serializable-2.0.0-py3-none-any.whl", hash = "sha256:1721e4c0368adeec965c183168da4b912024702f19e15e13f8577098b9a4f8fe", size = 22824, upload-time = "2025-02-09T13:41:54.236Z" }, ] -[[package]] -name = "py4j" -version = "0.10.9.7" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*' and sys_platform != 'darwin'", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", - "python_full_version < '3.11' and sys_platform == 'darwin'", -] -sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/b34255180c72c36ff7097f7c2cdca02abcbd89f5eebf7c7c41262a9a0637/py4j-0.10.9.7.tar.gz", hash = "sha256:0b6e5315bb3ada5cf62ac651d107bb2ebc02def3dee9d9548e3baac644ea8dbb", size = 1508234, upload-time = "2022-08-12T22:49:09.792Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/30/a58b32568f1623aaad7db22aa9eafc4c6c194b429ff35bdc55ca2726da47/py4j-0.10.9.7-py2.py3-none-any.whl", hash = "sha256:85defdfd2b2376eb3abf5ca6474b51ab7e0de341c75a02f46dc9b5976f5a5c1b", size = 200481, upload-time = "2022-08-12T22:49:07.05Z" }, -] - [[package]] name = "py4j" version = "0.10.9.9" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12' and sys_platform != 'darwin'", - "python_full_version >= '3.12' and sys_platform == 'darwin'", -] sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, @@ -1424,44 +1370,18 @@ wheels = [ [[package]] name = "pyspark" -version = "3.5.6" +version = "4.0.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.11.*' and sys_platform != 'darwin'", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", - "python_full_version < '3.11' and sys_platform == 'darwin'", -] dependencies = [ - { name = "py4j", version = "0.10.9.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/40/1414582f16c1d7b051c668c2e19c62d21a18bd181d944cb24f5ddbb2423f/pyspark-4.0.1.tar.gz", hash = "sha256:9d1f22d994f60369228397e3479003ffe2dd736ba79165003246ff7bd48e2c73", size = 434204896, upload-time = "2025-09-06T07:15:57.091Z" } [package.optional-dependencies] sql = [ - { name = "numpy", marker = "python_full_version < '3.12'" }, - { name = "pandas", version = "1.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "pyarrow", marker = "python_full_version < '3.12'" }, -] - -[[package]] -name = "pyspark" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12' and sys_platform != 'darwin'", - "python_full_version >= '3.12' and sys_platform == 'darwin'", -] -dependencies = [ - { name = "py4j", version = "0.10.9.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/0e/5b38d51f1b1c2618cccfbf35093268665af9a3bdb493e5a3ecd991def633/pyspark-4.0.0.tar.gz", hash = "sha256:38db1b4f6095a080d7605e578d775528990e66dc326311d93e94a71cfc24e5a5", size = 434132212, upload-time = "2025-05-23T03:29:33.916Z" } - -[package.optional-dependencies] -sql = [ - { name = "numpy", marker = "python_full_version >= '3.12'" }, - { name = "pandas", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pyarrow", marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pyarrow" }, ] [[package]] @@ -1731,8 +1651,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "matplotlib" }, { name = "numpy" }, - { name = "pandas", version = "1.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "pandas", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pandas" }, ] sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } wheels = [ @@ -1996,12 +1915,10 @@ name = "tmlt-core" source = { editable = "." } dependencies = [ { name = "numpy" }, - { name = "pandas", version = "1.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "pandas", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pandas" }, { name = "parameterized" }, { name = "pyarrow" }, - { name = "pyspark", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, extra = ["sql"], marker = "python_full_version < '3.12'" }, - { name = "pyspark", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, extra = ["sql"], marker = "python_full_version >= '3.12'" }, + { name = "pyspark", extra = ["sql"] }, { name = "pytest" }, { name = "randomgen", version = "1.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "randomgen", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -2067,21 +1984,13 @@ test = [ [package.metadata] requires-dist = [ - { name = "numpy", marker = "python_full_version < '3.12'", specifier = ">=1.23.2,<=2.2.6" }, - { name = "numpy", marker = "python_full_version >= '3.12'", specifier = ">=1.26.0,<=2.2.6" }, - { name = "pandas", marker = "python_full_version < '3.11'", specifier = ">=1.4.0,<3" }, - { name = "pandas", marker = "python_full_version == '3.11.*'", specifier = ">=1.5.0,<3" }, - { name = "pandas", marker = "python_full_version >= '3.12'", specifier = ">=2.2.0,<3" }, + { name = "numpy", specifier = ">=1.26.0,<=2.2.6" }, + { name = "pandas", specifier = ">=1.5.0,<3" }, { name = "parameterized", specifier = ">=0.7.4,<0.8" }, { name = "pyarrow", specifier = ">=18.0.0,<19" }, - { name = "pyspark", extras = ["sql"], marker = "sys_platform == 'darwin'", specifier = ">=3.5.0,<5" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version < '3.11' and sys_platform != 'darwin'", specifier = ">=3.3.1,<5" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version == '3.11.*' and sys_platform != 'darwin'", specifier = ">=3.4.0,<5" }, - { name = "pyspark", extras = ["sql"], marker = "python_full_version >= '3.12' and sys_platform != 'darwin'", specifier = ">=3.5.0,<5" }, + { name = "pyspark", extras = ["sql"], specifier = ">=3.4.0,<5" }, { name = "pytest", specifier = ">=7.3.1,<8" }, - { name = "randomgen", marker = "python_full_version < '3.10'", specifier = ">=1.20.0,<=1.26.0" }, - { name = "randomgen", marker = "python_full_version == '3.10.*'", specifier = ">=1.23.0,<=1.26.0" }, - { name = "randomgen", marker = "python_full_version >= '3.11'", specifier = ">=1.26.0,<=2" }, + { name = "randomgen", specifier = ">=1.26.0,<=2.3.0" }, { name = "scipy", marker = "python_full_version < '3.10'", specifier = ">=1.6.0,<2.0.0" }, { name = "scipy", marker = "python_full_version == '3.10.*'", specifier = ">=1.8.0,<2.0.0" }, { name = "scipy", marker = "python_full_version == '3.11.*'", specifier = ">=1.9.2,<2.0.0" }, From 98c956802e9f8c568fccaf20cd846725f2243bf6 Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Sun, 2 Nov 2025 17:10:19 -0500 Subject: [PATCH 5/7] linting --- benchmark/bounds.py | 2 +- benchmark/count_sum.py | 2 +- benchmark/noise_mechanism.py | 2 +- benchmark/public_join.py | 2 +- benchmark/quantile.py | 2 +- benchmark/sparkflatmap.py | 2 +- benchmark/sparkmap.py | 2 +- src/tmlt/core/random/rng.py | 1 - .../core/transformations/spark_transformations/nan.py | 3 +++ test/unit/domains/test_numpy_domains.py | 4 +--- test/unit/domains/test_pandas_domains.py | 10 +++------- test/unit/domains/test_spark_domains.py | 2 +- 12 files changed, 15 insertions(+), 19 deletions(-) diff --git a/benchmark/bounds.py b/benchmark/bounds.py index 8d45c02..51e8b05 100644 --- a/benchmark/bounds.py +++ b/benchmark/bounds.py @@ -11,7 +11,7 @@ import pandas as pd from benchmarking_utils import Timer, write_as_html from pyspark.sql import SparkSession, functions as sf -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.types import LongType, StructField, StructType from tmlt.core.domains.spark_domains import ( diff --git a/benchmark/count_sum.py b/benchmark/count_sum.py index 1fe632d..1b284c6 100644 --- a/benchmark/count_sum.py +++ b/benchmark/count_sum.py @@ -11,7 +11,7 @@ import pandas as pd from pyspark.sql import SparkSession from pyspark.sql import functions as sf -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.types import LongType, StructField, StructType from benchmarking_utils import Timer, write_as_html diff --git a/benchmark/noise_mechanism.py b/benchmark/noise_mechanism.py index d75b4b5..08edeee 100644 --- a/benchmark/noise_mechanism.py +++ b/benchmark/noise_mechanism.py @@ -7,7 +7,7 @@ import pandas as pd from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.types import IntegerType, StructField, StructType from benchmarking_utils import write_as_html diff --git a/benchmark/public_join.py b/benchmark/public_join.py index fc4813a..3ea7475 100644 --- a/benchmark/public_join.py +++ b/benchmark/public_join.py @@ -9,7 +9,7 @@ import pandas as pd from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.functions import lit from pyspark.sql.types import LongType from benchmarking_utils import write_as_html diff --git a/benchmark/quantile.py b/benchmark/quantile.py index d2bc092..9075067 100644 --- a/benchmark/quantile.py +++ b/benchmark/quantile.py @@ -8,7 +8,7 @@ import pandas as pd from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from tmlt.core.domains.spark_domains import ( SparkDataFrameDomain, diff --git a/benchmark/sparkflatmap.py b/benchmark/sparkflatmap.py index 873f114..78cacee 100644 --- a/benchmark/sparkflatmap.py +++ b/benchmark/sparkflatmap.py @@ -9,7 +9,7 @@ import pandas as pd from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.types import IntegerType, StringType, StructField, StructType from tmlt.core.domains.collections import ListDomain diff --git a/benchmark/sparkmap.py b/benchmark/sparkmap.py index 0ef2b0c..dacbb0a 100644 --- a/benchmark/sparkmap.py +++ b/benchmark/sparkmap.py @@ -9,7 +9,7 @@ import pandas as pd from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.types import IntegerType, StructField, StructType from tmlt.core.domains.collections import ListDomain diff --git a/src/tmlt/core/random/rng.py b/src/tmlt/core/random/rng.py index ff7d2db..1d07264 100644 --- a/src/tmlt/core/random/rng.py +++ b/src/tmlt/core/random/rng.py @@ -9,7 +9,6 @@ import numpy as np from randomgen.rdrand import RDRAND # pylint: disable=no-name-in-module from randomgen.wrapper import UserBitGenerator # pylint: disable=no-name-in-module -"""Can this be switched to something else to avoid import errors?""" try: _core_privacy_prng = np.random.Generator(RDRAND()) diff --git a/src/tmlt/core/transformations/spark_transformations/nan.py b/src/tmlt/core/transformations/spark_transformations/nan.py index 7ae2b66..44b5a3b 100644 --- a/src/tmlt/core/transformations/spark_transformations/nan.py +++ b/src/tmlt/core/transformations/spark_transformations/nan.py @@ -195,6 +195,7 @@ def stability_function(self, d_in: ExactNumberInput) -> ExactNumber: self.input_metric.validate(d_in) return ExactNumber(d_in) + # pylint: disable=invalid-unary-operand-type def __call__(self, sdf: DataFrame) -> DataFrame: """Drops rows containing +inf or -inf in ``self.columns``.""" return sdf.filter( @@ -209,6 +210,8 @@ def __call__(self, sdf: DataFrame) -> DataFrame: ) ) + # pylint: enable=invalid-unary-operand-type + class DropNaNs(Transformation): """Drops rows containing NaNs in one or more specified columns. diff --git a/test/unit/domains/test_numpy_domains.py b/test/unit/domains/test_numpy_domains.py index dc8b7b4..11f0bd3 100644 --- a/test/unit/domains/test_numpy_domains.py +++ b/test/unit/domains/test_numpy_domains.py @@ -801,9 +801,7 @@ def test_validate( @pytest.mark.parametrize( "dtype, expected, expectation", [ - # pylint: disable=line-too-long - (np.dtype(np.object_), NumpyStringDomain(), does_not_raise()), # type: ignore[attr-defined] - # pylint: enable=line-too-long + (np.dtype(np.object_), NumpyStringDomain(), does_not_raise()), ( np.dtype([("f1", np.int64)]), None, diff --git a/test/unit/domains/test_pandas_domains.py b/test/unit/domains/test_pandas_domains.py index 25d0d95..7366633 100644 --- a/test/unit/domains/test_pandas_domains.py +++ b/test/unit/domains/test_pandas_domains.py @@ -259,16 +259,14 @@ def test_validate( @pytest.mark.parametrize( "dtype, expected, expectation", - # pylint: disable=line-too-long [ ( np.dtype(dtype), PandasSeriesDomain(NumpyIntegerDomain(size=32)), does_not_raise(), ) - for dtype in [np.int8, np.int16, np.int32, np.bool_] # type: ignore[attr-defined] + for dtype in [np.int8, np.int16, np.int32, np.bool_] ] - # pylint: enable=line-too-long + [ ( np.dtype(np.float32), @@ -281,7 +279,7 @@ def test_validate( does_not_raise(), ), ( - np.dtype(np.object_), # type: ignore[attr-defined] + np.dtype(np.object_), PandasSeriesDomain(NumpyStringDomain()), does_not_raise(), ), @@ -290,9 +288,7 @@ def test_validate( PandasSeriesDomain(NumpyIntegerDomain(size=64)), does_not_raise(), ), - # pylint: disable=line-too-long - (np.dtype([("f1", np.int16)]), None, pytest.raises(KeyError)), # type: ignore - # pylint: enable=line-too-long + (np.dtype([("f1", np.int16)]), None, pytest.raises(KeyError)), ], ) def test_from_numpy_type( diff --git a/test/unit/domains/test_spark_domains.py b/test/unit/domains/test_spark_domains.py index a5df249..5943805 100644 --- a/test/unit/domains/test_spark_domains.py +++ b/test/unit/domains/test_spark_domains.py @@ -14,7 +14,7 @@ import pandas as pd import pytest from pyspark import Row -from pyspark.sql.dataframe import DataFrame +from pyspark.sql import DataFrame from pyspark.sql.session import SparkSession from pyspark.sql.types import ( DataType, From 14de39fb534bb77a8ff424766e7be6fa2d5b0bfa Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Sun, 2 Nov 2025 17:29:50 -0500 Subject: [PATCH 6/7] Minor changes --- noxfile.py | 6 ++--- uv.lock | 66 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/noxfile.py b/noxfile.py index 03f7f9e..ee13ebc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -71,8 +71,6 @@ def is_arm_mac(): "scipy": "==1.14.1", "randomgen": "==2.3.0", "pyarrow": "==16.1.0", - "py4j": "==0.10.9.9", - "tzdata": "==2025.2", }, ), DependencyConfiguration( @@ -97,7 +95,7 @@ def is_arm_mac(): "pandas": "==2.3.3", "numpy": "==2.2.6", "scipy": "==1.14.1", - "randomgen": "==1.26.1", + "randomgen": "==2.3.0", "pyarrow": "==16.1.0", }, ), @@ -124,7 +122,7 @@ def is_arm_mac(): "pandas": "==2.3.3", "numpy": "==2.2.6", "scipy": "==1.14.1", - "randomgen": "==1.26.1", + "randomgen": "==2.3.0", "pyarrow": "==16.1.0", }, ), diff --git a/uv.lock b/uv.lock index 2f6ef0c..7c8275c 100644 --- a/uv.lock +++ b/uv.lock @@ -906,34 +906,44 @@ wheels = [ [[package]] name = "numpy" -version = "1.26.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/94/ace0fdea5241a27d13543ee117cbc65868e82213fb31a8eb7fe9ff23f313/numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", size = 20631468, upload-time = "2024-02-05T23:48:01.194Z" }, - { url = "https://files.pythonhosted.org/packages/20/f7/b24208eba89f9d1b58c1668bc6c8c4fd472b20c45573cb767f59d49fb0f6/numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", size = 13966411, upload-time = "2024-02-05T23:48:29.038Z" }, - { url = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", size = 14219016, upload-time = "2024-02-05T23:48:54.098Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f", size = 18240889, upload-time = "2024-02-05T23:49:25.361Z" }, - { url = "https://files.pythonhosted.org/packages/24/03/6f229fe3187546435c4f6f89f6d26c129d4f5bed40552899fcf1f0bf9e50/numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", size = 13876746, upload-time = "2024-02-05T23:49:51.983Z" }, - { url = "https://files.pythonhosted.org/packages/39/fe/39ada9b094f01f5a35486577c848fe274e374bbf8d8f472e1423a0bbd26d/numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", size = 18078620, upload-time = "2024-02-05T23:50:22.515Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ef/6ad11d51197aad206a9ad2286dc1aac6a378059e06e8cf22cd08ed4f20dc/numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", size = 5972659, upload-time = "2024-02-05T23:50:35.834Z" }, - { url = "https://files.pythonhosted.org/packages/19/77/538f202862b9183f54108557bfda67e17603fc560c384559e769321c9d92/numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", size = 15808905, upload-time = "2024-02-05T23:51:03.701Z" }, - { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, - { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, - { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, - { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, ] [[package]] From fa4ccc25da826afa471aea95db52e2b0891c7a27 Mon Sep 17 00:00:00 2001 From: Charles Carlson Date: Tue, 11 Nov 2025 11:00:53 -0500 Subject: [PATCH 7/7] resolves pyspark.sql.classic.dataframe.DataFrame discrepancy --- noxfile.py | 108 +++++++++--------- src/tmlt/core/domains/base.py | 2 +- .../spark_transformations/test_agg.py | 2 +- 3 files changed, 57 insertions(+), 55 deletions(-) diff --git a/noxfile.py b/noxfile.py index ee13ebc..4c57cc0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -71,61 +71,63 @@ def is_arm_mac(): "scipy": "==1.14.1", "randomgen": "==2.3.0", "pyarrow": "==16.1.0", + # Probably don't need to specify "py4j": "==0.10.9.9", + # Probably don't need to specify "tzdata": "==2025.2", }, ), - DependencyConfiguration( - id="3.11-oldest", - python="3.11", - packages={ - "pyspark[sql]": "==3.4.0" if not is_mac() else "==3.5.0", - "sympy": "==1.8", - "pandas": "==1.5.0", - "numpy": "==1.26.0", - "scipy": "==1.9.2", - "randomgen": "==1.26.0", - "pyarrow": "==14.0.1", - }, - ), - DependencyConfiguration( - id="3.11-newest", - python="3.11", - packages={ - "pyspark[sql]": "==4.0.1", - "sympy": "==1.12", - "pandas": "==2.3.3", - "numpy": "==2.2.6", - "scipy": "==1.14.1", - "randomgen": "==2.3.0", - "pyarrow": "==16.1.0", - }, - ), - DependencyConfiguration( - id="3.12-oldest", - python="3.12", - packages={ - "pyspark[sql]": "==3.5.0", - "sympy": "==1.8", - "pandas": "==2.2.0", - "numpy": "==1.26.0", - "scipy": "==1.11.2", - "randomgen": "==1.26.0", - "pyarrow": "==14.0.1", - }, - ), - # 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797 - DependencyConfiguration( - id="3.12-newest", - python="3.12", - packages={ - "pyspark[sql]": "==4.0.1", - "sympy": "==1.12", - "pandas": "==2.3.3", - "numpy": "==2.2.6", - "scipy": "==1.14.1", - "randomgen": "==2.3.0", - "pyarrow": "==16.1.0", - }, - ), + # DependencyConfiguration( + # id="3.11-oldest", + # python="3.11", + # packages={ + # "pyspark[sql]": "==3.4.0" if not is_mac() else "==3.5.0", + # "sympy": "==1.8", + # "pandas": "==1.5.0", + # "numpy": "==1.26.0", + # "scipy": "==1.9.2", + # "randomgen": "==1.26.0", + # "pyarrow": "==14.0.1", + # }, + # ), + # DependencyConfiguration( + # id="3.11-newest", + # python="3.11", + # packages={ + # "pyspark[sql]": "==4.0.1", + # "sympy": "==1.12", + # "pandas": "==2.3.3", + # "numpy": "==2.2.6", + # "scipy": "==1.14.1", + # "randomgen": "==2.3.0", + # "pyarrow": "==16.1.0", + # }, + # ), + # DependencyConfiguration( + # id="3.12-oldest", + # python="3.12", + # packages={ + # "pyspark[sql]": "==3.5.0", + # "sympy": "==1.8", + # "pandas": "==2.2.0", + # "numpy": "==1.26.0", + # "scipy": "==1.11.2", + # "randomgen": "==1.26.0", + # "pyarrow": "==14.0.1", + # }, + # ), + # # 3.12 support was added in sympy 1.12.1 but internal cap is at 1.9 #1797 + # DependencyConfiguration( + # id="3.12-newest", + # python="3.12", + # packages={ + # "pyspark[sql]": "==4.0.1", + # "sympy": "==1.12", + # "pandas": "==2.3.3", + # "numpy": "==2.2.6", + # "scipy": "==1.14.1", + # "randomgen": "==2.3.0", + # "pyarrow": "==16.1.0", + # }, + # ), ] AUDIT_VERSIONS = ["3.10", "3.11", "3.12"] diff --git a/src/tmlt/core/domains/base.py b/src/tmlt/core/domains/base.py index 58a8b7c..ac4a5ba 100644 --- a/src/tmlt/core/domains/base.py +++ b/src/tmlt/core/domains/base.py @@ -21,7 +21,7 @@ def carrier_type(self) -> type: def validate(self, value: Any) -> None: """Raises an error if value is not in the domain.""" - if value.__class__ is not self.carrier_type: + if not isinstance(value, self.carrier_type): raise OutOfDomainError( self, value, diff --git a/test/unit/transformations/spark_transformations/test_agg.py b/test/unit/transformations/spark_transformations/test_agg.py index 09eb233..d7094a1 100644 --- a/test/unit/transformations/spark_transformations/test_agg.py +++ b/test/unit/transformations/spark_transformations/test_agg.py @@ -505,7 +505,7 @@ def test_correctness( count_distinct_groups = CountDistinctGrouped( input_domain=SparkGroupedDataFrameDomain( schema={ - "X": SparkIntegerColumnDescriptor(), + "X": SparkIntegerColumnDescriptor(allow_null=True), "Y": SparkStringColumnDescriptor(allow_null=True), "Z": SparkFloatColumnDescriptor( allow_nan=True, allow_null=True, allow_inf=True