tuplex · LeonhardFS · Mar 8, 2025 · Mar 7, 2025 · Mar 8, 2025 · Mar 8, 2025
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,23 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+    - id: check-yaml
+      files: ^tuplex/python/tuplex.*\.py$
+    - id: end-of-file-fixer
+      files: ^tuplex/python/tuplex.*\.py$
+    - id: trailing-whitespace
+      files: ^tuplex/python/tuplex.*\.py$
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.9.9
+  hooks:
+    # Run the linter.
+    - id: ruff
+      files: ^tuplex/python/tuplex.*\.py$
+      args: [ "--fix", "--select", "I" ]
+      types_or: [ python, pyi ]
+    # Run the formatter.
+    - id: ruff-format
+      files: ^tuplex/python/tuplex.*\.py$
+      types_or: [ python, pyi ]
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,3 +12,7 @@ requires = [
     "requests"
 ]
 build-backend = "setuptools.build_meta"
+
+
+[tool.ruff]
+include = ["pyproject.toml", "tuplex/python/tuplex/**/*.py"]
diff --git a/tuplex/core/src/Executor.cc b/tuplex/core/src/Executor.cc
@@ -104,15 +104,17 @@ namespace tuplex {
                     // save which thread executed this task
                 task->setID(std::this_thread::get_id());
 
-                _numPendingTasks.fetch_add(-1, std::memory_order_release);
-
-                // add task to done list
+                // Add task to done list, execute before decreasing pending task.
                 TRACE_LOCK("completedTasks");
                 _completedTasksMutex.lock();
                 _completedTasks.push_back(std::move(task));
                 _completedTasksMutex.unlock();
                 _numCompletedTasks.fetch_add(1, std::memory_order_release);
                 TRACE_UNLOCK("completedTasks");
+
+                // This needs to come last, because other threads may be waiting on it.
+                _numPendingTasks.fetch_add(-1, std::memory_order_release);
+
                 return true;
             }
         } else {

diff --git a/tuplex/python/tuplex/__init__.py b/tuplex/python/tuplex/__init__.py
@@ -9,29 +9,35 @@
 #  License: Apache 2.0                                                                                                 #
 # ----------------------------------------------------------------------------------------------------------------------#
 
-from tuplex.repl import *
-from .context import Context
-from .dataset import DataSet
+import logging
 
 # expose aws setup for better convenience
 import tuplex.distributed
-import logging
-from tuplex.distributed import setup_aws
-
+from tuplex.distributed import setup_aws as setup_aws
+from tuplex.repl import in_google_colab as in_google_colab
+from tuplex.repl import in_jupyter_notebook as in_jupyter_notebook
 from tuplex.utils.version import __version__ as __version__
 
+from .context import Context
+from .dataset import DataSet as DataSet
+
+
 # for convenience create a dummy function to return a default-configured Lambda context
 def LambdaContext(conf=None, name=None, s3_scratch_dir=None, **kwargs):
     import uuid
 
     if s3_scratch_dir is None:
         s3_scratch_dir = tuplex.distributed.default_scratch_dir()
-        logging.debug('Detected default S3 scratch dir for this user as {}'.format(s3_scratch_dir))
+        logging.debug(
+            "Detected default S3 scratch dir for this user as {}".format(s3_scratch_dir)
+        )
 
-    lambda_conf = {'backend': 'lambda',
-                   'partitionSize': '1MB',
-                   'aws.scratchDir': s3_scratch_dir,
-                   'aws.requesterPay': True}
+    lambda_conf = {
+        "backend": "lambda",
+        "partitionSize": "1MB",
+        "aws.scratchDir": s3_scratch_dir,
+        "aws.requesterPay": True,
+    }
 
     if conf:
         lambda_conf.update(conf)
@@ -40,13 +46,13 @@ def LambdaContext(conf=None, name=None, s3_scratch_dir=None, **kwargs):
     for k, v in kwargs.items():
         if k in conf.keys():
             lambda_conf[k] = v
-        elif 'tuplex.' + k in conf.keys():
-            lambda_conf['tuplex.' + k] = v
+        elif "tuplex." + k in conf.keys():
+            lambda_conf["tuplex." + k] = v
         else:
             lambda_conf[k] = v
 
     if name is None:
-        name = 'AWSLambdaContext-' + str(uuid.uuid4())[:8]
+        name = "AWSLambdaContext-" + str(uuid.uuid4())[:8]
 
     # There's currently a bug in the Lambda backend when transferring local data to S3: The full partition
     # gets transferred, not just what is needed.