feat(python): add snippet tests and Linux CI

.claude/CLAUDE.md

@@ -26,9 +26,20 @@ python python/loading/add_records.py
 black --line-length 120 python/
 isort --profile black python/
 pylint python/
-flake8 python/
+flake8 --max-line-length 120 --extend-ignore E203,E501,E704,W503 python/
 mypy python/
-bandit python/
+bandit -c pyproject.toml -r python/
+```
+
+**Testing:**
+
+`python/tests` runs every snippet end to end, each against its own temporary SQLite repository (`SENZING_ENGINE_CONFIGURATION_JSON` is not used). Snippets needing data, input or ctrl-c get an entry in `SNIPPETS` in `python/tests/test_snippets.py`.
+
+```bash
+export PYTHONPATH=/opt/senzing/er/sdk/python
+export LD_LIBRARY_PATH=/opt/senzing/er/lib
+python -m pytest
+python -m pytest -k loading    # run a subset
 ```
 
 ### Java

.github/workflows/bandit.yaml

@@ -23,7 +23,8 @@ jobs:
 
     steps:
       - name: Run Bandit Scan
-        uses: PyCQA/bandit-action@v1
+        uses: PyCQA/bandit-action@v1.0.1
         with:
           python-version: ${{ matrix.python-version }}
+          configfile: pyproject.toml
           targets: python

.github/workflows/python-linux-snippets.yaml

@@ -0,0 +1,74 @@
+name: python linux snippets
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "15 7 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  sdk-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      senzingsdk-versions: ${{ steps.cfg.outputs.senzingsdk-versions }}
+    steps:
+      - id: cfg
+        uses: senzing-factory/build-resources/sdk-versions@v4
+
+  python-linux-snippets:
+    needs: sdk-versions
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        senzingsdk-version: ${{ fromJSON(needs.sdk-versions.outputs.senzingsdk-versions) }}
+
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v7.0.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install test dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --group test
+
+      - name: install Senzing runtime
+        uses: senzing-factory/github-action-install-senzing-sdk@v5
+        with:
+          packages-to-install: "senzingsdk-runtime senzingsdk-setup"
+          senzingsdk-version: ${{ matrix.senzingsdk-version }}
+
+      - name: run python snippets
+        env:
+          LD_LIBRARY_PATH: "/opt/senzing/er/lib"
+          PYTHONPATH: "/opt/senzing/er/sdk/python"
+        run: |
+          python -m pytest -rs
+
+      - name: Notify Slack on failure
+        if: (failure() || cancelled()) && github.event_name == 'schedule'
+        uses: senzing-factory/build-resources/slack-failure-notification@v4
+        with:
+          job-status: ${{ job.status }}
+          slack-channel: ${{ secrets.SLACK_CHANNEL }}
+          slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          additional-info: "SDK: ${{ matrix.senzingsdk-version }}, Python: ${{ matrix.python-version }}"

.gitignore

@@ -135,8 +135,9 @@ dmypy.json
 *.code-workspace
 .history
 
-# Output folder used by examples
-resources/output/
+# Output folder used by examples; keep the folder, ignore its contents
+resources/output/*
+!resources/output/.gitkeep
 
 csharp/snippets/**/bin
 csharp/snippets/**/obj

.vscode/cspell.json

@@ -5,13 +5,15 @@
     "accessibilities",
     "analysing",
     "antaenc",
+    "autouse",
     "Axxxx",
     "barrycaceres",
     "bugtracker",
     "CCLA",
     "cfgp",
     "codehaus",
     "CODEOWNER",
+    "configfile",
     "configmanager",
     "configmgr",
     "configparser",
@@ -22,6 +24,7 @@
     "Dsenzing",
     "DSRC",
     "DYLD",
+    "executescript",
     "findsecbugs",
     "fpvs",
     "glassfish",
@@ -43,12 +46,15 @@
     "pydevproject",
     "pylint",
     "pypa",
+    "pyproject",
     "pytest",
+    "PYTHONPATH",
     "RESOURCEPATH",
     "Retryable",
     "roslynator",
     "Rotorville",
     "schemaname",
+    "SENZ",
     "Senzing",
     "senzingsdk",
     "setuptools",
@@ -64,6 +70,7 @@
     "taskkill",
     "temurin",
     "terashuf",
+    "testpaths",
     "tpascalcase",
     "truthset",
     "unresolve",

CHANGELOG.md

@@ -10,6 +10,23 @@ This project adheres to [Semantic Versioning].
 ### Added in Unreleased
 
 - Java Migration Guide (`java/Migration.md`) covering the version 3.x to 4.0 SDK changes
+- Python snippet tests (`python/tests`) running every snippet end to end against a temporary SQLite repository
+- GitHub Actions workflow running the Python snippet tests on Linux
+
+### Changed in Unreleased
+
+- Python snippets exit with status 1 and report to stderr when a Senzing error stops them
+- `resources/output/` is now part of the repository so snippets writing with-info output work on a fresh clone
+- `add_queue.py` uses a producer thread instead of separate processes, matching the Java and C# snippets
+- Python futures and queue snippets set their thread count with `MAX_WORKERS` (8, matching Java) instead of reading the
+  executor's private `_max_workers`
+
+### Fixed in Unreleased
+
+- `redo_with_info_continuous.py` raised a `TypeError` instead of exiting when a Senzing error occurred
+- `add_queue.py` loaded no records but exited successfully on Python 3.14+, and could stop early if the queue was
+  momentarily empty
+- `redo_continuous_futures.py` spun calling `get_redo_record()` when fewer redo records than workers were waiting
 
 ## [0.0.10] - 2025-08-11
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "code-snippets-v4"
-version = "1.2.8"
+version = "0.0.10"
 description = "Code snippets for Senzing V4"
 authors = [{ name = "senzing", email = "support@senzing.com" }]
 readme = "README.md"
@@ -67,6 +67,9 @@ test = [
   "coverage==7.16.1; python_version > '3.11'",
 ]
 
+[tool.pytest.ini_options]
+testpaths = ["python/tests"]
+
 [tool.setuptools]
 package-dir = { "" = "python" }
 

python/README.md

@@ -1 +1,15 @@
 # Python Snippets
+
+## Running the tests
+
+`python/tests` runs every snippet end to end as a subprocess, each against its own temporary SQLite repository, and checks it exits cleanly with nothing on stderr. Your `SENZING_ENGINE_CONFIGURATION_JSON` is not used and your repository is not touched.
+
+```bash
+export PYTHONPATH=/opt/senzing/er/sdk/python
+export LD_LIBRARY_PATH=/opt/senzing/er/lib
+python -m pytest
+```
+
+Senzing install locations default to `/opt/senzing/er`, `/opt/senzing/data` and `/etc/opt/senzing`; override them with the same environment variables the Java SnippetRunner uses: `SENZING_PATH`, `SENZING_DIR`, `SENZING_SUPPORT_DIR`, `SENZING_RESOURCE_DIR` and `SENZING_CONFIG_DIR`.
+
+Snippets are discovered automatically. A snippet needing data loaded, registered data sources, console input or a ctrl-c to stop it gets an entry in `SNIPPETS` in `tests/test_snippets.py`.

python/configuration/get_config_registry.py

@@ -18,3 +18,4 @@
     print(response)
 except SzError as err:
     print(f"{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/configuration/get_data_source_registry.py

@@ -21,3 +21,4 @@
     print(response)
 except SzError as err:
     print(f"{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/configuration/init_default_config.py

@@ -26,3 +26,4 @@
     print(f"New default config ID: {new_config_id}")
 except SzError as err:
     print(f"{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/configuration/register_data_sources.py

@@ -27,3 +27,4 @@
     print(f"New default config ID: {new_config_id}")
 except SzError as err:
     print(f"{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/deleting/delete_futures.py

@@ -12,6 +12,7 @@
 
 INPUT_FILE = Path("../../resources/data/del-500.jsonl").resolve()
 INSTANCE_NAME = Path(__file__).stem
+MAX_WORKERS = 8
 SETTINGS = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", "{}")
 
 
@@ -34,10 +35,10 @@ def futures_del(engine, input_file):
     success_recs = 0
 
     with open(input_file, "r", encoding="utf-8") as in_file:
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
             futures = {
                 executor.submit(delete_record, engine, record): record
-                for record in itertools.islice(in_file, executor._max_workers)
+                for record in itertools.islice(in_file, MAX_WORKERS)
             }
 
             while futures:
@@ -73,3 +74,4 @@ def futures_del(engine, input_file):
     futures_del(sz_engine, INPUT_FILE)
 except SzError as err:
     mock_logger("CRITICAL", err)
+    sys.exit(1)

python/deleting/delete_loop.py

@@ -54,3 +54,4 @@ def del_records_from_file(engine, input_file):
     del_records_from_file(sz_engine, INPUT_FILE)
 except SzError as err:
     mock_logger("CRITICAL", err)
+    sys.exit(1)

python/deleting/delete_with_info_futures.py

@@ -18,6 +18,7 @@
 
 INPUT_FILE = Path("../../resources/data/del-500.jsonl").resolve()
 INSTANCE_NAME = Path(__file__).stem
+MAX_WORKERS = 8
 OUTPUT_FILE = Path("../../resources/output/delete_file_with_info.jsonl").resolve()
 SETTINGS = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", "{}")
 
@@ -42,10 +43,10 @@ def futures_del(engine, input_file, output_file):
 
     with open(output_file, "w", encoding="utf-8") as out_file:
         with open(input_file, "r", encoding="utf-8") as in_file:
-            with concurrent.futures.ThreadPoolExecutor() as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
                 futures = {
                     executor.submit(delete_record, engine, record): record
-                    for record in itertools.islice(in_file, executor._max_workers)
+                    for record in itertools.islice(in_file, MAX_WORKERS)
                 }
 
                 while futures:
@@ -84,3 +85,4 @@ def futures_del(engine, input_file, output_file):
     futures_del(sz_engine, INPUT_FILE, OUTPUT_FILE)
 except SzError as err:
     mock_logger("CRITICAL", err)
+    sys.exit(1)

python/information/check_repository_performance.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
 import os
+import sys
 from pathlib import Path
 
 from senzing import SzError
@@ -15,4 +16,5 @@
     sz_diagnostic = sz_factory.create_diagnostic()
     print(sz_diagnostic.check_repository_performance(SECONDS_TO_RUN))
 except SzError as err:
-    print(f"\n{err.__class__.__name__} - {err}")
+    print(f"\n{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/information/get_license.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
 import os
+import sys
 from pathlib import Path
 
 from senzing import SzError
@@ -14,4 +15,5 @@
     sz_product = sz_factory.create_product()
     print(sz_product.get_license())
 except SzError as err:
-    print(f"\n{err.__class__.__name__} - {err}")
+    print(f"\n{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/information/get_repository_info.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
 import os
+import sys
 from pathlib import Path
 
 from senzing import SzError
@@ -14,4 +15,5 @@
     sz_diagnostic = sz_factory.create_diagnostic()
     print(sz_diagnostic.get_repository_info())
 except SzError as err:
-    print(f"\n{err.__class__.__name__} - {err}")
+    print(f"\n{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)

python/information/get_stats.py

@@ -12,6 +12,7 @@
 
 INPUT_FILE = Path("../../resources/data/load-500.jsonl").resolve()
 INSTANCE_NAME = Path(__file__).stem
+MAX_WORKERS = 8
 SETTINGS = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", "{}")
 
 
@@ -44,10 +45,9 @@ def futures_add(engine, input_file):
     error_recs = 0
 
     with open(input_file, "r", encoding="utf-8") as in_file:
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
             futures = {
-                executor.submit(add_record, engine, record): record
-                for record in itertools.islice(in_file, executor._max_workers)
+                executor.submit(add_record, engine, record): record for record in itertools.islice(in_file, MAX_WORKERS)
             }
 
             while futures:
@@ -86,3 +86,4 @@ def futures_add(engine, input_file):
     futures_add(sz_engine, INPUT_FILE)
 except SzError as err:
     mock_logger("CRITICAL", err)
+    sys.exit(1)

python/information/get_version.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
 import os
+import sys
 from pathlib import Path
 
 from senzing import SzError
@@ -14,4 +15,5 @@
     sz_product = sz_factory.create_product()
     print(sz_product.get_version())
 except SzError as err:
-    print(f"\n{err.__class__.__name__} - {err}")
+    print(f"\n{err.__class__.__name__} - {err}", file=sys.stderr)
+    sys.exit(1)