Feature/1270 (#761)

* embedded tuples

* embedded collection files

* embedding objects + test parallelism

* embedding dicts

* deep unreferencing tests

* defensive thread shutdown

* refresh fix (edge case for initial prefix)

---------

Co-authored-by: Wojtek <wskozlowski@dbzero.io>

Author: wskozlowski

AGENTS.md

@@ -23,6 +23,7 @@ All tests must pass before a change is considered complete.
 ### Running tests
 
 - Python tests: `./scripts/run_tests.sh`
+- Final Python test checks: `./scripts/run_tests.sh -j 6`
 - C++ tests after a `-t` build: `./build/release/tests.x`
 - During development, do not run stress tests by default; they are intentionally slow. Run focused tests specific to the feature or refactor being worked on before finalization.
 - If any C++ source under the native/core part of the project was modified, also run the C++ test suite (do not rely on the Python tests alone to cover native changes).

CONTRIBUTING.md

@@ -94,6 +94,9 @@ This project adheres to a code of conduct that all contributors are expected to
    # Run Python tests (excludes stress tests)
    ./scripts/run_tests.sh
 
+   # Run Python tests in parallel
+   ./scripts/run_tests.sh -j 6
+   
    # Run specific Python test
    ./scripts/run_tests.sh -k=test_name
 

dbzero/dbzero/dbzero.py

@@ -10,7 +10,7 @@ def load_dynamic(name, path):
 
 def __bootstrap__():
     global __bootstrap__, __loader__, __file__
-    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"]
+    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"]
     __file__ = None
     for path in paths:
         if os.path.isdir(path):

python_tests/conftest.py

@@ -13,7 +13,17 @@
 
 
 TEST_FILES_DIR_ROOT = os.path.join(os.getcwd(), "python_tests", "files")
-DB0_DIR = os.path.join(os.getcwd(), "db0-test-data")
+PYTEST_WORKER_ID = os.environ.get("PYTEST_XDIST_WORKER")
+WORKER_SUFFIX = f"-{PYTEST_WORKER_ID}" if PYTEST_WORKER_ID else ""
+DB0_DIR = os.path.join(os.getcwd(), f"db0-test-data{WORKER_SUFFIX}")
+
+
+def worker_path(path):
+    if not WORKER_SUFFIX:
+        return path
+    directory, filename = os.path.split(path)
+    name, extension = os.path.splitext(filename)
+    return os.path.join(directory, f"{name}{WORKER_SUFFIX}{extension}")
 
 
 def __extract_param(request, key, default):
@@ -211,4 +221,4 @@ def db0_large_lang_cache_no_autocommit():
     yield db0    
     db0.close()    
     if os.path.exists(DB0_DIR):
-        shutil.rmtree(DB0_DIR)
+        shutil.rmtree(DB0_DIR)

python_tests/test_copy_prefix.py

@@ -6,12 +6,12 @@
 import os
 import time
 from .memo_test_types import MemoTestClass, MemoTestSingleton
-from .conftest import DB0_DIR
+from .conftest import DB0_DIR, worker_path
 import multiprocessing
 
 
 def test_copy_current_prefix(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -30,7 +30,7 @@ def test_copy_current_prefix(db0_fixture):
 
 
 def test_recover_prefix_from_copy(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -58,7 +58,7 @@ def test_recover_prefix_from_copy(db0_fixture):
 
 
 def test_copy_prefix_custom_step_size(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     if os.path.exists(file_name):
         os.remove(file_name)
 
@@ -110,7 +110,7 @@ def writer_process(prefix, obj_count = 50, commit_count = 50, long_run = False):
 
 
 def test_copy_prefix_being_actively_modified(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     if os.path.exists(file_name):
         os.remove(file_name)
 
@@ -153,7 +153,7 @@ def test_copy_prefix_being_actively_modified(db0_fixture):
 
 
 def test_copy_prefix_fails_if_no_active_prefix(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -172,7 +172,7 @@ def test_copy_prefix_fails_if_no_active_prefix(db0_fixture):
 
 
 def test_copy_prefix_without_opening_it(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -207,7 +207,7 @@ def validate_current_prefix(expected_len = None, expected_min_len = None):
         return len(root.value)
 
     def validate_copy(copy_id, expected_len = None, expected_min_len = None):
-        file_name = f"./test-copy-{copy_id}.db0"
+        file_name = worker_path(f"./test-copy-{copy_id}.db0")
         os.remove(px_path)
         # restore the copy
         os.rename(file_name, px_path)
@@ -250,7 +250,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
         while True:
             if not p.is_alive():
                 break
-            file_name = f"./test-copy-{copy_id}.db0"
+            file_name = worker_path(f"./test-copy-{copy_id}.db0")
             if os.path.exists(file_name):
                 os.remove(file_name)
             # copy prefix without opening it, use default step size
@@ -266,7 +266,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
         total_len += obj_count * commit_count
 
         # make final stale copy (i.e. without active modifications)
-        final_copy = f"./test-copy-final.db0"
+        final_copy = worker_path("./test-copy-final.db0")
         if os.path.exists(final_copy):
             os.remove(final_copy)
         db0.copy_prefix(final_copy, prefix=px_name)    
@@ -284,7 +284,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
 
 
 def test_modify_copied_prefix(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -326,7 +326,7 @@ def modify_prefix():
 
 @pytest.mark.parametrize("db0_fixture", [{"autocommit": False}], indirect=True)
 def test_copy_prefix_of_recovered_copy(db0_fixture):
-    file_name = "./test-copy.db0"
+    file_name = worker_path("./test-copy.db0")
     # remove file if it exists
     if os.path.exists(file_name):
         os.remove(file_name)
@@ -398,7 +398,7 @@ def validate_current_prefix(expected_len = None, expected_min_len = None):
             return len(root.value)
 
         def validate_copy(copy_id, expected_len = None, expected_min_len = None):
-            file_name = f"./test-copy-{copy_id}.db0"
+            file_name = worker_path(f"./test-copy-{copy_id}.db0")
             os.remove(px_path)
             # restore the copy
             os.rename(file_name, px_path)                        
@@ -437,7 +437,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
         while True:
             if not p.is_alive():
                 break
-            file_name = f"./test-copy-{copy_id}.db0"
+            file_name = worker_path(f"./test-copy-{copy_id}.db0")
             if os.path.exists(file_name):
                 os.remove(file_name)
             db0.copy_prefix(file_name, prefix=px_name)
@@ -468,7 +468,7 @@ def validate_current_prefix(expected_len = None, expected_min_len = None):
             return len(root.value)
 
         def validate_copy(copy_id, expected_len = None, expected_min_len = None):
-            file_name = f"./test-copy-{copy_id}.db0"
+            file_name = worker_path(f"./test-copy-{copy_id}.db0")
             os.remove(px_path)
             # restore the copy
             os.rename(file_name, px_path)
@@ -513,7 +513,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
             while True:
                 if not p.is_alive():
                     break
-                file_name = f"./test-copy-{copy_id}.db0"
+                file_name = worker_path(f"./test-copy-{copy_id}.db0")
                 if os.path.exists(file_name):
                     os.remove(file_name)
                 # copy prefix without opening it, use default step size
@@ -529,7 +529,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
             total_len += obj_count * commit_count
 
             # make final stale copy (i.e. without active modifications)
-            final_copy = f"./test-copy-final.db0"
+            final_copy = worker_path("./test-copy-final.db0")
             if os.path.exists(final_copy):
                 os.remove(final_copy)
             db0.copy_prefix(final_copy, prefix=px_name)    
@@ -543,4 +543,4 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
                 # this is the restored version
                 total_len = last_len
 
-   
+   

python_tests/test_issues_14.py

@@ -3,7 +3,7 @@
 
 import dbzero as db0
 import pytest
-from .conftest import DB0_DIR
+from .conftest import DB0_DIR, worker_path
 from .memo_test_types import MemoTestSingleton, MemoTestClass
 import multiprocessing
 import os
@@ -43,7 +43,7 @@ def validate_current_prefix(expected_len = None, expected_min_len = None):
         return len(root.value)
 
     def validate_copy(copy_id, expected_len = None, expected_min_len = None):
-        file_name = f"./test-copy-{copy_id}.db0"
+        file_name = worker_path(f"./test-copy-{copy_id}.db0")
         os.remove(px_path)
         # restore the copy
         os.rename(file_name, px_path)
@@ -69,7 +69,7 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
         db0.open(px_name, "r")
 
         # make final stale copy (i.e. without active modifications)
-        final_copy = f"./test-copy-final.db0"
+        final_copy = worker_path("./test-copy-final.db0")
         if os.path.exists(final_copy):
             os.remove(final_copy)
         db0.copy_prefix(final_copy, prefix=px_name)