referencing immutable and embedded objects

Author: Wojtek

python_tests/test_memo_immutable.py

@@ -10,6 +10,109 @@
 import gc
 
 
+OBJECT_REF_STORAGE_CLASS = 13
+UNIQUE_ADDRESS_INSTANCE_ID_SHIFT = 14
+BASE32_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+
+
+def _base32_encode(data):
+    table = (
+        (0b11111000, 3), (0b00000111, -2), (0b11000000, 6), (0b00111110, 1),
+        (0b00000001, -4), (0b11110000, 4), (0b00001111, -1), (0b10000000, 7),
+        (0b01111100, 2), (0b00000011, -3), (0b11100000, 5), (0b00011111, 0),
+    )
+    strides = (1, 2, 1, 2, 2, 1, 2, 1)
+    result = []
+    data_at = 0
+    table_at = 0
+    stride_at = 0
+    while data_at < len(data):
+        enc_value = 0
+        in_val = data[data_at]
+        for _ in range(strides[stride_at]):
+            mask, shift = table[table_at]
+            if shift > 0:
+                enc_value |= (in_val & mask) >> shift
+            else:
+                enc_value |= (in_val & mask) << -shift
+                data_at += 1
+                in_val = data[data_at] if data_at != len(data) else 0
+            table_at = (table_at + 1) % len(table)
+        result.append(BASE32_CHARS[enc_value])
+        stride_at = (stride_at + 1) % len(strides)
+    return "".join(result)
+
+
+def _base32_decode(data):
+    table = (
+        (0b00011111, 3), (0b00011100, -2), (0b00000011, 6), (0b00011111, 1),
+        (0b00010000, -4), (0b00001111, 4), (0b00011110, -1), (0b00000001, 7),
+        (0b00011111, 2), (0b00011000, -3), (0b00000111, 5), (0b00011111, 0),
+    )
+    strides = (2, 3, 2, 3, 2)
+    result = bytearray([0])
+    table_at = 0
+    stride_at = 0
+    stride = strides[stride_at]
+    for char_at, char in enumerate(data):
+        value = BASE32_CHARS.index(char)
+        while True:
+            mask, shift = table[table_at]
+            if shift >= 0:
+                result[-1] |= (value & mask) << shift
+                table_at = (table_at + 1) % len(table)
+                stride -= 1
+                if stride == 0 and char_at != len(data) - 1:
+                    result.append(0)
+                    stride_at = (stride_at + 1) % len(strides)
+                    stride = strides[stride_at]
+                break
+            result[-1] |= (value & mask) >> -shift
+            table_at = (table_at + 1) % len(table)
+            stride -= 1
+            if stride == 0:
+                result.append(0)
+                stride_at += 1
+                stride = strides[stride_at]
+    return bytes(result)
+
+
+def _write_packed_int(value):
+    result = bytearray([value & 0x7F])
+    value >>= 7
+    while value:
+        result.insert(0, (value & 0x7F) | 0x80)
+        value >>= 7
+    return bytes(result)
+
+
+def _read_packed_int(data, at):
+    value = 0
+    while data[at] & 0x80:
+        value |= data[at] & 0x7F
+        value <<= 7
+        at += 1
+    value |= data[at] & 0x7F
+    return value, at + 1
+
+
+def _decode_uuid(uuid):
+    data = _base32_decode(uuid)
+    fixture_uuid = int.from_bytes(data[:8], "little")
+    unique_address, at = _read_packed_int(data, 8)
+    storage_class, _ = _read_packed_int(data, at)
+    return fixture_uuid, unique_address, storage_class
+
+
+def _encode_uuid(fixture_uuid, unique_address, storage_class):
+    data = (
+        fixture_uuid.to_bytes(8, "little")
+        + _write_packed_int(unique_address)
+        + _write_packed_int(storage_class)
+    )
+    return _base32_encode(data)
+
+
 @db0.memo(immutable=True, no_default_tags=True)
 @dataclass
 class MemoImmutableClass1:
@@ -36,6 +139,13 @@ class MemoImmutableNestedPayload:
     count: int
 
 
+@db0.memo(no_default_tags=True)
+class MemoRegularFetchUUIDPayload:
+    def __init__(self, name, count):
+        self.name = name
+        self.count = count
+
+
 @db0.memo(immutable=True, no_default_tags=True)
 class MemoImmutableNestedHolder:
     def __init__(self, name, count, label):
@@ -50,6 +160,27 @@ def __init__(self, nested, label):
         self.label = label
 
 
+@db0.memo(immutable=True, no_default_tags=True)
+class MemoImmutableDeepLeaf:
+    def __init__(self, name, count):
+        self.name = name
+        self.count = count
+
+
+@db0.memo(immutable=True, no_default_tags=True)
+class MemoImmutableDeepMiddle:
+    def __init__(self, name, count):
+        self.name = name
+        self.leaf = MemoImmutableDeepLeaf(name=f"{name}-leaf", count=count)
+
+
+@db0.memo(immutable=True, no_default_tags=True)
+class MemoImmutableDeepRoot:
+    def __init__(self, name, count):
+        self.middle = MemoImmutableDeepMiddle(name=name, count=count)
+        self.label = "deep-root"
+
+
 @db0.memo(immutable=True, no_default_tags=True)
 class MemoImmutableTupleHolder:
     def __init__(self, payload):
@@ -86,6 +217,120 @@ def test_create_memo_immutable(db0_fixture):
     _ = MemoImmutableClass1(data="immutable data", value=42)
 
 
+def test_uuid_and_fetch_regular_memo_object(db0_fixture):
+    obj = MemoRegularFetchUUIDPayload("regular uuid", 101)
+    obj_uuid = db0.uuid(obj)
+
+    assert db0.fetch(obj_uuid) is obj
+    assert db0.fetch(MemoRegularFetchUUIDPayload, obj_uuid) is obj
+
+    db0.tags(obj).add("keep-regular-fetch-uuid")
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened = db0.fetch(obj_uuid)
+    assert isinstance(reopened, MemoRegularFetchUUIDPayload)
+    assert reopened.name == "regular uuid"
+    assert reopened.count == 101
+
+
+def test_uuid_and_fetch_immutable_root_object(db0_fixture):
+    obj = MemoImmutableClass1(data="immutable uuid", value=102)
+    db0.tags(obj).add("keep-immutable-fetch-uuid")
+    obj_uuid = db0.uuid(obj)
+
+    assert db0.fetch(obj_uuid) is obj
+    assert db0.fetch(MemoImmutableClass1, obj_uuid) is obj
+
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened = db0.fetch(obj_uuid)
+    assert isinstance(reopened, MemoImmutableClass1)
+    assert reopened.data == "immutable uuid"
+    assert reopened.value == 102
+
+
+def test_uuid_and_fetch_embedded_nested_immutable_object(db0_fixture):
+    root = MemoImmutableNestedHolder(name="embedded uuid", count=103, label="root")
+    db0.tags(root).add("keep-embedded-fetch-uuid")
+    nested = root.nested
+    nested_uuid = db0.uuid(nested)
+
+    assert nested_uuid != db0.uuid(root)
+    fetched = db0.fetch(nested_uuid)
+    fetched_by_type = db0.fetch(MemoImmutableNestedPayload, nested_uuid)
+    assert isinstance(fetched, MemoImmutableNestedPayload)
+    assert fetched.name == "embedded uuid"
+    assert fetched.count == 103
+    assert db0.uuid(fetched) == nested_uuid
+    assert db0.uuid(fetched_by_type) == nested_uuid
+
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened = db0.fetch(nested_uuid)
+    assert isinstance(reopened, MemoImmutableNestedPayload)
+    assert reopened.name == "embedded uuid"
+    assert reopened.count == 103
+
+
+def test_uuid_and_fetch_deeply_embedded_immutable_objects(db0_fixture):
+    root = MemoImmutableDeepRoot(name="deep embedded uuid", count=104)
+    db0.tags(root).add("keep-deep-embedded-fetch-uuid")
+    middle = root.middle
+    leaf = middle.leaf
+    middle_uuid = db0.uuid(middle)
+    leaf_uuid = db0.uuid(leaf)
+
+    assert middle_uuid != db0.uuid(root)
+    assert leaf_uuid != middle_uuid
+    fetched_middle = db0.fetch(middle_uuid)
+    fetched_leaf = db0.fetch(leaf_uuid)
+    assert isinstance(fetched_middle, MemoImmutableDeepMiddle)
+    assert isinstance(fetched_leaf, MemoImmutableDeepLeaf)
+    assert fetched_middle.name == "deep embedded uuid"
+    assert fetched_leaf.name == "deep embedded uuid-leaf"
+    assert db0.uuid(fetched_middle) == middle_uuid
+    assert db0.uuid(fetched_leaf) == leaf_uuid
+
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened_middle = db0.fetch(middle_uuid)
+    reopened_leaf = db0.fetch(leaf_uuid)
+    assert isinstance(reopened_middle, MemoImmutableDeepMiddle)
+    assert isinstance(reopened_leaf, MemoImmutableDeepLeaf)
+    assert reopened_middle.name == "deep embedded uuid"
+    assert reopened_middle.leaf.name == "deep embedded uuid-leaf"
+    assert reopened_leaf.name == "deep embedded uuid-leaf"
+    assert reopened_leaf.count == 104
+
+
+def test_fetch_rejects_invalid_embedded_uuid_inside_existing_allocation(db0_fixture):
+    root = MemoImmutableDeepRoot(name="bad embedded uuid", count=105)
+    db0.tags(root).add("keep-invalid-embedded-fetch-uuid")
+    leaf_uuid = db0.uuid(root.middle.leaf)
+    fixture_uuid, unique_address, storage_class = _decode_uuid(leaf_uuid)
+    assert storage_class == OBJECT_REF_STORAGE_CLASS
+
+    address = unique_address >> UNIQUE_ADDRESS_INSTANCE_ID_SHIFT
+    instance_id = unique_address & ((1 << UNIQUE_ADDRESS_INSTANCE_ID_SHIFT) - 1)
+    invalid_unique_address = ((address + 1) << UNIQUE_ADDRESS_INSTANCE_ID_SHIFT) | instance_id
+    invalid_uuid = _encode_uuid(fixture_uuid, invalid_unique_address, storage_class)
+
+    with pytest.raises(Exception):
+        db0.fetch(invalid_uuid)
+
+
 def test_tag_and_find_immutable_instance(db0_fixture):
     obj_1 = MemoImmutableClass1(data="immutable data", value=42)
     db0.tags(obj_1).add("tag1", "tag2")
@@ -176,23 +421,28 @@ def test_prebound_immutable_nested_object_embeds_into_owner(db0_fixture):
     assert db0.uuid(inner) != db0.uuid(obj)
 
 
-@pytest.mark.skip(
-    reason="TODO: embedded immutable references can be created, but unreferencing by inner address is not implemented yet."
-)
 def test_regular_memo_can_reference_embedded_immutable_nested_object(db0_fixture):
     outer = MemoImmutableNestedHolder(name="referenced child", count=21, label="root")
     db0.tags(outer).add("keep-reference-source")
     inner = outer.nested
 
     holder = MemoSetReferenceHolder(inner)
+    db0.tags(holder).add("keep-regular-embedded-reference")
+    holder_id = db0.uuid(holder)
 
     assert db0.uuid(inner) != db0.uuid(outer)
     assert db0.uuid(holder)
 
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened = db0.fetch(holder_id)
+    assert reopened.payload.name == "referenced child"
+    assert reopened.payload.count == 21
+
 
-@pytest.mark.skip(
-    reason="TODO: embedded immutable references can be created, but unreferencing by inner address is not implemented yet."
-)
 def test_db0_collections_can_store_embedded_immutable_nested_object_reference(db0_fixture):
     outer = MemoImmutableNestedHolder(name="collection child", count=22, label="root")
     db0.tags(outer).add("keep-collection-source")
@@ -201,16 +451,30 @@ def test_db0_collections_can_store_embedded_immutable_nested_object_reference(db
     list_holder = MemoSetReferenceHolder(db0.list([inner]))
     set_holder = MemoSetReferenceHolder(db0.set([inner]))
     dict_holder = MemoSetReferenceHolder(db0.dict({"child": inner, inner: "value"}))
+    db0.tags(list_holder).add("keep-list-embedded-reference")
+    db0.tags(set_holder).add("keep-set-embedded-reference")
+    db0.tags(dict_holder).add("keep-dict-embedded-reference")
+    list_holder_id = db0.uuid(list_holder)
+    set_holder_id = db0.uuid(set_holder)
+    dict_holder_id = db0.uuid(dict_holder)
 
     assert db0.uuid(inner) != db0.uuid(outer)
     assert db0.uuid(list_holder)
     assert db0.uuid(set_holder)
     assert db0.uuid(dict_holder)
 
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    assert db0.fetch(list_holder_id).payload[0].name == "collection child"
+    assert next(iter(db0.fetch(set_holder_id).payload)).count == 22
+    reopened_dict = db0.fetch(dict_holder_id).payload
+    assert reopened_dict["child"].name == "collection child"
+    assert reopened_dict[reopened_dict["child"]] == "value"
+
 
-@pytest.mark.skip(
-    reason="TODO: embedded immutable references can be created, but unreferencing by inner address is not implemented yet."
-)
 def test_index_can_store_embedded_immutable_nested_object_reference(db0_fixture):
     outer = MemoImmutableNestedHolder(name="index child", count=23, label="root")
     db0.tags(outer).add("keep-index-source")
@@ -219,10 +483,24 @@ def test_index_can_store_embedded_immutable_nested_object_reference(db0_fixture)
 
     index.add(1, inner)
     index.flush()
+    holder = MemoSetReferenceHolder(index)
+    db0.tags(holder).add("keep-index-embedded-reference")
+    holder_id = db0.uuid(holder)
 
     assert db0.uuid(inner) != db0.uuid(outer)
     assert len(index) == 1
 
+    db0.commit()
+    db0.close()
+    db0.init(DB0_DIR)
+    db0.open("my-test-prefix", "rw")
+
+    reopened_index = db0.fetch(holder_id).payload
+    retrieved = list(reopened_index.select())
+    assert len(retrieved) == 1
+    assert retrieved[0].name == "index child"
+    assert retrieved[0].count == 23
+
 
 def test_read_embedded_tuple_field(db0_fixture):
     payload = tuple(f"alpha-{index}" for index in range(12)) + (7, b"bytes", None)