From db5d3f19d51a4b20c5f33b3c1d3d2565caa2eae7 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Wed, 29 Jul 2026 03:35:25 +0000 Subject: [PATCH 1/4] fix(hns): cache root bucket properly in _get_dirs_and_update_cache --- gcsfs/core.py | 22 +++++++++++--------- gcsfs/tests/integration/test_extended_hns.py | 15 +++++++++++++ gcsfs/tests/test_core.py | 22 ++++++++++---------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index 1bae97e60..1c1c8c1cf 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -1902,21 +1902,22 @@ def _get_dirs_and_update_cache(self, path, objects, prefix="", update_cache=True while parent: dir_key = self.split_path(parent)[1] - if not dir_key or len(parent) < len(path.rstrip("/")): + if len(parent) < len(path.rstrip("/")): break if prefix and not parent.startswith(full_prefix): # If this parent doesn't match the prefix, neither will its parents. break - dirs[parent] = { - "Key": dir_key, - "Size": 0, - "name": parent, - "StorageClass": "DIRECTORY", - "type": "directory", - "size": 0, - } + if dir_key: + dirs[parent] = { + "Key": dir_key, + "Size": 0, + "name": parent, + "StorageClass": "DIRECTORY", + "type": "directory", + "size": 0, + } if not prefix and update_cache: listing = cache_entries.setdefault(parent, {}) @@ -1924,7 +1925,8 @@ def _get_dirs_and_update_cache(self, path, objects, prefix="", update_cache=True if name not in listing: listing[name] = previous - previous = dirs[parent] + if parent in dirs: + previous = dirs[parent] parent = self._parent(parent) if not prefix and update_cache: cache_entries_list = {k: list(v.values()) for k, v in cache_entries.items()} diff --git a/gcsfs/tests/integration/test_extended_hns.py b/gcsfs/tests/integration/test_extended_hns.py index 515dd7181..bc8a05fc7 100644 --- a/gcsfs/tests/integration/test_extended_hns.py +++ b/gcsfs/tests/integration/test_extended_hns.py @@ -1328,6 +1328,21 @@ def test_find_updates_dircache_without_prefix( assert test_structure["nested_dir"] in dir_with_files_listing # Check content of the 'nested_dir' cache + + @pytest.mark.parametrize("withdirs_param", [True, False]) + def test_find_updates_dircache_for_root_bucket( + self, gcs_hns, test_structure, withdirs_param + ): + """Test that find() populates the dircache for the root bucket itself.""" + root_bucket = TEST_HNS_BUCKET + gcs_hns.invalidate_cache() + assert not gcs_hns.dircache + + # Run find on the root bucket to populate the cache + gcs_hns.find(root_bucket, withdirs=withdirs_param) + + # Verify that the cache is now populated for the root bucket + assert root_bucket in gcs_hns.dircache nested_dir_listing = { d["name"] for d in gcs_hns.dircache[test_structure["nested_dir"]] } diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index ae42c8d66..712e4a1ed 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -1946,10 +1946,10 @@ def test_attrs(gcs): assert gcs.getxattr(a, "something") == "not" -def test_request_user_project(gcs_factory): - gcs = gcs_factory(requester_pays=True, project=TEST_PROJECT) +def test_request_user_project(gcs, gcs_factory): + gcs_inst = gcs_factory(requester_pays=True, project=TEST_PROJECT) # test directly against `_call` to inspect the result - r = gcs.call( + r = gcs_inst.call( "GET", "b/{}/o", TEST_BUCKET, @@ -1963,11 +1963,11 @@ def test_request_user_project(gcs_factory): assert result["userProject"] == [TEST_PROJECT] -def test_request_user_project_string(gcs_factory): - gcs = gcs_factory(requester_pays=TEST_PROJECT) - assert gcs.requester_pays == TEST_PROJECT +def test_request_user_project_string(gcs, gcs_factory): + gcs_inst = gcs_factory(requester_pays=TEST_PROJECT) + assert gcs_inst.requester_pays == TEST_PROJECT # test directly against `_call` to inspect the result - r = gcs.call( + r = gcs_inst.call( "GET", "b/{}/o", TEST_BUCKET, @@ -1981,10 +1981,10 @@ def test_request_user_project_string(gcs_factory): assert result["userProject"] == [TEST_PROJECT] -def test_request_header(gcs_factory): - gcs = gcs_factory(requester_pays=True) +def test_request_header(gcs, gcs_factory): + gcs_inst = gcs_factory(requester_pays=True) # test directly against `_call` to inspect the result - r = gcs.call( + r = gcs_inst.call( "GET", "b/{}/o", TEST_BUCKET, @@ -2025,7 +2025,7 @@ def test_requester_pays_fails_without_user_project(requester_pays_bucket, gcs_fa fs.ls(requester_pays_bucket) -def test_fs_requester_pays_on_bucket_without_requester_pays(gcs_factory): +def test_fs_requester_pays_on_bucket_without_requester_pays(gcs, gcs_factory): """Test that metadata and data operations work when fs has requester_pays=True but the bucket does not have requester-pays enabled.""" fs = gcs_factory(requester_pays=True) From 0d7700a700084db9b176d32bff28773f27c94e2b Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 31 Jul 2026 04:46:35 +0000 Subject: [PATCH 2/4] test(hns): fix test_find_updates_dircache_for_root_bucket placement --- gcsfs/tests/integration/test_extended_hns.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gcsfs/tests/integration/test_extended_hns.py b/gcsfs/tests/integration/test_extended_hns.py index bc8a05fc7..bc7431a87 100644 --- a/gcsfs/tests/integration/test_extended_hns.py +++ b/gcsfs/tests/integration/test_extended_hns.py @@ -1328,6 +1328,16 @@ def test_find_updates_dircache_without_prefix( assert test_structure["nested_dir"] in dir_with_files_listing # Check content of the 'nested_dir' cache + nested_dir_listing = { + d["name"] for d in gcs_hns.dircache[test_structure["nested_dir"]] + } + assert test_structure["nested_file"] in nested_dir_listing + + # Check content of the 'empty_dir' cache + empty_dir_listing = { + d["name"] for d in gcs_hns.dircache[test_structure["empty_dir"]] + } + assert not empty_dir_listing @pytest.mark.parametrize("withdirs_param", [True, False]) def test_find_updates_dircache_for_root_bucket( @@ -1343,16 +1353,6 @@ def test_find_updates_dircache_for_root_bucket( # Verify that the cache is now populated for the root bucket assert root_bucket in gcs_hns.dircache - nested_dir_listing = { - d["name"] for d in gcs_hns.dircache[test_structure["nested_dir"]] - } - assert test_structure["nested_file"] in nested_dir_listing - - # Check content of the 'empty_dir' cache - empty_dir_listing = { - d["name"] for d in gcs_hns.dircache[test_structure["empty_dir"]] - } - assert not empty_dir_listing def test_find_maxdepth_updates_cache(self, gcs_hns, test_structure): """Test that find with maxdepth updates cache for deeper objects.""" From c7069ab0b51709213f047753892622a24ef20767 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 31 Jul 2026 11:28:17 +0000 Subject: [PATCH 3/4] test(hns): assert root bucket dircache listing contains base_dir --- gcsfs/tests/integration/test_extended_hns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gcsfs/tests/integration/test_extended_hns.py b/gcsfs/tests/integration/test_extended_hns.py index bc7431a87..5d0ef4e17 100644 --- a/gcsfs/tests/integration/test_extended_hns.py +++ b/gcsfs/tests/integration/test_extended_hns.py @@ -1353,6 +1353,10 @@ def test_find_updates_dircache_for_root_bucket( # Verify that the cache is now populated for the root bucket assert root_bucket in gcs_hns.dircache + root_bucket_listing = { + d["name"].rstrip("/") for d in gcs_hns.dircache[root_bucket] + } + assert test_structure["base_dir"] in root_bucket_listing def test_find_maxdepth_updates_cache(self, gcs_hns, test_structure): """Test that find with maxdepth updates cache for deeper objects.""" From 4cc5ff0545ebd119894d22fac0821c05b7ce63f4 Mon Sep 17 00:00:00 2001 From: yuxin00j Date: Fri, 31 Jul 2026 11:55:19 +0000 Subject: [PATCH 4/4] Revert unnecessary changes to test_core.py --- gcsfs/tests/test_core.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index 712e4a1ed..ae42c8d66 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -1946,10 +1946,10 @@ def test_attrs(gcs): assert gcs.getxattr(a, "something") == "not" -def test_request_user_project(gcs, gcs_factory): - gcs_inst = gcs_factory(requester_pays=True, project=TEST_PROJECT) +def test_request_user_project(gcs_factory): + gcs = gcs_factory(requester_pays=True, project=TEST_PROJECT) # test directly against `_call` to inspect the result - r = gcs_inst.call( + r = gcs.call( "GET", "b/{}/o", TEST_BUCKET, @@ -1963,11 +1963,11 @@ def test_request_user_project(gcs, gcs_factory): assert result["userProject"] == [TEST_PROJECT] -def test_request_user_project_string(gcs, gcs_factory): - gcs_inst = gcs_factory(requester_pays=TEST_PROJECT) - assert gcs_inst.requester_pays == TEST_PROJECT +def test_request_user_project_string(gcs_factory): + gcs = gcs_factory(requester_pays=TEST_PROJECT) + assert gcs.requester_pays == TEST_PROJECT # test directly against `_call` to inspect the result - r = gcs_inst.call( + r = gcs.call( "GET", "b/{}/o", TEST_BUCKET, @@ -1981,10 +1981,10 @@ def test_request_user_project_string(gcs, gcs_factory): assert result["userProject"] == [TEST_PROJECT] -def test_request_header(gcs, gcs_factory): - gcs_inst = gcs_factory(requester_pays=True) +def test_request_header(gcs_factory): + gcs = gcs_factory(requester_pays=True) # test directly against `_call` to inspect the result - r = gcs_inst.call( + r = gcs.call( "GET", "b/{}/o", TEST_BUCKET, @@ -2025,7 +2025,7 @@ def test_requester_pays_fails_without_user_project(requester_pays_bucket, gcs_fa fs.ls(requester_pays_bucket) -def test_fs_requester_pays_on_bucket_without_requester_pays(gcs, gcs_factory): +def test_fs_requester_pays_on_bucket_without_requester_pays(gcs_factory): """Test that metadata and data operations work when fs has requester_pays=True but the bucket does not have requester-pays enabled.""" fs = gcs_factory(requester_pays=True)