From 41281e8cc5ac156bc41c1128a3ca1ed663bbef19 Mon Sep 17 00:00:00 2001 From: Eric Cao Date: Sat, 7 Mar 2026 16:05:42 +0800 Subject: [PATCH 1/6] fix(webui): improve objects list performance for branches with uncommitted deletes When displaying objects on a branch, use the committed-only ref (branch@) instead of the full branch ref to list objects. This avoids scanning the staging area twice - once for objects.list and once for refs.changes. For branches with many uncommitted deletes, this significantly improves page load performance. The '@' modifier tells lakeFS to only return committed data, skipping the staging area scan entirely. Fixes #10008 --- webui/src/pages/repositories/repository/objects.jsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/webui/src/pages/repositories/repository/objects.jsx b/webui/src/pages/repositories/repository/objects.jsx index ca4a6f97d7d..768f28878e7 100644 --- a/webui/src/pages/repositories/repository/objects.jsx +++ b/webui/src/pages/repositories/repository/objects.jsx @@ -993,11 +993,14 @@ const TreeContainer = ({ ); } else { // Show all objects - return objects.list(repo.id, reference.id, path, after, config.pre_sign_support_ui); + // Use committed-only ref (branch@) for branches to avoid scanning staging area twice + // This significantly improves performance when there are many uncommitted deletes + const listRef = reference.type === RefTypeBranch ? reference.id + '@' : reference.id; + return objects.list(repo.id, listRef, path, after, config.pre_sign_support_ui); } // TODO: Review and remove this eslint-disable once dependencies are validated // eslint-disable-next-line react-hooks/exhaustive-deps - }, [repo.id, reference.id, path, after, refreshToken, showChangesOnly, internalRefresh, lastSeenPath, delimiter]); + }, [repo.id, reference.id, reference.type, path, after, refreshToken, showChangesOnly, internalRefresh, lastSeenPath, delimiter]); // Merge changes with objects for highlighting const mergedResults = React.useMemo( From 2109d867c1d16b10e5be19bb5930cd634d23689a Mon Sep 17 00:00:00 2001 From: Eric Cao Date: Thu, 12 Mar 2026 09:48:35 +0800 Subject: [PATCH 2/6] fix(webui): include added entries in mergeResults for committed-only listing When using committed-only ref (branch@) for objects.list to improve performance, uncommitted added files are not returned by the API. This fix updates mergeResults to also include 'added' type changes in the missingItems filter, ensuring newly uploaded files are shown in the objects list. Fixes the issue reported by @UdiBen: uncommitted additions no longer appear in the objects list when using branch@ for performance. --- webui/src/lib/components/repository/mergeResults.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webui/src/lib/components/repository/mergeResults.ts b/webui/src/lib/components/repository/mergeResults.ts index ae920e7a907..3f345df0bed 100644 --- a/webui/src/lib/components/repository/mergeResults.ts +++ b/webui/src/lib/components/repository/mergeResults.ts @@ -46,11 +46,12 @@ export function mergeResults( } }); - // Add missing items only for removed entries or deleted prefixes + // Add missing items for removed entries, added entries, and prefixes + // When using committed-only ref (branch@) for objects.list, added entries are not in the list // Avoid adding items that come after the last result path (both are sorted lexicographically) const lastResultPath = last(results)?.path; const missingItems = changesData.results - .filter((change) => change.type === 'removed' || change.path_type === 'common_prefix') + .filter((change) => change.type === 'removed' || change.type === 'added' || change.path_type === 'common_prefix') .filter((change) => lastResultPath && change.path <= lastResultPath) .filter((change) => !results.find((result) => result.path === change.path)); From 92d190264b640b2e84cb28edc77770a829e34952 Mon Sep 17 00:00:00 2001 From: Udi Date: Thu, 19 Mar 2026 22:07:22 +0200 Subject: [PATCH 3/6] fix(webui): show uncommitted files when no committed objects exist When results from committed-only listing (branch@) are empty, the mergeResults filter `lastResultPath && ...` dropped all changes since lastResultPath was undefined. This prevented uploaded files from appearing in fresh repos with no commits. Co-Authored-By: Claude Opus 4.6 (1M context) minor --- .../lib/components/repository/mergeResults.test.ts | 8 +++++--- webui/src/lib/components/repository/mergeResults.ts | 6 ++++-- webui/src/pages/repositories/repository/objects.jsx | 13 ++++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/webui/src/lib/components/repository/mergeResults.test.ts b/webui/src/lib/components/repository/mergeResults.test.ts index 94ee46e0ff3..f8cbcf01e6d 100644 --- a/webui/src/lib/components/repository/mergeResults.test.ts +++ b/webui/src/lib/components/repository/mergeResults.test.ts @@ -155,7 +155,7 @@ describe('mergeResults', () => { }); describe('missing items (removed files/directories)', () => { - it('does not add removed items when results is empty', () => { + it('adds changed items when results is empty (e.g. fresh repo with only uncommitted uploads)', () => { const results: Entry[] = []; const changesData: ChangesData = { results: [{ path: 'deleted.txt', type: 'removed', path_type: 'object' }], @@ -163,8 +163,10 @@ describe('mergeResults', () => { const merged = mergeResults(results, changesData, false); - // When results is empty, we don't know the range, so no missing items are added - expect(merged).toEqual([]); + // When results is empty (no committed objects), changes should still appear + expect(merged).toEqual([ + { path: 'deleted.txt', path_type: 'object', type: 'removed', diff_type: 'removed' }, + ]); }); it('adds removed files that are missing from results', () => { diff --git a/webui/src/lib/components/repository/mergeResults.ts b/webui/src/lib/components/repository/mergeResults.ts index 3f345df0bed..861b228756a 100644 --- a/webui/src/lib/components/repository/mergeResults.ts +++ b/webui/src/lib/components/repository/mergeResults.ts @@ -51,8 +51,10 @@ export function mergeResults( // Avoid adding items that come after the last result path (both are sorted lexicographically) const lastResultPath = last(results)?.path; const missingItems = changesData.results - .filter((change) => change.type === 'removed' || change.type === 'added' || change.path_type === 'common_prefix') - .filter((change) => lastResultPath && change.path <= lastResultPath) + .filter( + (change) => change.type === 'removed' || change.type === 'added' || change.path_type === 'common_prefix', + ) + .filter((change) => !lastResultPath || change.path <= lastResultPath) .filter((change) => !results.find((result) => result.path === change.path)); // Merge regular results with change info diff --git a/webui/src/pages/repositories/repository/objects.jsx b/webui/src/pages/repositories/repository/objects.jsx index 768f28878e7..ef1a925ca4e 100644 --- a/webui/src/pages/repositories/repository/objects.jsx +++ b/webui/src/pages/repositories/repository/objects.jsx @@ -1000,7 +1000,18 @@ const TreeContainer = ({ } // TODO: Review and remove this eslint-disable once dependencies are validated // eslint-disable-next-line react-hooks/exhaustive-deps - }, [repo.id, reference.id, reference.type, path, after, refreshToken, showChangesOnly, internalRefresh, lastSeenPath, delimiter]); + }, [ + repo.id, + reference.id, + reference.type, + path, + after, + refreshToken, + showChangesOnly, + internalRefresh, + lastSeenPath, + delimiter, + ]); // Merge changes with objects for highlighting const mergedResults = React.useMemo( From ef723affe125dc08bd78236542c12cd44b4de831 Mon Sep 17 00:00:00 2001 From: Udi Date: Fri, 20 Mar 2026 00:49:51 +0200 Subject: [PATCH 4/6] fix(webui): show added entries that sort after committed objects When uploaded files sort lexicographically after all committed objects (e.g. test-upload.txt > lakes.parquet), the lastResultPath filter in mergeResults dropped them. Now the filter is only applied when there are more pages (hasMore), so on the last page all changes are shown. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../repository/mergeResults.test.ts | 25 ++++++++++++++++--- .../lib/components/repository/mergeResults.ts | 7 ++++-- .../pages/repositories/repository/objects.jsx | 4 +-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/webui/src/lib/components/repository/mergeResults.test.ts b/webui/src/lib/components/repository/mergeResults.test.ts index f8cbcf01e6d..6556c4ce5d8 100644 --- a/webui/src/lib/components/repository/mergeResults.test.ts +++ b/webui/src/lib/components/repository/mergeResults.test.ts @@ -203,7 +203,7 @@ describe('mergeResults', () => { }); }); - it('does not add removed items that come after last result path', () => { + it('does not add removed items that come after last result path when there are more pages', () => { const results: Entry[] = [ { path: 'a.txt', path_type: 'object' }, { path: 'b.txt', path_type: 'object' }, @@ -212,13 +212,32 @@ describe('mergeResults', () => { results: [{ path: 'z.txt', type: 'removed', path_type: 'object' }], }; - const merged = mergeResults(results, changesData, false); + const merged = mergeResults(results, changesData, false, true); - // z.txt should not be added because it comes after b.txt lexicographically + // z.txt should not be added because it comes after b.txt and there are more pages expect(merged).toHaveLength(2); expect(merged.find((r) => r.path === 'z.txt')).toBeUndefined(); }); + it('adds items after last result path on the last page', () => { + const results: Entry[] = [ + { path: 'a.txt', path_type: 'object' }, + { path: 'b.txt', path_type: 'object' }, + ]; + const changesData: ChangesData = { + results: [{ path: 'z.txt', type: 'added', path_type: 'object' }], + }; + + const merged = mergeResults(results, changesData, false, false); + + // z.txt should be added because this is the last page + expect(merged).toHaveLength(3); + expect(merged.find((r) => r.path === 'z.txt')).toMatchObject({ + path: 'z.txt', + diff_type: 'added', + }); + }); + it('does not duplicate items that exist in both results and changes', () => { const results: Entry[] = [{ path: 'file.txt', path_type: 'object' }]; const changesData: ChangesData = { diff --git a/webui/src/lib/components/repository/mergeResults.ts b/webui/src/lib/components/repository/mergeResults.ts index 861b228756a..ba37403dbae 100644 --- a/webui/src/lib/components/repository/mergeResults.ts +++ b/webui/src/lib/components/repository/mergeResults.ts @@ -8,12 +8,14 @@ import { compareLexicographically } from '../../utils'; * @param results - Array of object entries from the main listing * @param changesData - Changes data containing results array with change information * @param showChangesOnly - Whether to show only changes (if true, no merging needed) + * @param hasMore - Whether there are more pages of results after the current one * @returns Merged and sorted results with diff_type annotations */ export function mergeResults( results: Entry[] | undefined | null, changesData: ChangesData | undefined | null, showChangesOnly = false, + hasMore = false, ): EntryWithDiff[] { if (showChangesOnly || !results || !changesData?.results) { // Ensure regular results are also sorted lexicographically @@ -48,13 +50,14 @@ export function mergeResults( // Add missing items for removed entries, added entries, and prefixes // When using committed-only ref (branch@) for objects.list, added entries are not in the list - // Avoid adding items that come after the last result path (both are sorted lexicographically) + // On paginated results, only add items within the current page range to avoid duplicates + // On the last page (no more results), include all remaining changes const lastResultPath = last(results)?.path; const missingItems = changesData.results .filter( (change) => change.type === 'removed' || change.type === 'added' || change.path_type === 'common_prefix', ) - .filter((change) => !lastResultPath || change.path <= lastResultPath) + .filter((change) => !hasMore || !lastResultPath || change.path <= lastResultPath) .filter((change) => !results.find((result) => result.path === change.path)); // Merge regular results with change info diff --git a/webui/src/pages/repositories/repository/objects.jsx b/webui/src/pages/repositories/repository/objects.jsx index ef1a925ca4e..57075cfa392 100644 --- a/webui/src/pages/repositories/repository/objects.jsx +++ b/webui/src/pages/repositories/repository/objects.jsx @@ -1015,8 +1015,8 @@ const TreeContainer = ({ // Merge changes with objects for highlighting const mergedResults = React.useMemo( - () => mergeResults(results, changesData, showChangesOnly), - [results, changesData, showChangesOnly], + () => mergeResults(results, changesData, showChangesOnly, !!nextPage), + [results, changesData, showChangesOnly, nextPage], ); const initialState = { From aa9368cce6ac4ea21687543bbeeea4cf76d360b3 Mon Sep 17 00:00:00 2001 From: Udi Date: Fri, 20 Mar 2026 01:14:48 +0200 Subject: [PATCH 5/6] fix(webui): only bypass page-range filter for added entries, not removed Removed entries exist in committed data (branch@) and appear on some page of the objects list, so the lastResultPath filter correctly prevents duplicating them. Added entries never appear in committed results, so they need the hasMore bypass to show on the last page. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/repository/mergeResults.test.ts | 17 ++++++++++++++--- .../lib/components/repository/mergeResults.ts | 13 +++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/webui/src/lib/components/repository/mergeResults.test.ts b/webui/src/lib/components/repository/mergeResults.test.ts index 6556c4ce5d8..902adb21751 100644 --- a/webui/src/lib/components/repository/mergeResults.test.ts +++ b/webui/src/lib/components/repository/mergeResults.test.ts @@ -155,7 +155,7 @@ describe('mergeResults', () => { }); describe('missing items (removed files/directories)', () => { - it('adds changed items when results is empty (e.g. fresh repo with only uncommitted uploads)', () => { + it('does not add removed items when results is empty', () => { const results: Entry[] = []; const changesData: ChangesData = { results: [{ path: 'deleted.txt', type: 'removed', path_type: 'object' }], @@ -163,9 +163,20 @@ describe('mergeResults', () => { const merged = mergeResults(results, changesData, false); - // When results is empty (no committed objects), changes should still appear + // Removed entries exist in committed data, so with empty results there's nothing to show + expect(merged).toEqual([]); + }); + + it('adds added items when results is empty (e.g. fresh repo with only uncommitted uploads)', () => { + const results: Entry[] = []; + const changesData: ChangesData = { + results: [{ path: 'uploaded.txt', type: 'added', path_type: 'object' }], + }; + + const merged = mergeResults(results, changesData, false); + expect(merged).toEqual([ - { path: 'deleted.txt', path_type: 'object', type: 'removed', diff_type: 'removed' }, + { path: 'uploaded.txt', path_type: 'object', type: 'added', diff_type: 'added' }, ]); }); diff --git a/webui/src/lib/components/repository/mergeResults.ts b/webui/src/lib/components/repository/mergeResults.ts index ba37403dbae..3fdcb640b37 100644 --- a/webui/src/lib/components/repository/mergeResults.ts +++ b/webui/src/lib/components/repository/mergeResults.ts @@ -51,13 +51,22 @@ export function mergeResults( // Add missing items for removed entries, added entries, and prefixes // When using committed-only ref (branch@) for objects.list, added entries are not in the list // On paginated results, only add items within the current page range to avoid duplicates - // On the last page (no more results), include all remaining changes const lastResultPath = last(results)?.path; + const inPageRange = (path: string) => lastResultPath && path <= lastResultPath; const missingItems = changesData.results .filter( (change) => change.type === 'removed' || change.type === 'added' || change.path_type === 'common_prefix', ) - .filter((change) => !hasMore || !lastResultPath || change.path <= lastResultPath) + .filter((change) => { + if (change.type === 'added') { + // Added entries never appear in committed results (branch@), + // so include them on the last page even if beyond lastResultPath + return inPageRange(change.path) || !hasMore; + } + // Removed/changed entries exist in committed results on some page, + // only show within the current page range + return inPageRange(change.path); + }) .filter((change) => !results.find((result) => result.path === change.path)); // Merge regular results with change info From 2c11a9f15e64fe574fd015c3c6f01e51289ddb87 Mon Sep 17 00:00:00 2001 From: Udi Date: Fri, 20 Mar 2026 01:32:38 +0200 Subject: [PATCH 6/6] Fix --- .../repository/mergeResults.test.ts | 107 ++++++++++++------ 1 file changed, 71 insertions(+), 36 deletions(-) diff --git a/webui/src/lib/components/repository/mergeResults.test.ts b/webui/src/lib/components/repository/mergeResults.test.ts index 902adb21751..f86314442d6 100644 --- a/webui/src/lib/components/repository/mergeResults.test.ts +++ b/webui/src/lib/components/repository/mergeResults.test.ts @@ -163,23 +163,10 @@ describe('mergeResults', () => { const merged = mergeResults(results, changesData, false); - // Removed entries exist in committed data, so with empty results there's nothing to show + // When results is empty, we don't know the range, so no missing items are added expect(merged).toEqual([]); }); - it('adds added items when results is empty (e.g. fresh repo with only uncommitted uploads)', () => { - const results: Entry[] = []; - const changesData: ChangesData = { - results: [{ path: 'uploaded.txt', type: 'added', path_type: 'object' }], - }; - - const merged = mergeResults(results, changesData, false); - - expect(merged).toEqual([ - { path: 'uploaded.txt', path_type: 'object', type: 'added', diff_type: 'added' }, - ]); - }); - it('adds removed files that are missing from results', () => { const results: Entry[] = [{ path: 'file1.txt', path_type: 'object' }]; const changesData: ChangesData = { @@ -214,7 +201,7 @@ describe('mergeResults', () => { }); }); - it('does not add removed items that come after last result path when there are more pages', () => { + it('does not add removed items that come after last result path', () => { const results: Entry[] = [ { path: 'a.txt', path_type: 'object' }, { path: 'b.txt', path_type: 'object' }, @@ -223,32 +210,13 @@ describe('mergeResults', () => { results: [{ path: 'z.txt', type: 'removed', path_type: 'object' }], }; - const merged = mergeResults(results, changesData, false, true); + const merged = mergeResults(results, changesData, false); - // z.txt should not be added because it comes after b.txt and there are more pages + // z.txt should not be added because it comes after b.txt lexicographically expect(merged).toHaveLength(2); expect(merged.find((r) => r.path === 'z.txt')).toBeUndefined(); }); - it('adds items after last result path on the last page', () => { - const results: Entry[] = [ - { path: 'a.txt', path_type: 'object' }, - { path: 'b.txt', path_type: 'object' }, - ]; - const changesData: ChangesData = { - results: [{ path: 'z.txt', type: 'added', path_type: 'object' }], - }; - - const merged = mergeResults(results, changesData, false, false); - - // z.txt should be added because this is the last page - expect(merged).toHaveLength(3); - expect(merged.find((r) => r.path === 'z.txt')).toMatchObject({ - path: 'z.txt', - diff_type: 'added', - }); - }); - it('does not duplicate items that exist in both results and changes', () => { const results: Entry[] = [{ path: 'file.txt', path_type: 'object' }]; const changesData: ChangesData = { @@ -286,6 +254,73 @@ describe('mergeResults', () => { }); }); + describe('added entries with committed-only listing (branch@)', () => { + it('includes added entries when committed results are empty (fresh repo upload)', () => { + const results: Entry[] = []; + const changesData: ChangesData = { + results: [{ path: 'uploaded.txt', type: 'added', path_type: 'object' }], + }; + + const merged = mergeResults(results, changesData, false); + + expect(merged).toEqual([ + { path: 'uploaded.txt', path_type: 'object', type: 'added', diff_type: 'added' }, + ]); + }); + + it('includes added entries that sort after last committed object on last page', () => { + const results: Entry[] = [ + { path: 'a.txt', path_type: 'object' }, + { path: 'lakes.parquet', path_type: 'object' }, + ]; + const changesData: ChangesData = { + results: [{ path: 'test-upload.txt', type: 'added', path_type: 'object' }], + }; + + // hasMore=false (last page) + const merged = mergeResults(results, changesData, false, false); + + expect(merged).toHaveLength(3); + expect(merged.find((r) => r.path === 'test-upload.txt')).toMatchObject({ + path: 'test-upload.txt', + diff_type: 'added', + }); + }); + + it('excludes added entries beyond page range when there are more pages', () => { + const results: Entry[] = [ + { path: 'a.txt', path_type: 'object' }, + { path: 'b.txt', path_type: 'object' }, + ]; + const changesData: ChangesData = { + results: [{ path: 'z.txt', type: 'added', path_type: 'object' }], + }; + + // hasMore=true (not last page) + const merged = mergeResults(results, changesData, false, true); + + expect(merged).toHaveLength(2); + expect(merged.find((r) => r.path === 'z.txt')).toBeUndefined(); + }); + + it('includes added entries within page range regardless of hasMore', () => { + const results: Entry[] = [ + { path: 'a.txt', path_type: 'object' }, + { path: 'z.txt', path_type: 'object' }, + ]; + const changesData: ChangesData = { + results: [{ path: 'new-file.txt', type: 'added', path_type: 'object' }], + }; + + const merged = mergeResults(results, changesData, false, true); + + expect(merged).toHaveLength(3); + expect(merged.find((r) => r.path === 'new-file.txt')).toMatchObject({ + diff_type: 'added', + }); + }); + }); + describe('sorting', () => { it('sorts merged results lexicographically', () => { const results: Entry[] = [