diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c86ec77b26f1..e7e41033105c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -291,6 +291,8 @@ Optimizations
 
 * GITHUB#15779: Improve BytesRefHash.add performance by optimize rehash operation (tyronecai)
 
+* GITHUB#15896: Skip heavy TreeSet opts for the first group in SearchGroup#merge (Binlong Gao)
+
 * GITHUB#15886: Cache frozen FieldType to skip redundant schema validation. (Tim Brooks)
 
 Bug Fixes
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
index a2830e52734a..c05b0b11265c 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/SearchGroup.java
@@ -248,13 +248,17 @@ private void updateNextGroup(int topN, ShardIter<T> shard) {
           // System.out.println("      competes=" + competes);
 
           if (competes) {
-            // Group's sort changed -- remove & re-insert
-            if (mergedGroup.inQueue) {
+            // Group's sort changed -- remove & re-insert, update first group in place for
+            // efficiency
+            boolean skipHeavyOps = queue.first() == mergedGroup;
+            if (mergedGroup.inQueue && !skipHeavyOps) {
               queue.remove(mergedGroup);
             }
             mergedGroup.topValues = group.sortValues;
             mergedGroup.minShardIndex = shard.shardIndex;
-            queue.add(mergedGroup);
+            if (!skipHeavyOps) {
+              queue.add(mergedGroup);
+            }
             mergedGroup.inQueue = true;
           }
         }