Improve performance of StringUtils.replaceEach.

.changes/next-release/bugfix-AWSSDKforJavav2-e4b6d09.json

@@ -0,0 +1,6 @@
+{
+    "type": "bugfix",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Improve the performance of StringUtils.replaceEach for inputs with many matches."
+}

utils/src/main/java/software/amazon/awssdk/utils/StringUtils.java

@@ -748,7 +748,7 @@ public static String replaceEach(String text, String[] searchList, String[] repl
         // mchyzer Performance note: This creates very few new objects (one major goal)
         // let me know if there are performance requests, we can create a harness to measure
 
-        if (isEmpty(text)) {
+        if (isEmpty(text) || searchList == null || replacementList == null) {
             return text;
         }
 
@@ -763,31 +763,29 @@ public static String replaceEach(String text, String[] searchList, String[] repl
                                                + replacementLength);
         }
 
-        // keep track of which still have matches
-        boolean[] noMoreMatchesForReplIndex = new boolean[searchLength];
+        // Cache, for each search string, the index of its next match at or after the current position.
+        // -1 means the search string is skipped (empty or null replacement) or has no more matches.
+        // A search string is only searched again once the current position has moved past its cached
+        // match, which keeps the total work linear in the length of the text.
+        int[] nextMatch = new int[searchLength];
 
         // index on index that the match was found
         int textIndex = -1;
         int replaceIndex = -1;
-        int tempIndex;
 
         // index of replace array that will replace the search string found
-        // NOTE: logic duplicated below START
         for (int i = 0; i < searchLength; i++) {
-            if (noMoreMatchesForReplIndex[i] || isEmpty(searchList[i]) || replacementList[i] == null) {
+            if (isEmpty(searchList[i]) || replacementList[i] == null) {
+                nextMatch[i] = -1;
                 continue;
             }
-            tempIndex = text.indexOf(searchList[i]);
+            nextMatch[i] = text.indexOf(searchList[i]);
 
-            // see if we need to keep searching for this
-            if (tempIndex == -1) {
-                noMoreMatchesForReplIndex[i] = true;
-            } else if (textIndex == -1 || tempIndex < textIndex) {
-                textIndex = tempIndex;
+            if (nextMatch[i] != -1 && (textIndex == -1 || nextMatch[i] < textIndex)) {
+                textIndex = nextMatch[i];
                 replaceIndex = i;
             }
         }
-        // NOTE: logic mostly below END
 
         // no search strings found, we are done
         if (textIndex == -1) {
@@ -816,39 +814,29 @@ public static String replaceEach(String text, String[] searchList, String[] repl
 
         while (textIndex != -1) {
 
-            for (int i = start; i < textIndex; i++) {
-                buf.append(text.charAt(i));
-            }
+            buf.append(text, start, textIndex);
             buf.append(replacementList[replaceIndex]);
 
             start = textIndex + searchList[replaceIndex].length();
 
             textIndex = -1;
             replaceIndex = -1;
             // find the next earliest match
-            // NOTE: logic mostly duplicated above START
             for (int i = 0; i < searchLength; i++) {
-                if (noMoreMatchesForReplIndex[i] || searchList[i] == null ||
-                    searchList[i].isEmpty() || replacementList[i] == null) {
+                if (nextMatch[i] == -1) {
                     continue;
                 }
-                tempIndex = text.indexOf(searchList[i], start);
-
-                // see if we need to keep searching for this
-                if (tempIndex == -1) {
-                    noMoreMatchesForReplIndex[i] = true;
-                } else if (textIndex == -1 || tempIndex < textIndex) {
-                    textIndex = tempIndex;
+                if (nextMatch[i] < start) {
+                    // the cached match was consumed or overlapped by the last replacement
+                    nextMatch[i] = text.indexOf(searchList[i], start);
+                }
+                if (nextMatch[i] != -1 && (textIndex == -1 || nextMatch[i] < textIndex)) {
+                    textIndex = nextMatch[i];
                     replaceIndex = i;
                 }
             }
-            // NOTE: logic duplicated above END
-
-        }
-        int textLength = text.length();
-        for (int i = start; i < textLength; i++) {
-            buf.append(text.charAt(i));
         }
+        buf.append(text, start, text.length());
         return buf.toString();
     }
 

utils/src/test/java/software/amazon/awssdk/utils/StringUtilsTest.java

@@ -18,9 +18,11 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static software.amazon.awssdk.utils.StringUtils.replacePrefixIgnoreCase;
 
+import java.util.Random;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
@@ -177,4 +179,187 @@ public void repeat_negativeCount_shouldThrowIae() {
     public void repeat_maxCount_shouldThrowOom() {
         StringUtils.repeat("a", Integer.MAX_VALUE);
     }
+
+    @Test
+    public void replaceEach_javadocExamples() {
+        assertNull(StringUtils.replaceEach(null, new String[] {"a"}, new String[] {"b"}));
+        assertEquals("", StringUtils.replaceEach("", new String[] {"a"}, new String[] {"b"}));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[0], new String[0]));
+        assertEquals("aba", StringUtils.replaceEach("aba", null, null));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[0], null));
+        assertEquals("aba", StringUtils.replaceEach("aba", null, new String[0]));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[] {"a"}, null));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[] {null}, new String[] {"a"}));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[] {"a"}, new String[] {null}));
+        assertEquals("aba", StringUtils.replaceEach("aba", new String[] {""}, new String[] {"x"}));
+        assertEquals("b", StringUtils.replaceEach("aba", new String[] {"a"}, new String[] {""}));
+        assertEquals("wcte", StringUtils.replaceEach("abcde", new String[] {"ab", "d"}, new String[] {"w", "t"}));
+        assertEquals("dcte", StringUtils.replaceEach("abcde", new String[] {"ab", "d"}, new String[] {"d", "t"}));
+        assertEquals("xbx", StringUtils.replaceEach("aba", new String[] {null, "a"}, new String[] {"z", "x"}));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void replaceEach_lengthMismatch_throws() {
+        StringUtils.replaceEach("abc", new String[] {"a", "b"}, new String[] {"x"});
+    }
+
+    @Test
+    public void replaceEach_overlappingAndTiedMatches() {
+        assertEquals("XY", StringUtils.replaceEach("aaa", new String[] {"aa", "a"}, new String[] {"X", "Y"}));
+        assertEquals("22", StringUtils.replaceEach("abab", new String[] {"ba", "ab"}, new String[] {"1", "2"}));
+        assertEquals("1c", StringUtils.replaceEach("abc", new String[] {"ab", "a"}, new String[] {"1", "2"}));
+        assertEquals("2bc", StringUtils.replaceEach("abc", new String[] {"a", "ab"}, new String[] {"2", "1"}));
+        assertEquals("1b", StringUtils.replaceEach("xab", new String[] {"xa", "ab"}, new String[] {"1", "2"}));
+    }
+
+    @Test
+    public void replaceEach_randomInputs_matchesReferenceImplementation() {
+        // Short texts over a two-letter alphabet, which give many overlaps and ties.
+        assertRandomInputsMatchReference(new Random(42), new String[] {"a", "b"}, new String[] {"X", "Y", ""},
+                                         12, 4, 3, 10);
+        // Longer texts with URL-encoding characters and replacements of different lengths.
+        assertRandomInputsMatchReference(new Random(7), new String[] {"a", "b", "c", "%", "7", "E"},
+                                         new String[] {"X", "YY", "", "%20", "a"}, 40, 7, 4, 20);
+    }
+
+    private static void assertRandomInputsMatchReference(Random random, String[] alphabet, String[] replacements,
+                                                         int maxTextLength, int maxNeedles, int maxNeedleLength,
+                                                         int nullOneIn) {
+        for (int iteration = 0; iteration < 20_000; iteration++) {
+            String text = randomString(random, alphabet, random.nextInt(maxTextLength));
+            int needles = random.nextInt(maxNeedles);
+            String[] search = new String[needles];
+            String[] replace = new String[needles];
+            for (int i = 0; i < needles; i++) {
+                search[i] = random.nextInt(nullOneIn) == 0 ? null
+                                                           : randomString(random, alphabet, random.nextInt(maxNeedleLength));
+                replace[i] = random.nextInt(nullOneIn) == 0 ? null : replacements[random.nextInt(replacements.length)];
+            }
+            assertEquals("text=" + text, referenceReplaceEach(text, search, replace),
+                         StringUtils.replaceEach(text, search, replace));
+        }
+    }
+
+    @Test
+    public void replaceEach_noMatches_returnsSameInstance() {
+        String text = "hello world";
+        assertSame(text, StringUtils.replaceEach(text, new String[] {"x", "yz"}, new String[] {"1", "2"}));
+    }
+
+    @Test
+    public void replaceEach_matchesAtStartAndEnd() {
+        assertEquals("1bc2", StringUtils.replaceEach("abcd", new String[] {"a", "d"}, new String[] {"1", "2"}));
+        assertEquals("1", StringUtils.replaceEach("abcd", new String[] {"abcd"}, new String[] {"1"}));
+        assertEquals("", StringUtils.replaceEach("abcd", new String[] {"abcd"}, new String[] {""}));
+    }
+
+    @Test
+    public void replaceEach_adjacentMatches() {
+        assertEquals("111", StringUtils.replaceEach("aaa", new String[] {"a"}, new String[] {"1"}));
+        assertEquals("1212", StringUtils.replaceEach("abab", new String[] {"a", "b"}, new String[] {"1", "2"}));
+        assertEquals("2121", StringUtils.replaceEach("baba", new String[] {"a", "b"}, new String[] {"1", "2"}));
+    }
+
+    @Test
+    public void replaceEach_replacementsAreNotReplacedAgain() {
+        assertEquals("ba", StringUtils.replaceEach("ab", new String[] {"a", "b"}, new String[] {"b", "a"}));
+        assertEquals("aaaa", StringUtils.replaceEach("aa", new String[] {"a"}, new String[] {"aa"}));
+        assertEquals("&amp;lt;", StringUtils.replaceEach("&lt;", new String[] {"&", "<"}, new String[] {"&amp;", "&lt;"}));
+    }
+
+    @Test
+    public void replaceEach_duplicateSearchStrings_firstOneWins() {
+        assertEquals("11", StringUtils.replaceEach("aa", new String[] {"a", "a"}, new String[] {"1", "2"}));
+        assertEquals("22", StringUtils.replaceEach("aa", new String[] {"a", "a"}, new String[] {null, "2"}));
+    }
+
+    @Test
+    public void replaceEach_searchStringConsumedByEarlierMatch_isFoundAgainLater() {
+        // "bc" first matches at index 1, but "ab" consumes that "b". The next "bc" is at index 4.
+        assertEquals("1c-2", StringUtils.replaceEach("abc-bc", new String[] {"ab", "bc"}, new String[] {"1", "2"}));
+        // "a" is cached at index 2, but the long match at index 0 consumes it.
+        assertEquals("L-1", StringUtils.replaceEach("xxa-a", new String[] {"a", "xxa"}, new String[] {"1", "L"}));
+    }
+
+    @Test
+    public void replaceEach_searchStringUnusedForAWhile_isStillReplacedLater() {
+        String text = StringUtils.repeat("+", 1000) + "%7E" + StringUtils.repeat("*", 1000) + "%7E+";
+        String expected = StringUtils.repeat("%20", 1000) + "~" + StringUtils.repeat("%2A", 1000) + "~%20";
+        assertEquals(expected, StringUtils.replaceEach(text, new String[] {"+", "*", "%7E"},
+                                                       new String[] {"%20", "%2A", "~"}));
+    }
+
+    @Test
+    public void replaceEach_xmlEscaping() {
+        String[] search = {"&", "\"", "'", "<", ">", "\r", "\n"};
+        String[] replace = {"&amp;", "&quot;", "&apos;", "&lt;", "&gt;", "&#x0D;", "&#x0A;"};
+        assertEquals("&lt;a href=&quot;x&quot;&gt;Tom &amp; Jerry&apos;s&lt;/a&gt;&#x0D;&#x0A;",
+                     StringUtils.replaceEach("<a href=\"x\">Tom & Jerry's</a>\r\n", search, replace));
+    }
+
+    @Test
+    public void replaceEach_urlEncodingFixups() {
+        String[] search = {"+", "*", "%7E", "%2F"};
+        String[] replace = {"%20", "%2A", "~", "/"};
+        assertEquals("a%20b%2Ac~d/e", StringUtils.replaceEach("a+b*c%7Ed%2Fe", search, replace));
+        assertEquals("%20%20~~/%2A", StringUtils.replaceEach("++%7E%7E%2F*", search, replace));
+        assertEquals("%7", StringUtils.replaceEach("%7", search, replace));
+    }
+
+    @Test
+    public void replaceEach_nonAsciiText() {
+        assertEquals("\u00e9X\ud83d\ude00Y", StringUtils.replaceEach("\u00e9a\ud83d\ude00b", new String[] {"a", "b"},
+                                                                   new String[] {"X", "Y"}));
+        assertEquals("smile", StringUtils.replaceEach("\ud83d\ude00", new String[] {"\ud83d\ude00"},
+                                                      new String[] {"smile"}));
+    }
+
+    @Test
+    public void replaceEach_largeInputWithManyMatches_replacesAll() {
+        int n = 100_000;
+        String text = StringUtils.repeat("&", n) + "<";
+        assertEquals(StringUtils.repeat("&amp;", n) + "&lt;",
+                     StringUtils.replaceEach(text, new String[] {"&", "<"}, new String[] {"&amp;", "&lt;"}));
+    }
+
+    private static String randomString(Random random, String[] alphabet, int length) {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < length; i++) {
+            sb.append(alphabet[random.nextInt(alphabet.length)]);
+        }
+        return sb.toString();
+    }
+
+    /**
+     * The previous, straightforward implementation of {@link StringUtils#replaceEach}, which rescans every
+     * search string after every replacement. Used to check that the optimized version gives the same output.
+     */
+    private static String referenceReplaceEach(String text, String[] searchList, String[] replacementList) {
+        if (text == null || text.isEmpty()) {
+            return text;
+        }
+        StringBuilder buf = new StringBuilder();
+        int start = 0;
+        while (true) {
+            int textIndex = -1;
+            int replaceIndex = -1;
+            for (int i = 0; i < searchList.length; i++) {
+                if (searchList[i] == null || searchList[i].isEmpty() || replacementList[i] == null) {
+                    continue;
+                }
+                int tempIndex = text.indexOf(searchList[i], start);
+                if (tempIndex != -1 && (textIndex == -1 || tempIndex < textIndex)) {
+                    textIndex = tempIndex;
+                    replaceIndex = i;
+                }
+            }
+            if (textIndex == -1) {
+                break;
+            }
+            buf.append(text, start, textIndex).append(replacementList[replaceIndex]);
+            start = textIndex + searchList[replaceIndex].length();
+        }
+        buf.append(text, start, text.length());
+        return buf.toString();
+    }
 }