findFirst(String input, int fromIndex);
+
+ /**
+ * Finds every non-overlapping match in {@code input}, in order.
+ *
+ *
+ * The default implementation streams through {@link #findFirst} so that
+ * implementations only need to provide that one primitive; override this
+ * method if the underlying engine offers a more efficient bulk-match API.
+ */
+ default List findAll(String input) {
+ List matches = new ArrayList<>();
+ int pos = 0;
+ while (pos <= input.length()) {
+ Optional match = findFirst(input, pos);
+ if (!match.isPresent()) {
+ break;
+ }
+ RegexMatch m = match.get();
+ matches.add(m);
+ pos = m.getIndex() + Math.max(m.getLength(), 1);
+ }
+ return matches;
+ }
+
+ /**
+ * Splits {@code input} around every match of this pattern, following
+ * {@link java.util.regex.Pattern#split(CharSequence)} semantics (no
+ * trailing empty strings).
+ */
+ String[] split(String input);
+}
diff --git a/src/test/java/com/api/jsonata4java/expressions/FunctionErrorTest.java b/src/test/java/com/api/jsonata4java/expressions/FunctionErrorTest.java
index 6119bb53..188733d7 100644
--- a/src/test/java/com/api/jsonata4java/expressions/FunctionErrorTest.java
+++ b/src/test/java/com/api/jsonata4java/expressions/FunctionErrorTest.java
@@ -25,7 +25,6 @@
import static com.api.jsonata4java.expressions.utils.Utils.test;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
-import com.api.jsonata4java.expressions.functions.ReplaceFunction;
/**
* A JUnit test class used in order to ease debugging of particular problematic cases.
@@ -179,10 +178,6 @@ public void testRegex_case012() throws Exception {
.replaceAll("\\$\\$", "\\\\\\$")
.replaceAll("([^\\\\]|^)\\$([^0-9^<])", "$1\\\\\\$$2"));
- assertEquals("abc\\$$1xyz", ReplaceFunction.jsonata2JavaReplacement("abc$$$1xyz"));
- assertEquals("abc\\$wxyz", ReplaceFunction.jsonata2JavaReplacement("abc$wxyz"));
- assertEquals("abc\\$$1efg\\$wxyz", ReplaceFunction.jsonata2JavaReplacement("abc$$$1efg$wxyz"));
-
assertEquals("$265", "265USD".replaceAll("([0-9]+)USD", "\\$$1"));
test("$replace(\"265USD\", /([0-9]+)USD/, \"$$$1\")", "\"$265\"", null, null);
}
diff --git a/src/test/java/com/api/jsonata4java/expressions/RegularExpressionTest.java b/src/test/java/com/api/jsonata4java/expressions/RegularExpressionTest.java
index 074c02c4..3a3b54b3 100644
--- a/src/test/java/com/api/jsonata4java/expressions/RegularExpressionTest.java
+++ b/src/test/java/com/api/jsonata4java/expressions/RegularExpressionTest.java
@@ -42,15 +42,15 @@ public void testToString() {
@Test
public void find() {
- assertTrue(new RegularExpression("/^.*$/").getPattern().matcher("asdfgh").find());
- assertTrue(new RegularExpression("/^ab.*ef$/").getPattern().matcher("abcdef").find());
- assertTrue(new RegularExpression("/c.*f/").getPattern().matcher("abcdefgh").find());
+ assertTrue(new RegularExpression("/^.*$/").getPattern().test("asdfgh"));
+ assertTrue(new RegularExpression("/^ab.*ef$/").getPattern().test("abcdef"));
+ assertTrue(new RegularExpression("/c.*f/").getPattern().test("abcdefgh"));
}
@Test
public void findCaseInsensitive() {
- assertFalse(new RegularExpression("/^ab.*ef$/").getPattern().matcher("ABCdef").find());
+ assertFalse(new RegularExpression("/^ab.*ef$/").getPattern().test("ABCdef"));
assertTrue(new RegularExpression(RegularExpression.Type.CASEINSENSITIVE, "/^ab.*ef$/")
- .getPattern().matcher("ABCdef").find());
+ .getPattern().test("ABCdef"));
}
}
diff --git a/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPattern.java b/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPattern.java
new file mode 100644
index 00000000..e45438e9
--- /dev/null
+++ b/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPattern.java
@@ -0,0 +1,106 @@
+/**
+ * (c) Copyright 2018, 2019 IBM Corporation
+ * 1 New Orchard Road,
+ * Armonk, New York, 10504-1722
+ * United States
+ * +1 914 499 1900
+ * support: Nathaniel Mills wnm3@us.ibm.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.api.jsonata4java.expressions.regex;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import com.google.re2j.Matcher;
+import com.google.re2j.Pattern;
+
+/**
+ * Demonstration {@link RegexPattern} implementation backed by Google's RE2/J
+ * (https://github.com/google/re2j), a pure-Java, linear-time regex engine
+ * immune to ReDoS (https://en.wikipedia.org/wiki/ReDoS). Shows that
+ * JSONata4Java's regex engine hook (see {@link RegexPattern}/{@link RegexEngine})
+ * is not tied to {@link java.util.regex.Pattern}.
+ *
+ *
+ * Lives under src/test since it's only used to demonstrate/exercise the
+ * hook (see {@link RE2RegexPatternTest}) and isn't part of the library's
+ * runtime dependencies.
+ */
+public final class RE2RegexPattern implements RegexPattern {
+
+ private final Pattern pattern;
+
+ public RE2RegexPattern(String regexPattern, RegexFlags flags) {
+ int re2Flags = 0;
+ if (flags.isCaseInsensitive()) {
+ re2Flags |= Pattern.CASE_INSENSITIVE;
+ }
+ if (flags.isMultiline()) {
+ re2Flags |= Pattern.MULTILINE;
+ }
+ this.pattern = Pattern.compile(regexPattern, re2Flags);
+ }
+
+ /**
+ * Factory for use with {@code Expressions.parse}/{@code Expression.jsonata},
+ * e.g. {@code Expression.jsonata(expr, RE2RegexPattern::new)}.
+ */
+ public static RegexEngine engine() {
+ return RE2RegexPattern::new;
+ }
+
+ @Override
+ public boolean test(String input) {
+ return pattern.matcher(input).find();
+ }
+
+ @Override
+ public Optional findFirst(String input, int fromIndex) {
+ if (fromIndex < 0 || fromIndex > input.length()) {
+ return Optional.empty();
+ }
+ Matcher matcher = pattern.matcher(input);
+ if (!matcher.find(fromIndex)) {
+ return Optional.empty();
+ }
+ return Optional.of(toRegexMatch(matcher));
+ }
+
+ @Override
+ public List findAll(String input) {
+ List matches = new ArrayList<>();
+ Matcher matcher = pattern.matcher(input);
+ while (matcher.find()) {
+ matches.add(toRegexMatch(matcher));
+ }
+ return matches;
+ }
+
+ @Override
+ public String[] split(String input) {
+ return pattern.split(input);
+ }
+
+ private static RegexMatch toRegexMatch(Matcher matcher) {
+ List groups = new ArrayList<>();
+ int groupCount = matcher.groupCount();
+ for (int i = 1; i <= groupCount; i++) {
+ groups.add(matcher.group(i));
+ }
+ return new RegexMatch(matcher.group(), matcher.start(), groups);
+ }
+}
diff --git a/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPatternTest.java b/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPatternTest.java
new file mode 100644
index 00000000..de1768fc
--- /dev/null
+++ b/src/test/java/com/api/jsonata4java/expressions/regex/RE2RegexPatternTest.java
@@ -0,0 +1,105 @@
+/**
+ * (c) Copyright 2018, 2019 IBM Corporation
+ * 1 New Orchard Road,
+ * Armonk, New York, 10504-1722
+ * United States
+ * +1 914 499 1900
+ * support: Nathaniel Mills wnm3@us.ibm.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.api.jsonata4java.expressions.regex;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+import com.api.jsonata4java.Expression;
+import com.api.jsonata4java.expressions.EvaluateException;
+import com.api.jsonata4java.expressions.ParseException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Test;
+
+/**
+ * End-to-end tests demonstrating that {@link Expression#jsonata(String, RegexEngine)}
+ * works correctly when plugging in a regex engine other than the default
+ * {@link java.util.regex.Pattern}-backed one -- here, {@link RE2RegexPattern}.
+ */
+public class RE2RegexPatternTest {
+
+ private static JsonNode evaluate(String expression) throws ParseException, IOException, EvaluateException {
+ return Expression.jsonata(expression, RE2RegexPattern.engine()).evaluate(null);
+ }
+
+ @Test
+ public void testMatch() throws Exception {
+ // Note: regex literals containing whitespace hit a pre-existing,
+ // unrelated JSONata4Java grammar limitation (its lexer's global
+ // whitespace-skip rule strips spaces out of regex literals too), so
+ // this intentionally avoids whitespace in the pattern.
+ JsonNode result = evaluate("$match(\"hello-world\", /o-w/)");
+ assertEquals("o-w", result.get("match").asText());
+ assertEquals(4, result.get("index").asInt());
+ }
+
+ @Test
+ public void testMatchCaseInsensitive() throws Exception {
+ JsonNode result = evaluate("$match(\"HELLO\", /hello/i)");
+ assertEquals("HELLO", result.get("match").asText());
+ }
+
+ @Test
+ public void testContains() throws Exception {
+ assertTrue(evaluate("$contains(\"hello\", /ell/)").asBoolean());
+ assertFalse(evaluate("$contains(\"hello\", /xyz/)").asBoolean());
+ }
+
+ @Test
+ public void testReplaceWithGroupReference() throws Exception {
+ JsonNode result = evaluate("$replace(\"John Smith\", /(\\w+)\\s(\\w+)/, \"$2, $1\")");
+ assertEquals("Smith, John", result.asText());
+ }
+
+ @Test
+ public void testSplit() throws Exception {
+ JsonNode result = evaluate("$split(\"a1b2c3\", /[0-9]/)");
+ ObjectMapper mapper = new ObjectMapper();
+ assertEquals(mapper.readTree("[\"a\",\"b\",\"c\"]"), result);
+ }
+
+ @Test
+ public void testRejectsBackreferences() {
+ // RE2 (and therefore RE2/J) deliberately does not support
+ // backreferences, since they make linear-time matching impossible --
+ // this is exactly the ReDoS-prone construct a custom RegexEngine is
+ // meant to let callers reject at the source.
+ try {
+ evaluate("$contains(\"aa\", /(a)\\1/)");
+ org.junit.Assert.fail("Expected an exception compiling a backreference with RE2/J");
+ } catch (Exception expected) {
+ // expected: RE2/J's Pattern.compile rejects the unsupported syntax
+ }
+ }
+
+ @Test
+ public void testEvalUsesEnclosingRegexEngine() throws Exception {
+ // $eval()'d expressions must inherit the enclosing expression's
+ // regex engine rather than silently falling back to the default
+ // java.util.regex-backed one.
+ JsonNode result = evaluate("$eval(\"$match('hello-world', /o-w/).match\")");
+ assertEquals("o-w", result.asText());
+ }
+}