This is an automated email from the ASF dual-hosted git repository.
heneveld pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
The following commit(s) were added to refs/heads/master by this push:
new c4c167a30e improved quote handling for ssh etc
c4c167a30e is described below
commit c4c167a30ebf24c4259e1c04373546af547aec56
Author: Alex Heneveld <[email protected]>
AuthorDate: Fri Nov 11 21:19:15 2022 +0000
improved quote handling for ssh etc
---
.../brooklyn/core/workflow/ShorthandProcessor.java | 27 ++++----
.../core/workflow/WorkflowStepDefinition.java | 6 +-
.../core/workflow/steps/LoadWorkflowStep.java | 2 +-
.../core/workflow/steps/RetryWorkflowStep.java | 2 +-
.../workflow/steps/SetVariableWorkflowStep.java | 23 ++++---
.../core/workflow/ShorthandProcessorTest.java | 17 ++++--
.../workflow/WorkflowInputOutputExtensionTest.java | 69 +++++++++++++++++++--
.../brooklyn/util/text/QuotedStringTokenizer.java | 71 +++++++++++++++-------
.../util/text/QuotedStringTokenizerTest.java | 53 +++++++++++++---
9 files changed, 201 insertions(+), 69 deletions(-)
diff --git
a/core/src/main/java/org/apache/brooklyn/core/workflow/ShorthandProcessor.java
b/core/src/main/java/org/apache/brooklyn/core/workflow/ShorthandProcessor.java
index 74cf3cf6e9..4165678b91 100644
---
a/core/src/main/java/org/apache/brooklyn/core/workflow/ShorthandProcessor.java
+++
b/core/src/main/java/org/apache/brooklyn/core/workflow/ShorthandProcessor.java
@@ -51,6 +51,7 @@ public class ShorthandProcessor {
private final String template;
boolean finalMatchRaw = false;
+ boolean failOnMismatch = true;
public ShorthandProcessor(String template) {
this.template = template;
@@ -60,18 +61,24 @@ public class ShorthandProcessor {
return new ShorthandProcessorAttempt(this, input).call();
}
- /** whether the last match should preserve quotes and spaces */
+ /** whether the last match should preserve quotes and spaces; default
false */
public ShorthandProcessor withFinalMatchRaw(boolean finalMatchRaw) {
this.finalMatchRaw = finalMatchRaw;
return this;
}
+ /** whether to fail on mismatched quotes in the input, default true */
+ public ShorthandProcessor withFailOnMismatch(boolean failOnMismatch) {
+ this.failOnMismatch = failOnMismatch;
+ return this;
+ }
+
static class ShorthandProcessorAttempt {
private final List<String> templateTokens;
private final String inputOriginal;
private final QuotedStringTokenizer qst;
private final String template;
- private final boolean finalMatchRaw;
+ private final ShorthandProcessor options;
int optionalDepth = 0;
int optionalSkippingInput = 0;
private String inputRemaining;
@@ -80,14 +87,14 @@ public class ShorthandProcessor {
ShorthandProcessorAttempt(ShorthandProcessor proc, String input) {
this.template = proc.template;
- this.finalMatchRaw = proc.finalMatchRaw;
+ this.options = proc;
this.qst = qst(template);
this.templateTokens = qst.remainderAsList();
this.inputOriginal = input;
}
private QuotedStringTokenizer qst(String x) {
- return
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(false).keepInternalQuotes(true).failOnOpenQuote(true).build(x);
+ return
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(false).expectQuotesDelimited(true).failOnOpenQuote(options.failOnMismatch).build(x);
}
public synchronized Maybe<Map<String,Object>> call() {
@@ -243,7 +250,7 @@ public class ShorthandProcessor {
if (!qstInput.hasMoreTokens()) return Maybe.absent("End of
input when looking for variable "+t);
if (!templateTokens.stream().filter(x ->
!x.equals("]")).findFirst().isPresent()) {
- // last word (whether optional or not) takes
everything, but trimmed parsed and joined with spaces
+ // last word (whether optional or not) takes everything
value = getRemainderPossiblyRaw(qstInput);
inputRemaining = "";
@@ -284,13 +291,9 @@ public class ShorthandProcessor {
private String getRemainderPossiblyRaw(QuotedStringTokenizer qstInput)
{
String value;
- if (finalMatchRaw) {
- value = Strings.join(qstInput.remainderRaw(), "");
- } else {
- List<String> remainder = qstInput.remainderAsList();
- value = remainder.stream()
- .map(qstInput::unwrapIfQuoted)
- .collect(Collectors.joining(" "));
+ value = Strings.join(qstInput.remainderRaw(), "");
+ if (!options.finalMatchRaw) {
+ return qstInput.unwrapIfQuoted(value);
}
return value;
}
diff --git
a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowStepDefinition.java
b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowStepDefinition.java
index 16a89024cc..be18cba08a 100644
---
a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowStepDefinition.java
+++
b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowStepDefinition.java
@@ -138,10 +138,10 @@ public abstract class WorkflowStepDefinition {
abstract public void populateFromShorthand(String value);
protected void populateFromShorthandTemplate(String template, String
value) {
- populateFromShorthandTemplate(template, value, false);
+ populateFromShorthandTemplate(template, value, false, true);
}
- protected void populateFromShorthandTemplate(String template, String
value, boolean finalMatchRaw) {
- Maybe<Map<String, Object>> result = new
ShorthandProcessor(template).withFinalMatchRaw(finalMatchRaw).process(value);
+ protected void populateFromShorthandTemplate(String template, String
value, boolean finalMatchRaw, boolean failOnMismatch) {
+ Maybe<Map<String, Object>> result = new
ShorthandProcessor(template).withFinalMatchRaw(finalMatchRaw).withFailOnMismatch(failOnMismatch).process(value);
if (result.isAbsent()) throw new IllegalArgumentException("Invalid
shorthand expression: '"+value+"'", Maybe.Absent.getException(result));
input.putAll((Map) CollectionMerger.builder().build().merge(input,
result.get()));
diff --git
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/LoadWorkflowStep.java
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/LoadWorkflowStep.java
index 1eb14fcab8..c642f4cfb8 100644
---
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/LoadWorkflowStep.java
+++
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/LoadWorkflowStep.java
@@ -64,7 +64,7 @@ public class LoadWorkflowStep extends WorkflowStepDefinition {
@Override
public void populateFromShorthand(String expression) {
- populateFromShorthandTemplate(SHORTHAND, expression, true);
+ populateFromShorthandTemplate(SHORTHAND, expression);
}
@Override
diff --git
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/RetryWorkflowStep.java
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/RetryWorkflowStep.java
index cbfa716311..1fae9d2011 100644
---
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/RetryWorkflowStep.java
+++
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/RetryWorkflowStep.java
@@ -123,7 +123,7 @@ public class RetryWorkflowStep extends
WorkflowStepDefinition {
if (Strings.isBlank(initialS)) {
throw new IllegalArgumentException("initial duration required
for backoff");
}
- result.initial =
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(false).keepInternalQuotes(true).failOnOpenQuote(true).build(initialS).remainderAsList().stream()
+ result.initial =
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(false).expectQuotesDelimited(true).failOnOpenQuote(true).build(initialS).remainderAsList().stream()
.map(Duration::of).collect(Collectors.toList());
String factor = (String) resultM.get().get("factor");
diff --git
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/SetVariableWorkflowStep.java
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/SetVariableWorkflowStep.java
index e6b8fd2f2c..540b7b133a 100644
---
a/core/src/main/java/org/apache/brooklyn/core/workflow/steps/SetVariableWorkflowStep.java
+++
b/core/src/main/java/org/apache/brooklyn/core/workflow/steps/SetVariableWorkflowStep.java
@@ -45,7 +45,6 @@ import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
-import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -74,7 +73,7 @@ public class SetVariableWorkflowStep extends
WorkflowStepDefinition {
@Override
public void populateFromShorthand(String expression) {
- populateFromShorthandTemplate(SHORTHAND, expression, true);
+ populateFromShorthandTemplate(SHORTHAND, expression, true, true);
}
@Override
@@ -134,7 +133,7 @@ public class SetVariableWorkflowStep extends
WorkflowStepDefinition {
private final LetMergeMode merge;
private String specialCoercionMode;
private final boolean typeSpecified;
- private QuotedStringTokenizer qst;
+// private QuotedStringTokenizer qst;
public SetVariableEvaluation(WorkflowStepInstanceExecutionContext
context, TypeToken<T> type, Object unresolvedValue) {
@@ -282,8 +281,7 @@ public class SetVariableWorkflowStep extends
WorkflowStepDefinition {
public void mergeInto(Object input, Function<String,TypeToken<?>>
explicitType, BiConsumer<String,Object> holderAdd) {
if (input instanceof String) {
- qst =
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(true).keepInternalQuotes(true).failOnOpenQuote(false).build((String)
input);
- List<String> wordsByQuote = qst.remainderRaw();
+ List<String> wordsByQuote = qst((String) input).remainderRaw();
wordsByQuote.forEach(word -> {
if (Strings.isBlank(word)) return;
Maybe<Object> wordResolved =
processMaybe(MutableList.of(word), explicitType);
@@ -299,12 +297,22 @@ public class SetVariableWorkflowStep extends
WorkflowStepDefinition {
if (Strings.isBlank(input)) return input;
// first deal with internal quotes
- qst =
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(true).keepInternalQuotes(true).failOnOpenQuote(false).build(input);
- List<String> wordsByQuote = qst.remainderAsList();
+ QuotedStringTokenizer qst = qst(input);
+ List<String> wordsByQuote;
+ if (qst.isQuoted(input)) {
+ // special treatment if whole line is quoted
+ wordsByQuote = MutableList.of(input);
+ } else {
+ wordsByQuote = qst.remainderAsList();
+ }
// then look for operators etc
return process(wordsByQuote, null);
}
+ QuotedStringTokenizer qst(String input) {
+ return
QuotedStringTokenizer.builder().includeQuotes(true).includeDelimiters(true).expectQuotesDelimited(true).failOnOpenQuote(true).build(input);
+ }
+
Object process(List<String> w, Function<String,TypeToken<?>>
explicitType) {
// if no tokens, treat as null
if (w.isEmpty()) return null;
@@ -324,6 +332,7 @@ public class SetVariableWorkflowStep extends
WorkflowStepDefinition {
// tokens include space delimiters and are still quotes, so unwrap
then just stitch together. will preserve spaces.
boolean resolveToString = w.size()>1;
+ QuotedStringTokenizer qst = qst("");
List<Object> objs = w.stream().map(t -> {
if (qst.isQuoted(t)) return qst.unwrapIfQuoted(t);
TypeToken<?> target = resolveToString ?
TypeToken.of(String.class) : TypeToken.of(Object.class);
diff --git
a/core/src/test/java/org/apache/brooklyn/core/workflow/ShorthandProcessorTest.java
b/core/src/test/java/org/apache/brooklyn/core/workflow/ShorthandProcessorTest.java
index b1deac4b09..3d1e91ee01 100644
---
a/core/src/test/java/org/apache/brooklyn/core/workflow/ShorthandProcessorTest.java
+++
b/core/src/test/java/org/apache/brooklyn/core/workflow/ShorthandProcessorTest.java
@@ -33,6 +33,11 @@ public class ShorthandProcessorTest extends
BrooklynMgmtUnitTestSupport {
Asserts.assertEquals(new
ShorthandProcessor(template).process(input).get(), expected);
}
+ void assertShorthandOfGivesError(String template, String input,
Map<String,Object> expected) {
+ Asserts.assertEquals(new
ShorthandProcessor(template).withFailOnMismatch(false).process(input).get(),
expected);
+ Asserts.assertFails(() -> new
ShorthandProcessor(template).withFailOnMismatch(true).process(input).get());
+ }
+
void assertShorthandFinalMatchRawOfGives(String template, String input,
Map<String,Object> expected) {
Asserts.assertEquals(new
ShorthandProcessor(template).withFinalMatchRaw(true).process(input).get(),
expected);
}
@@ -51,7 +56,7 @@ public class ShorthandProcessorTest extends
BrooklynMgmtUnitTestSupport {
assertShorthandOfGives("${x}", "hello world", MutableMap.of("x",
"hello world"));
assertShorthandOfGives("${x} \" is \" ${y}", "a is b c",
MutableMap.of("x", "a", "y", "b c"));
- assertShorthandOfGives("${x} \" is \" ${y}", "a is b \"c\"",
MutableMap.of("x", "a", "y", "b c"));
+ assertShorthandOfGives("${x} \" is \" ${y}", "a is b \"c\"",
MutableMap.of("x", "a", "y", "b \"c\""));
// don't allow intermediate multi-token matching; we could add but not
needed yet
// assertShorthandOfGives("${x} \" is \" ${y}", "a b is b c",
MutableMap.of("x", "a b", "y", "b c"));
@@ -62,16 +67,16 @@ public class ShorthandProcessorTest extends
BrooklynMgmtUnitTestSupport {
assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is c",
MutableMap.of("x", "this is b", "y", "c"));
// quotes with spaces before/after removed
- assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is c is
\"is quoted\"", MutableMap.of("x", "this is b", "y", "c is is quoted"));
+ assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is c is
\"is quoted\"", MutableMap.of("x", "this is b", "y", "c is \"is quoted\""));
// if you want quotes, you have to wrap them in quotes
assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is \"\\\"c
is is quoted\\\"\"", MutableMap.of("x", "this is b", "y", "\"c is is
quoted\""));
// and only quotes at word end are considered, per below
- assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is \"\\\"c
is \"is quoted\"\\\"\"", MutableMap.of("x", "this is b", "y", "\"c is \"is
quoted\"\""));
- assertShorthandOfGives("${x} \" is \" ${y}", "\"this is b\" is \"\\\"c
is \"is quoted\"\\\"\" too", MutableMap.of("x", "this is b", "y", "\"c is
\"is quoted\"\" too"));
+ assertShorthandOfGivesError("${x} \" is \" ${y}", "\"this is b\" is
\"\\\"c is \"is quoted\"\\\"\"", MutableMap.of("x", "this is b", "y", "\"c is
\"is quoted\"\""));
+ assertShorthandOfGivesError("${x} \" is \" ${y}", "\"this is b\" is
\"\\\"c is \"is quoted\"\\\"\" too", MutableMap.of("x", "this is b", "y",
"\"\\\"c is \"is quoted\"\\\"\" too"));
// preserve spaces in a word
assertShorthandOfGives("${x}", "\" sp a ces \"", MutableMap.of("x",
" sp a ces "));
- assertShorthandOfGives("${x}", "\" sp a ces \" and then some",
MutableMap.of("x", " sp a ces and then some"));
+ assertShorthandOfGives("${x}", "\" sp a ces \" and then some",
MutableMap.of("x", "\" sp a ces \" and then some"));
// if you want quotes, you have to wrap them in quotes
assertShorthandOfGives("${x}", "\"\\\"c is is quoted\\\"\"",
MutableMap.of("x", "\"c is is quoted\""));
@@ -81,7 +86,7 @@ public class ShorthandProcessorTest extends
BrooklynMgmtUnitTestSupport {
// so this gives an error
assertShorthandFailsWith("${x}", "\"c is \"is", e ->
Asserts.expectedFailureContainsIgnoreCase(e, "mismatched", "quot"));
// and this is treated as one quoted string
- assertShorthandOfGives("${x}", "\"\\\"c is \"is quoted\"\\\"\"",
MutableMap.of("x", "\"c is \"is quoted\"\""));
+ assertShorthandOfGivesError("${x}", "\"\\\"c is \"is
quoted\"\\\"\"", MutableMap.of("x", "\"c is \"is quoted\"\""));
}
@Test
diff --git
a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowInputOutputExtensionTest.java
b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowInputOutputExtensionTest.java
index 3f335b089d..7496dc568a 100644
---
a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowInputOutputExtensionTest.java
+++
b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowInputOutputExtensionTest.java
@@ -50,7 +50,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.function.BiFunction;
import java.util.function.Consumer;
+import java.util.function.Function;
public class WorkflowInputOutputExtensionTest extends
BrooklynMgmtUnitTestSupport {
@@ -343,12 +345,67 @@ public class WorkflowInputOutputExtensionTest extends
BrooklynMgmtUnitTestSuppor
}
@Test
- public void testLetQuoteVar() throws Exception {
- Object output = invokeWorkflowStepsWithLogging(MutableList.of(
- "let person = Anna",
- "let note = \"${person}\" is ${person}",
- "log NOTE: ${note}"));
- Asserts.assertEquals(lastLogWatcher.getMessages().stream().filter(s ->
s.startsWith("NOTE:")).findAny().get(), "NOTE: ${person} is Anna");
+ public void testLetQuoteVar() {
+ Consumer<String> invoke = input -> { try {
+ invokeWorkflowStepsWithLogging(MutableList.of(
+ "let person = Anna",
+ "let note = "+input,
+ "log NOTE: ${note}"));
+ } catch (Exception e) {
+ throw Exceptions.propagate(e);
+ } };
+ BiFunction<String,String,String> assertLetGives = (input,expected) -> {
+ invoke.accept(input);
+ String note =
Strings.removeFromStart(lastLogWatcher.getMessages().stream().filter(s ->
s.startsWith("NOTE:")).findAny().get(), "NOTE: ");
+ Asserts.assertEquals(note, expected);
+ return note;
+ };
+
+ assertLetGives.apply("\"${person}\"", "${person}");
+ assertLetGives.apply("\"${person}\" is ${person}", "${person} is
Anna");
+ assertLetGives.apply("\"${person} is\" ${person}", "${person} is
Anna");
+ assertLetGives.apply("\"${person}\" is '${person}'", "${person} is
${person}");
+ assertLetGives.apply("\"${person}\" is '\"${person}\"'", "${person} is
\"${person}\"");
+ assertLetGives.apply("\"${person}\" is 'person'", "${person} is
person");
+ Asserts.assertFails(() -> invoke.accept("\"${person}\" is ''person
'"));
+ assertLetGives.apply("\"${person} is person \"", "${person} is
person ");
+ Asserts.assertFails(() -> invoke.accept("\"\"${person}\" is person
\""));
+ assertLetGives.apply("\"'${person}' is person \"", "'${person}' is
person ");
+ assertLetGives.apply("\"\\\"${person}\\\" is person \"",
"\"${person}\" is person ");
+ Asserts.assertFails(() -> invoke.accept("\"\\\"${person}\\\" is
\"person \""));
+ assertLetGives.apply("\"\\\"${person}\" is \"person \"", "\"${person}
is person ");
+ }
+
+ @Test
+ public void testSetSensorQuoteVar() {
+ Consumer<String> invoke = input -> { try {
+ invokeWorkflowStepsWithLogging(MutableList.of(
+ "let person = Anna",
+ "set-sensor note = "+input,
+ "let note = ${entity.sensor.note}",
+ "log NOTE: ${note}"));
+ } catch (Exception e) {
+ throw Exceptions.propagate(e);
+ } };
+ BiFunction<String,String,String> assertSetSensorGives =
(input,expected) -> {
+ invoke.accept(input);
+ String note =
Strings.removeFromStart(lastLogWatcher.getMessages().stream().filter(s ->
s.startsWith("NOTE:")).findAny().get(), "NOTE: ");
+ Asserts.assertEquals(note, expected);
+ return note;
+ };
+
+ assertSetSensorGives.apply("\"${person}\" is ${person}", "\"Anna\" is
Anna");
+ assertSetSensorGives.apply("\"${person}\" is '${person}'", "\"Anna\"
is 'Anna'");
+ assertSetSensorGives.apply("\"${person} is\" ${person}", "\"Anna is\"
Anna");
+ assertSetSensorGives.apply("\"${person}\" is ''${person}''", "\"Anna\"
is ''Anna''");
+ assertSetSensorGives.apply("\"${person}\" is 'person'", "\"Anna\" is
'person'");
+ assertSetSensorGives.apply("\"${person}\" is ''person '", "\"Anna\"
is ''person '"); // doesn't fail because quotes preserved
+ assertSetSensorGives.apply("\"${person} is person \"", "Anna is
person ");
+ Asserts.assertFails(() -> invoke.accept("\"\"${person}\" is person
\""));
+ assertSetSensorGives.apply("\"'${person}' is person \"", "'Anna' is
person ");
+ assertSetSensorGives.apply("\"\\\"${person}\\\" is person \"",
"\"Anna\" is person ");
+ Asserts.assertFails(() -> invoke.accept("\"\\\"${person}\\\" is
\"person \""));
+ assertSetSensorGives.apply("\"\\\"${person}\" is \"person \"",
"\"\\\"Anna\" is \"person \"");
}
@Test
diff --git
a/utils/common/src/main/java/org/apache/brooklyn/util/text/QuotedStringTokenizer.java
b/utils/common/src/main/java/org/apache/brooklyn/util/text/QuotedStringTokenizer.java
index 1742801af0..dbd1d928e4 100644
---
a/utils/common/src/main/java/org/apache/brooklyn/util/text/QuotedStringTokenizer.java
+++
b/utils/common/src/main/java/org/apache/brooklyn/util/text/QuotedStringTokenizer.java
@@ -39,7 +39,8 @@ public class QuotedStringTokenizer {
final boolean includeQuotes;
final String delimiters;
final boolean includeDelimiters;
- final boolean keepInternalQuotes;
+ /** see the static parseXxxx methods for explanation */
+ final boolean expectQuotesDelimited;
final boolean failOnOpenQuote;
public static String DEFAULT_QUOTE_CHARS = "\"\'";
@@ -69,13 +70,13 @@ public class QuotedStringTokenizer {
public QuotedStringTokenizer(String stringToTokenize, String quoteChars,
boolean includeQuotes, String delimiters, boolean includeDelimiters) {
this(stringToTokenize, quoteChars, includeQuotes, delimiters,
includeDelimiters, false, false);
}
- public QuotedStringTokenizer(String stringToTokenize, String quoteChars,
boolean includeQuotes, String delimiters, boolean includeDelimiters, boolean
keepInternalQuotes, boolean failOnOpenQuote) {
+ public QuotedStringTokenizer(String stringToTokenize, String quoteChars,
boolean includeQuotes, String delimiters, boolean includeDelimiters, boolean
expectQuotesDelimited, boolean failOnOpenQuote) {
delegate = new StringTokenizer(stringToTokenize==null ? "" :
stringToTokenize, (delimiters==null ? DEFAULT_DELIMITERS : delimiters), true);
this.quoteChars = quoteChars==null ? DEFAULT_QUOTE_CHARS() :
quoteChars;
this.includeQuotes = includeQuotes;
this.delimiters = delimiters==null ? DEFAULT_DELIMITERS : delimiters;
this.includeDelimiters = includeDelimiters;
- this.keepInternalQuotes = keepInternalQuotes;
+ this.expectQuotesDelimited = expectQuotesDelimited;
this.failOnOpenQuote = failOnOpenQuote;
updateNextToken();
}
@@ -85,14 +86,14 @@ public class QuotedStringTokenizer {
private boolean includeQuotes=true;
private String delimiterChars=DEFAULT_DELIMITERS;
private boolean includeDelimiters=false;
- private boolean keepInternalQuotes=false;
+ private boolean expectQuotesDelimited=false;
private boolean failOnOpenQuote=false;
public QuotedStringTokenizer build(String stringToTokenize) {
- return new QuotedStringTokenizer(stringToTokenize, quoteChars,
includeQuotes, delimiterChars, includeDelimiters, keepInternalQuotes,
failOnOpenQuote);
+ return new QuotedStringTokenizer(stringToTokenize, quoteChars,
includeQuotes, delimiterChars, includeDelimiters, expectQuotesDelimited,
failOnOpenQuote);
}
public List<String> buildList(String stringToTokenize) {
- return new QuotedStringTokenizer(stringToTokenize, quoteChars,
includeQuotes, delimiterChars, includeDelimiters, keepInternalQuotes,
failOnOpenQuote).remainderAsList();
+ return new QuotedStringTokenizer(stringToTokenize, quoteChars,
includeQuotes, delimiterChars, includeDelimiters, expectQuotesDelimited,
failOnOpenQuote).remainderAsList();
}
public Builder quoteChars(String quoteChars) { this.quoteChars =
quoteChars; return this; }
@@ -100,23 +101,26 @@ public class QuotedStringTokenizer {
public Builder includeQuotes(boolean includeQuotes) {
this.includeQuotes = includeQuotes; return this; }
public Builder delimiterChars(String delimiterChars) {
this.delimiterChars = delimiterChars; return this; }
public Builder addDelimiterChars(String delimiterChars) {
this.delimiterChars = this.delimiterChars + delimiterChars; return this; }
- public Builder includeDelimiters(boolean includeDelimiters) {
this.includeDelimiters = includeDelimiters; return this; }
- public Builder keepInternalQuotes(boolean keepInternalQuotes) {
this.keepInternalQuotes = keepInternalQuotes; return this; }
+ public Builder includeDelimiters(boolean includeDelimiters) {
this.includeDelimiters = includeDelimiters; return this; }
+ public Builder keepInternalQuotes(boolean expectQuotesDelimited) {
this.expectQuotesDelimited = expectQuotesDelimited; return this; }
+ public Builder expectQuotesDelimited(boolean expectQuotesDelimited) {
this.expectQuotesDelimited = expectQuotesDelimited; return this; }
public Builder failOnOpenQuote(boolean failOnOpenQuote) {
this.failOnOpenQuote = failOnOpenQuote; return this; }
}
public static Builder builder() {
return new Builder();
}
- /** this is the historic default in Brooklyn, ensuring delimiters
(whitespace) inside quotes aren't used as delimitiers, then stripping out those
quotes;
- * good for more things but won't keep quoted quotes */
- public static List<String> parseStrippingInternalQuotes(String s) {
- return
QuotedStringTokenizer.builder().keepInternalQuotes(false).buildList(s);
+ /** this is the historic default in Brooklyn, respecting quotes anywhere
(processing them before delimiters),
+ * good if quotes are not offset with whitespace (eg x="foo,bar",y="baz"
with a , as a delimiter) */
+ public static List<String> parseWithQuotesAnywhere(String s) {
+ return
QuotedStringTokenizer.builder().expectQuotesDelimited(false).buildList(s);
}
- /** revision to QST to look only at delimiters (whitespace), and unescape
quotes inside. forgives malformed content, but at least it preserves quotes. */
- public static List<String> parseKeepingInternalQuotes(String s) {
- return
QuotedStringTokenizer.builder().keepInternalQuotes(true).buildList(s);
+ /** revision to QST to acknowledging quotes only if offset by delimiters;
+ * good if delimiters are whitespace, (eg x = "foo bar" y = "baz"); in
this mode, fail will cause it to fail if there are any unescaped quotes,
+ * but in non-fail mode it will avoid getting confused by an apostrophe as
a quote! */
+ public static List<String> parseWithDelimitedQuotesOnly(String s) {
+ return
QuotedStringTokenizer.builder().expectQuotesDelimited(true).buildList(s);
}
/** use java StreamTokenizer with most of the defaults, which does most of
what we want -- unescaping internal quotes -- and following its usual style of
distinguishing words from identifiers etc*/
@@ -185,13 +189,29 @@ public class QuotedStringTokenizer {
if (peekedNextToken==null) throw new NoSuchElementException();
String lastToken = peekedNextToken;
updateNextToken();
- return includeQuotes ? lastToken : keepInternalQuotes ?
unwrapIfQuoted(lastToken) : unquoteToken(lastToken);
+ return includeQuotes ? lastToken : expectQuotesDelimited ?
unwrapIfQuoted(lastToken) : unquoteToken(lastToken);
}
+ /** returns true if the given string is one quoted string. if fail is
specified, if it looks quoted but has unescaped internal quotes of the same
type, it will fail. */
public boolean isQuoted(String token) {
- if (Strings.isEmpty(token)) return false;
+ if (Strings.isBlank(token)) return false;
+ token = token.trim();
if (!isQuoteChar(token.charAt(0))) return false;
- if (isOpenQuote(token)) return false;
+ if (isUnterminatedQuotedPhrase(token)) return false;
+ if (!token.endsWith(""+token.charAt(0))) return false;
+
+ String quoteFreeRequired = token.substring(1, token.length() - 1);
+ quoteFreeRequired = Strings.replaceAllRegex(quoteFreeRequired,
"\\\\.", "");
+ while (true) {
+ int i = quoteFreeRequired.indexOf("" + token.charAt(0));
+ if (i==-1) break;
+ quoteFreeRequired = quoteFreeRequired.substring(i+1);
+ if (quoteFreeRequired.isEmpty()) break;
+ if (delimiters.contains(""+quoteFreeRequired.charAt(0))) return
false; // must not have " followed by delimeter
+ // if fail is specified, quoted strings must escape their contents
+ if (failOnOpenQuote) throw new
IllegalArgumentException("Mismatched inner quotes");
+ }
+
return true;
}
@@ -276,17 +296,21 @@ public class QuotedStringTokenizer {
}
private void pullUntilValid(StringBuffer nextToken) {
- while (isOpenQuote(nextToken.toString()) && delegate.hasMoreTokens()) {
+ while (isUnterminatedQuotedPhrase(nextToken.toString()) &&
delegate.hasMoreTokens()) {
//keep appending until the quote is ended or there are no more
quotes
nextToken.append(delegate.nextToken());
}
- if (failOnOpenQuote && isOpenQuote(nextToken.toString())) {
- throw new IllegalArgumentException("Mismatched quotation marks in
expression");
+ if (failOnOpenQuote) {
+ if (isUnterminatedQuotedPhrase(nextToken.toString())) {
+ throw new IllegalArgumentException("Mismatched quotation marks
in expression");
+ }
+ // do this for stricter error checks
+ isQuoted(nextToken.toString());
}
}
- boolean isOpenQuote(String stringToCheck) {
- if (!keepInternalQuotes) return hasOpenQuote(stringToCheck,
quoteChars);
+ boolean isUnterminatedQuotedPhrase(String stringToCheck) {
+ if (!expectQuotesDelimited) return hasOpenQuote(stringToCheck,
quoteChars);
if (stringToCheck.isEmpty()) return false;
char start = stringToCheck.charAt(0);
@@ -311,6 +335,7 @@ public class QuotedStringTokenizer {
return hasOpenQuote(stringToCheck, DEFAULT_QUOTE_CHARS);
}
+ /** detects whether there are mismatched quotes, ignoring delimeters, but
attending to escapes, finding a quote char, then skipping everything up to its
match, and repeating */
public static boolean hasOpenQuote(String stringToCheck, String
quoteChars) {
String x = stringToCheck;
if (x==null) return false;
diff --git
a/utils/common/src/test/java/org/apache/brooklyn/util/text/QuotedStringTokenizerTest.java
b/utils/common/src/test/java/org/apache/brooklyn/util/text/QuotedStringTokenizerTest.java
index 7c00a53256..aa31e89da7 100644
---
a/utils/common/src/test/java/org/apache/brooklyn/util/text/QuotedStringTokenizerTest.java
+++
b/utils/common/src/test/java/org/apache/brooklyn/util/text/QuotedStringTokenizerTest.java
@@ -20,17 +20,12 @@ package org.apache.brooklyn.util.text;
import static org.testng.Assert.assertEquals;
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StreamTokenizer;
-import java.io.StringReader;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import org.apache.brooklyn.test.Asserts;
-import org.apache.brooklyn.util.text.QuotedStringTokenizer;
+import org.apache.brooklyn.util.exceptions.Exceptions;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -68,17 +63,55 @@ public class QuotedStringTokenizerTest {
// testResultingTokens("foo,,baz", "\"", false, ",", false, "foo", "",
"baz");
}
+ String testIsQuoted(String expression, boolean isQuotedExpected, boolean
shouldFailInStrictMode, String ...expectedTokens) {
+ QuotedStringTokenizer.Builder qb =
QuotedStringTokenizer.builder().expectQuotesDelimited(true);
+ QuotedStringTokenizer qbi = qb.build(expression);
+
+ Asserts.assertEquals(qbi.remainderAsList(),
Arrays.asList(expectedTokens));
+ Asserts.assertEquals(qbi.isQuoted(expression), isQuotedExpected);
+ Object result = null;
+ try {
+ QuotedStringTokenizer qb2 =
QuotedStringTokenizer.builder().expectQuotesDelimited(true).failOnOpenQuote(true).build(expression);
+ result = qb2.remainderAsList();
+ } catch (Exception e) {
+ if (!shouldFailInStrictMode) throw Exceptions.propagate(e);
+ }
+ if (shouldFailInStrictMode && result!=null) Asserts.fail("Should have
failed on input: "+expression+" -- instead returned "+result);
+
+ return qbi.unwrapIfQuoted(expression);
+ }
+
+ void testUnquoted(String expression, boolean shouldFailInStrictMode,
String expectedTokenWrapped, String expectedTokenUnwrapped) {
+ String unq = testIsQuoted(expression, true, shouldFailInStrictMode,
expectedTokenWrapped);
+ Asserts.assertEquals(unq, expectedTokenUnwrapped);
+ }
+
@Test
- public void testTokenizingKeepingInternalQuotes() throws Exception {
+ public void testIsQuoted() {
+ QuotedStringTokenizer.Builder qb =
QuotedStringTokenizer.builder().expectQuotesDelimited(true);
+ QuotedStringTokenizer qbi = qb.build("");
+ testIsQuoted("\"x \" ''y z \"1'", false, true, "\"x \"", "''y z \"1'");
+
+ testUnquoted("\"x \\\"y z \\\"1\"", false, "\"x \\\"y z \\\"1\"", "x
\"y z \"1");
+ testUnquoted("\"x \"y z \"1\"", true, "\"x \"y z \"1\"", "x \"y z
\"1");
+ testIsQuoted("\"x \" ''y z \"1'", false, true, "\"x \"", "''y z \"1'");
+ testIsQuoted("\"x \" 'y z \"1'", false, false, "\"x \"", "'y z \"1'");
+ testIsQuoted("\"x \"y\" z \"1\"", false, true, "\"x \"y\"", "z",
"\"1\"");
+ testIsQuoted("\\\"x \"y\" z \"1\"", false, false, "\\\"x", "\"y\"",
"z", "\"1\"");
+ }
+
+ @Test
+ public void testTokenizingWithQuotesDelimited() throws Exception {
testResultingTokens("foo,bar,baz", "\"", false, ",", false, true,
true, "foo", "bar", "baz");
testResultingTokens("\"foo,bar\",baz", "\"", false, ",", false, true,
true, "foo,bar", "baz");
testResultingTokens("\"foo,,bar\",baz", "\"", false, ",", false, true,
true, "foo,,bar", "baz");
testResultingTokens("\"foo,',bar\",baz", "\"", false, ",", false,
true, true, "foo,',bar", "baz");
testResultingTokens("\"foo,\',bar\",baz", "\"", false, ",", false,
true, true, "foo,',bar", "baz");
testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ",", false,
true, true, "foo \"\"bar\"\" baz");
- testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ", ", false,
true, true, "foo", "\"bar\"", "baz");
- testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ",",
false, true, true, "foo \"\"bar\"\" baz");
- testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ", ",
false, true, true, "foo \"\"bar\"", "baz\"");
+ testResultingTokens("foo \"\"bar\"\" baz", "\"", false, ", ", false,
true, false, "foo", "\"bar\"", "baz");
+
+ testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ",",
false, true, false, "foo \"\"bar\"\" baz");
+ testResultingTokens("\"foo \"\"bar\"\" baz\"", "\"", false, ", ",
false, true, false, "foo \"\"bar\"", "baz\"");
}
@Test
