This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 1b06db5978 Issue #3051 : Harden Filter Rows date constant parsing
(#8185)
1b06db5978 is described below
commit 1b06db59785d4f450cd3fc127804ead5d8b4f7af
Author: Matt Casters <[email protected]>
AuthorDate: Mon Aug 31 15:25:45 2026 +0200
Issue #3051 : Harden Filter Rows date constant parsing (#8185)
Parse date/timestamp condition constants with the stored mask first,
then fall back to Hop's compatible date format so legacy pipelines
still evaluate. Keep the condition editor open when conversion fails.
---
.../main/java/org/apache/hop/core/Condition.java | 64 ++++++++-
.../java/org/apache/hop/core/ConditionTest.java | 154 +++++++++++++++++++++
.../transforms/filterrows/FilterRowsTest.java | 35 +++++
.../transforms/joinrows/JoinRowsDialog.java | 2 +-
.../hop/ui/core/dialog/EnterValueDialog.java | 11 +-
.../apache/hop/ui/core/widget/ConditionEditor.java | 12 +-
6 files changed, 266 insertions(+), 12 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Condition.java
b/core/src/main/java/org/apache/hop/core/Condition.java
index ced122d131..3ac1437544 100644
--- a/core/src/main/java/org/apache/hop/core/Condition.java
+++ b/core/src/main/java/org/apache/hop/core/Condition.java
@@ -41,6 +41,7 @@ import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.ValueMetaAndData;
+import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.Utils;
@@ -831,6 +832,14 @@ public class Condition implements Cloneable {
length = valueMeta.getLength();
precision = valueMeta.getPrecision();
mask = valueMeta.getConversionMask();
+ // Persist the mask getString() actually used so later loads do not
depend on
+ // HOP_DEFAULT_DATE_FORMAT remaining unchanged.
+ if (valueMeta.isDate() && Utils.isEmpty(mask)) {
+ mask =
+ valueMeta.getType() == IValueMeta.TYPE_TIMESTAMP
+ ? ValueMetaBase.DEFAULT_TIMESTAMP_FORMAT_MASK
+ : ValueMetaBase.DEFAULT_DATE_FORMAT_MASK;
+ }
}
public int getHopType() {
@@ -854,23 +863,64 @@ public class Condition implements Cloneable {
}
/**
- * Convert the text stored to the desired data type in a compatible way
+ * Convert the text stored to the desired data type in a compatible way.
*
- * @return
+ * <p>Date and Timestamp constants historically stored {@code text} in
Hop's compatible date
+ * format ({@code yyyy/MM/dd HH:mm:ss.SSS}) while keeping the
user-selected conversion mask.
+ * Parse with the stored mask first, then fall back to that compatible
format and the type
+ * default so both new and legacy pipelines work.
*/
public Object createValueData() throws HopException {
if (isNullValue()) {
return null;
}
IValueMeta valueMeta = createValueMeta();
-
- ValueMetaAndData val = new ValueMetaAndData(valueMeta.getName(), text);
- val.setValueMeta(valueMeta);
-
IValueMeta stringValueMeta = new ValueMetaString(valueMeta.getName());
+ if (valueMeta.isDate()) {
+ return convertDateConstant(valueMeta, stringValueMeta);
+ }
+
stringValueMeta.setConversionMetadata(valueMeta);
+ return stringValueMeta.convertDataUsingConversionMetaData(text);
+ }
- return
stringValueMeta.convertDataUsingConversionMetaData(val.getValueData());
+ private Object convertDateConstant(IValueMeta valueMeta, IValueMeta
stringValueMeta)
+ throws HopException {
+ HopValueException firstError = null;
+ for (String tryMask : dateConstantMasks(valueMeta)) {
+ IValueMeta parseMeta = valueMeta.clone();
+ parseMeta.setConversionMask(tryMask);
+ stringValueMeta.setConversionMetadata(parseMeta);
+ try {
+ return stringValueMeta.convertDataUsingConversionMetaData(text);
+ } catch (HopValueException e) {
+ if (firstError == null) {
+ firstError = e;
+ }
+ }
+ }
+ if (firstError != null) {
+ throw firstError;
+ }
+ throw new HopValueException("Unable to convert constant [" + text + "]
to a date");
+ }
+
+ private List<String> dateConstantMasks(IValueMeta valueMeta) {
+ List<String> masks = new ArrayList<>();
+ addDateConstantMask(masks, mask);
+ addDateConstantMask(masks, ValueMetaBase.COMPATIBLE_DATE_FORMAT_PATTERN);
+ addDateConstantMask(
+ masks,
+ valueMeta.getType() == IValueMeta.TYPE_TIMESTAMP
+ ? ValueMetaBase.DEFAULT_TIMESTAMP_FORMAT_MASK
+ : ValueMetaBase.DEFAULT_DATE_FORMAT_MASK);
+ return masks;
+ }
+
+ private void addDateConstantMask(List<String> masks, String candidate) {
+ if (StringUtils.isNotEmpty(candidate) && !masks.contains(candidate)) {
+ masks.add(candidate);
+ }
}
}
diff --git a/core/src/test/java/org/apache/hop/core/ConditionTest.java
b/core/src/test/java/org/apache/hop/core/ConditionTest.java
index eb911d84d0..d1bd19ddb5 100644
--- a/core/src/test/java/org/apache/hop/core/ConditionTest.java
+++ b/core/src/test/java/org/apache/hop/core/ConditionTest.java
@@ -20,15 +20,26 @@ package org.apache.hop.core;
import static org.apache.hop.core.Condition.Function;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.row.ValueMetaAndData;
+import org.apache.hop.core.row.value.ValueMetaBase;
+import org.apache.hop.core.row.value.ValueMetaBigNumber;
+import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaTimestamp;
import org.apache.hop.core.util.TestUtil;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
@@ -126,4 +137,147 @@ class ConditionTest {
assertNull(condition.getRightValue());
assertEquals(Function.LARGER_EQUAL, condition.getFunction());
}
+
+ @Test
+ void dateConstantWithMatchingMaskEvaluates() throws Exception {
+ Condition condition = dateLessThanConstant("2022-01-01", "yyyy-MM-dd");
+ assertDateLessThan(condition);
+ }
+
+ @Test
+ void dateConstantWithLegacyCompatibleTextEvaluates() throws Exception {
+ // Issue #3051: text stored in the canonical Hop date format, mask is the
user format.
+ Condition condition = dateLessThanConstant("2022/01/01 00:00:00.000",
"yyyy-MM-dd");
+ assertDateLessThan(condition);
+ }
+
+ @Test
+ void dateConstantXmlRoundTripKeepsCustomMask() throws Exception {
+ SimpleDateFormat iso = isoDate();
+ ValueMetaDate dateMeta = new ValueMetaDate("constant");
+ dateMeta.setConversionMask("yyyy-MM-dd");
+ Condition original =
+ new Condition(
+ "date",
+ Function.SMALLER,
+ null,
+ new ValueMetaAndData(dateMeta, iso.parse("2022-01-01")));
+
+ Condition copy = new Condition(original.getXml());
+ assertEquals("yyyy-MM-dd", copy.getRightValue().getMask());
+ assertDateLessThan(copy);
+ }
+
+ @Test
+ void dateConstantConstructorTextIsParseable() throws Exception {
+ SimpleDateFormat iso = isoDate();
+ Date date = iso.parse("2022-01-01");
+ ValueMetaDate dateMeta = new ValueMetaDate("constant");
+ dateMeta.setConversionMask("yyyy-MM-dd");
+
+ Condition.CValue value = new Condition.CValue(new
ValueMetaAndData(dateMeta, date));
+ assertEquals("yyyy-MM-dd", value.getMask());
+ assertInstanceOf(Date.class, value.createValueData());
+ }
+
+ @Test
+ void dateConstantPersistsDefaultMaskWhenMissing() throws Exception {
+ Date date = isoDate().parse("2022-01-01");
+ Condition.CValue value =
+ new Condition.CValue(new ValueMetaAndData(new
ValueMetaDate("constant"), date));
+ assertEquals(ValueMetaBase.DEFAULT_DATE_FORMAT_MASK, value.getMask());
+ assertInstanceOf(Date.class, value.createValueData());
+ }
+
+ @Test
+ void timestampConstantWithLegacyCompatibleTextConverts() throws Exception {
+ Condition.CValue value = new Condition.CValue();
+ value.setName("constant");
+ value.setType("Timestamp");
+ value.setText("2022/01/01 12:34:56.789");
+ value.setMask("yyyy-MM-dd HH:mm:ss.SSS");
+ value.setNullValue(false);
+ value.setLength(-1);
+ value.setPrecision(-1);
+
+ assertNotNull(value.createValueData());
+ }
+
+ @Test
+ void timestampConstantConstructorTextIsParseable() throws Exception {
+ ValueMetaTimestamp timestampMeta = new ValueMetaTimestamp("constant");
+ timestampMeta.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS");
+ Timestamp timestamp = Timestamp.valueOf("2022-01-01 12:34:56.789");
+
+ Condition.CValue value = new Condition.CValue(new
ValueMetaAndData(timestampMeta, timestamp));
+ assertEquals("yyyy-MM-dd HH:mm:ss.SSS", value.getMask());
+ assertNotNull(value.createValueData());
+ }
+
+ @Test
+ void nullDateConstantStaysNull() throws Exception {
+ Condition.CValue value =
+ new Condition.CValue(new ValueMetaAndData(new
ValueMetaDate("constant"), null));
+ assertTrue(value.isNullValue());
+ assertNull(value.createValueData());
+ }
+
+ @Test
+ void unparseableDateConstantStillFails() {
+ Condition.CValue value = new Condition.CValue();
+ value.setName("constant");
+ value.setType("Date");
+ value.setText("not-a-date");
+ value.setMask("yyyy-MM-dd");
+ value.setNullValue(false);
+ value.setLength(-1);
+ value.setPrecision(-1);
+
+ assertThrows(HopException.class, value::createValueData);
+ }
+
+ @Test
+ void integerAndBigNumberConstantsStillConvert() throws Exception {
+ Condition.CValue integer =
+ new Condition.CValue(new ValueMetaAndData(new
ValueMetaInteger("constant"), 100L));
+ assertEquals(100L, integer.createValueData());
+
+ Condition.CValue bigNumber =
+ new Condition.CValue(
+ new ValueMetaAndData(new ValueMetaBigNumber("constant"), new
BigDecimal("123.45")));
+ assertEquals(0, new BigDecimal("123.45").compareTo((BigDecimal)
bigNumber.createValueData()));
+ }
+
+ private static Condition dateLessThanConstant(String text, String mask) {
+ Condition.CValue constant = new Condition.CValue();
+ constant.setName("constant");
+ constant.setType("Date");
+ constant.setText(text);
+ constant.setMask(mask);
+ constant.setNullValue(false);
+ constant.setLength(-1);
+ constant.setPrecision(-1);
+
+ Condition condition = new Condition();
+ condition.setLeftValueName("date");
+ condition.setFunction(Function.SMALLER);
+ condition.setRightValue(constant);
+ return condition;
+ }
+
+ private static void assertDateLessThan(Condition condition) throws Exception
{
+ SimpleDateFormat iso = isoDate();
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaDate("date"));
+
+ assertTrue(condition.evaluate(rowMeta, new Object[]
{iso.parse("2021-12-31")}));
+ assertFalse(condition.evaluate(rowMeta, new Object[]
{iso.parse("2022-01-01")}));
+ assertFalse(condition.evaluate(rowMeta, new Object[]
{iso.parse("2022-01-02")}));
+ }
+
+ private static SimpleDateFormat isoDate() {
+ SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd");
+ iso.setLenient(false);
+ return iso;
+ }
}
diff --git
a/plugins/transforms/filterrows/src/test/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsTest.java
b/plugins/transforms/filterrows/src/test/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsTest.java
index a3cde5988d..23254bcdb2 100644
---
a/plugins/transforms/filterrows/src/test/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsTest.java
+++
b/plugins/transforms/filterrows/src/test/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.pipeline.transforms.filterrows;
import static org.apache.hop.core.Condition.Function.EQUAL;
import static org.apache.hop.core.Condition.Function.REGEXP;
+import static org.apache.hop.core.Condition.Function.SMALLER;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -200,6 +201,40 @@ class FilterRowsTest {
assertFalse(meta.getCondition().evaluate(rowMeta, matchingRow));
}
+ /**
+ * Regression for #3051: a Date constant whose stored text is Hop's
compatible format while the
+ * mask is the user format must still evaluate after init() clones and
caches the condition.
+ */
+ @Test
+ void dateConstantWithLegacyCompatibleTextEvaluatesAfterInit() throws
Exception {
+ Condition.CValue constant = new Condition.CValue();
+ constant.setName("constant");
+ constant.setType("Date");
+ constant.setText("2022/01/01 00:00:00.000");
+ constant.setMask("yyyy-MM-dd");
+ constant.setNullValue(false);
+ constant.setLength(-1);
+ constant.setPrecision(-1);
+
+ Condition metadataCondition = new Condition();
+ metadataCondition.setLeftValueName("value");
+ metadataCondition.setFunction(SMALLER);
+ metadataCondition.setRightValue(constant);
+
+ FilterRows transform = createTransform(metadataCondition);
+ assertTrue(transform.init());
+ Condition runtimeCondition = transform.getData().condition;
+
+ SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd");
+ iso.setLenient(false);
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaDate("value"));
+
+ assertTrue(runtimeCondition.evaluate(rowMeta, new Object[]
{iso.parse("2021-12-31")}));
+ assertFalse(runtimeCondition.evaluate(rowMeta, new Object[]
{iso.parse("2022-01-01")}));
+ assertFalse(runtimeCondition.evaluate(rowMeta, new Object[]
{iso.parse("2022-01-02")}));
+ }
+
private static Stream<Arguments> variableValueTypes() throws Exception {
ValueMetaDate dateMeta = new ValueMetaDate("value");
dateMeta.setConversionMask("yyyy-MM-dd");
diff --git
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsDialog.java
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsDialog.java
index f111d6fb3b..3f79c40c56 100644
---
a/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsDialog.java
+++
b/plugins/transforms/joinrows/src/main/java/org/apache/hop/pipeline/transforms/joinrows/JoinRowsDialog.java
@@ -199,7 +199,7 @@ public class JoinRowsDialog extends BaseTransformDialog {
ke);
}
- wCondition = new ConditionEditor(shell, SWT.BORDER, condition,
inputfields);
+ wCondition = new ConditionEditor(shell, SWT.BORDER, condition,
inputfields, variables);
FormData fdCondition = new FormData();
fdCondition.left = new FormAttachment(0, 0);
diff --git
a/ui/src/main/java/org/apache/hop/ui/core/dialog/EnterValueDialog.java
b/ui/src/main/java/org/apache/hop/ui/core/dialog/EnterValueDialog.java
index 3f099951ab..ae58de38fa 100644
--- a/ui/src/main/java/org/apache/hop/ui/core/dialog/EnterValueDialog.java
+++ b/ui/src/main/java/org/apache/hop/ui/core/dialog/EnterValueDialog.java
@@ -324,14 +324,21 @@ public class EnterValueDialog extends Dialog {
public void getData() {
wValueType.setText(valueMeta.getTypeDesc());
try {
- if (valueData != null) {
+ if (valueData instanceof String stringData && !valueMeta.isString()) {
+ // Raw unparsed constant (variable expression or a date stored in
another format).
+ setInputText(stringData);
+ } else if (valueData != null) {
String value = valueMeta.getString(valueData);
if (value != null) {
setInputText(value);
}
}
} catch (HopValueException e) {
- setInputText(valueMeta.toString());
+ if (valueData instanceof String stringData) {
+ setInputText(stringData);
+ } else {
+ setInputText(valueMeta.toString());
+ }
}
setFormats();
diff --git
a/ui/src/main/java/org/apache/hop/ui/core/widget/ConditionEditor.java
b/ui/src/main/java/org/apache/hop/ui/core/widget/ConditionEditor.java
index c5d67e3809..c09c923cf3 100644
--- a/ui/src/main/java/org/apache/hop/ui/core/widget/ConditionEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ConditionEditor.java
@@ -358,9 +358,17 @@ public class ConditionEditor extends Canvas implements
MouseMoveListener {
new ValueMetaAndData(new
ValueMetaString("constant"), null));
}
}
+ IValueMeta valueMeta = v.createValueMeta();
+ Object valueData;
+ try {
+ valueData = v.createValueData();
+ } catch (Exception parseException) {
+ // Keep the stored text so a mask/text mismatch or a
variable expression
+ // can still be edited instead of failing to open the
dialog.
+ valueData = v.getText();
+ }
EnterValueDialog evd =
- new EnterValueDialog(
- shell, SWT.NONE, v.createValueMeta(),
v.createValueData(), variables);
+ new EnterValueDialog(shell, SWT.NONE, valueMeta,
valueData, variables);
evd.setModalDialog(
true); // To keep the condition editor from being
closed with a value dialog
// still
