This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 03accc4245 fix error handling for get data from XML, fixes #5921
(#7463)
03accc4245 is described below
commit 03accc4245b5f986723ad9d29e3103b6e062763b
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed Jul 8 20:05:32 2026 +0200
fix error handling for get data from XML, fixes #5921 (#7463)
---
.../transforms/xml/getxmldata/GetXmlData.java | 260 +++++++++++----------
.../transforms/xml/getxmldata/GetXMLDataTest.java | 116 +++++++++
2 files changed, 259 insertions(+), 117 deletions(-)
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
index 204cfcdd95..6353b07fda 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlData.java
@@ -443,152 +443,178 @@ public class GetXmlData extends
BaseTransform<GetXmlDataMeta, GetXmlDataData> {
private boolean ReadNextString() {
- try {
- // Grab another row ...
- data.readrow = getRow();
+ // Loop so that, when error handling is enabled, we can skip an offending
input row and
+ // immediately continue with the next one without recursing.
+ while (true) {
+ try {
+ // Grab another row ...
+ data.readrow = getRow();
- if (data.readrow == null) {
- // finished processing!
+ if (data.readrow == null) {
+ // finished processing!
- if (isDetailed()) {
- logDetailed(BaseMessages.getString(PKG,
"GetXMLData.Log.FinishedProcessing"));
+ if (isDetailed()) {
+ logDetailed(BaseMessages.getString(PKG,
"GetXMLData.Log.FinishedProcessing"));
+ }
+ return false;
}
- return false;
- }
- if (first) {
- first = false;
+ if (first) {
+ first = false;
- data.nrReadRow = getInputRowMeta().size();
- data.inputRowMeta = getInputRowMeta();
- data.outputRowMeta = data.inputRowMeta.clone();
- meta.getFields(data.outputRowMeta, getTransformName(), null, null,
this, metadataProvider);
+ data.nrReadRow = getInputRowMeta().size();
+ data.inputRowMeta = getInputRowMeta();
+ data.outputRowMeta = data.inputRowMeta.clone();
+ meta.getFields(
+ data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
- // Get total previous fields
- data.totalpreviousfields = data.inputRowMeta.size();
+ // Get total previous fields
+ data.totalpreviousfields = data.inputRowMeta.size();
- // Create convert meta-data objects that will contain Date & Number
formatters
- data.convertRowMeta = new RowMeta();
- for (IValueMeta valueMeta : data.convertRowMeta.getValueMetaList()) {
- data.convertRowMeta.addValueMeta(
- ValueMetaFactory.cloneValueMeta(valueMeta,
IValueMeta.TYPE_STRING));
- }
+ // Create convert meta-data objects that will contain Date & Number
formatters
+ data.convertRowMeta = new RowMeta();
+ for (IValueMeta valueMeta : data.convertRowMeta.getValueMetaList()) {
+ data.convertRowMeta.addValueMeta(
+ ValueMetaFactory.cloneValueMeta(valueMeta,
IValueMeta.TYPE_STRING));
+ }
- // For String to <type> conversions, we allocate a conversion meta
data row as well...
- //
- data.convertRowMeta =
data.outputRowMeta.cloneToType(IValueMeta.TYPE_STRING);
+ // For String to <type> conversions, we allocate a conversion meta
data row as well...
+ //
+ data.convertRowMeta =
data.outputRowMeta.cloneToType(IValueMeta.TYPE_STRING);
- // Check is XML field is provided
- if (Utils.isEmpty(meta.getXmlField())) {
- logError(BaseMessages.getString(PKG, "GetXMLData.Log.NoField"));
- throw new HopException(BaseMessages.getString(PKG,
"GetXMLData.Log.NoField"));
- }
+ // Check is XML field is provided
+ if (Utils.isEmpty(meta.getXmlField())) {
+ logError(BaseMessages.getString(PKG, "GetXMLData.Log.NoField"));
+ throw new HopException(BaseMessages.getString(PKG,
"GetXMLData.Log.NoField"));
+ }
- // cache the position of the field
- if (data.indexOfXmlField < 0) {
- data.indexOfXmlField =
getInputRowMeta().indexOfValue(meta.getXmlField());
+ // cache the position of the field
if (data.indexOfXmlField < 0) {
- // The field is unreachable !
- logError(
- BaseMessages.getString(
- PKG, "GetXMLData.Log.ErrorFindingField",
meta.getXmlField()));
- throw new HopException(
- BaseMessages.getString(
- PKG, "GetXMLData.Exception.CouldnotFindField",
meta.getXmlField()));
+ data.indexOfXmlField =
getInputRowMeta().indexOfValue(meta.getXmlField());
+ if (data.indexOfXmlField < 0) {
+ // The field is unreachable !
+ logError(
+ BaseMessages.getString(
+ PKG, "GetXMLData.Log.ErrorFindingField",
meta.getXmlField()));
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "GetXMLData.Exception.CouldnotFindField",
meta.getXmlField()));
+ }
}
}
- }
- if (meta.isInFields()) {
- // get XML field value
- String fieldvalue = getInputRowMeta().getString(data.readrow,
data.indexOfXmlField);
+ if (meta.isInFields()) {
+ // get XML field value
+ String fieldvalue = getInputRowMeta().getString(data.readrow,
data.indexOfXmlField);
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG, "GetXMLData.Log.XMLStream", meta.getXmlField(),
fieldvalue));
- }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "GetXMLData.Log.XMLStream", meta.getXmlField(),
fieldvalue));
+ }
- if (meta.isAFile()) {
- FileObject file = null;
try {
- // XML source is a file.
- file = HopVfs.getFileObject(resolve(fieldvalue), variables);
-
- if (meta.isIgnoreEmptyFile() && file.getContent().getSize() == 0) {
- logBasic(
- BaseMessages.getString(
- PKG, "GetXMLData.Error.FileSizeZero", "" +
file.getName()));
- return ReadNextString();
- }
+ if (meta.isAFile()) {
+ FileObject file = null;
+ try {
+ // XML source is a file.
+ file = HopVfs.getFileObject(resolve(fieldvalue), variables);
+
+ if (meta.isIgnoreEmptyFile() && file.getContent().getSize() ==
0) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "GetXMLData.Error.FileSizeZero", "" +
file.getName()));
+ continue;
+ }
- // Open the XML document
- if (!setDocument(null, file, false, false)) {
- throw new HopException(
- BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_CREATE_DOCUMENT));
- }
+ // Open the XML document
+ if (!setDocument(null, file, false, false)) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_CREATE_DOCUMENT));
+ }
- if (!applyXPath()) {
- throw new HopException(
- BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_APPLY_XPATH));
- }
+ if (!applyXPath()) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_APPLY_XPATH));
+ }
- addFileToResultFilesname(file);
+ addFileToResultFilesname(file);
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG,
- CONST_GET_XMLDATA_LOG_LOOP_FILE_OCCURENCES,
- "" + data.nodesize,
- file.getName().getBaseName()));
- }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ CONST_GET_XMLDATA_LOG_LOOP_FILE_OCCURENCES,
+ "" + data.nodesize,
+ file.getName().getBaseName()));
+ }
- } catch (Exception e) {
- throw new HopException(e);
- } finally {
- try {
- if (file != null) {
- file.close();
+ } catch (Exception e) {
+ throw new HopException(e);
+ } finally {
+ try {
+ if (file != null) {
+ file.close();
+ }
+ } catch (Exception e) {
+ // Ignore close errors
+ }
+ }
+ } else {
+ boolean url = false;
+ boolean xmltring = true;
+ if (meta.isReadUrl()) {
+ url = true;
+ xmltring = false;
}
- } catch (Exception e) {
- // Ignore close errors
- }
- }
- } else {
- boolean url = false;
- boolean xmltring = true;
- if (meta.isReadUrl()) {
- url = true;
- xmltring = false;
- }
- // Open the XML document
- if (!setDocument(fieldvalue, null, xmltring, url)) {
- throw new HopException(
- BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_CREATE_DOCUMENT));
- }
+ // Open the XML document
+ if (!setDocument(fieldvalue, null, xmltring, url)) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_CREATE_DOCUMENT));
+ }
- // Apply XPath and set node list
- if (!applyXPath()) {
- throw new HopException(
- BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_APPLY_XPATH));
- }
- if (isDetailed()) {
- logDetailed(
- BaseMessages.getString(
- PKG, CONST_GET_XMLDATA_LOG_LOOP_FILE_OCCURENCES, "" +
data.nodesize));
+ // Apply XPath and set node list
+ if (!applyXPath()) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
CONST_GET_XMLDATA_LOG_UNABLE_APPLY_XPATH));
+ }
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, CONST_GET_XMLDATA_LOG_LOOP_FILE_OCCURENCES, "" +
data.nodesize));
+ }
+ }
+ } catch (HopException e) {
+ // A problem while reading or parsing the XML coming from the
input field is a
+ // per-row data error (e.g. the field is null or the content is
not valid XML).
+ // When error handling is enabled, divert the offending input row
to the error
+ // stream and continue with the next one instead of aborting the
whole transform.
+ if (getTransformMeta().isDoingErrorHandling()) {
+ // Keep the same row structure processPutRow() uses so both
error paths align.
+ Object[] errorRowData =
+ RowDataUtil.createResizedCopy(data.readrow,
data.outputRowMeta.size());
+ putError(
+ data.outputRowMeta,
+ errorRowData,
+ 1,
+ e.toString(),
+ meta.getXmlField(),
+ "GetXMLData002");
+ continue;
+ }
+ throw e;
}
}
+ return true;
+ } catch (Exception e) {
+ logError(BaseMessages.getString(PKG, "GetXMLData.Log.UnexpectedError",
e.toString()));
+ stopAll();
+ logError(Const.getStackTracker(e));
+ setErrors(1);
+ return false;
}
- } catch (Exception e) {
- logError(BaseMessages.getString(PKG, "GetXMLData.Log.UnexpectedError",
e.toString()));
- stopAll();
- logError(Const.getStackTracker(e));
- setErrors(1);
- return false;
}
- return true;
}
private void addFileToResultFilesname(FileObject file) {
diff --git
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXMLDataTest.java
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXMLDataTest.java
index 549da286d5..51016a0db0 100644
---
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXMLDataTest.java
+++
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXMLDataTest.java
@@ -41,6 +41,7 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.RowProducer;
import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.ITransform;
+import org.apache.hop.pipeline.transform.TransformErrorMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
import org.apache.hop.pipeline.transforms.injector.InjectorMeta;
@@ -453,4 +454,119 @@ class GetXMLDataTest {
assertEquals("${xml_path}", gxdm.getInputFields().get(0).getXPath());
assertEquals("data/owner",
gxdm.getInputFields().get(0).getResolvedXPath());
}
+
+ /** Build the five string fields used by the "in fields" test pipelines. */
+ private GetXmlDataField[] createXmlDataFields() {
+ String[] names = {"objectid", "sapident", "quantity", "merkmalname",
"merkmalswert"};
+ String[] xpaths = {"ObjectID", "SAPIDENT", "Quantity", "Merkmalname",
"Merkmalswert"};
+
+ GetXmlDataField[] fields = new GetXmlDataField[names.length];
+ for (int idx = 0; idx < fields.length; idx++) {
+ GetXmlDataField field = new GetXmlDataField();
+ field.setName(names[idx]);
+ field.setXPath(xpaths[idx]);
+
field.setElementType(GetXmlDataField.getElementTypeCode(GetXmlDataField.ELEMENT_TYPE_NODE));
+ field.setType(ValueMetaFactory.getValueMetaName(IValueMeta.TYPE_STRING));
+ field.setFormat("");
+ field.setLength(-1);
+ field.setPrecision(-1);
+ field.setCurrencySymbol("");
+ field.setDecimalSymbol("");
+ field.setGroupSymbol("");
+
field.setTrimType(GetXmlDataField.getTrimTypeCode(GetXmlDataField.TYPE_TRIM_NONE));
+ fields[idx] = field;
+ }
+ return fields;
+ }
+
+ /**
+ * With error handling enabled and XML coming from an input field, a row
whose XML field is null
+ * (or otherwise unparseable) must be diverted to the error stream while the
transform keeps
+ * processing the remaining rows, instead of aborting the whole transform.
Regression test for the
+ * "transform still stops when it should continue" bug.
+ */
+ @Test
+ void testErrorHandlingContinuesOnBadXml() throws Exception {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("getxmldata-errorhandling");
+
+ PluginRegistry registry = PluginRegistry.getInstance();
+
+ // Injector
+ String injectorTransformName = "injector transform";
+ InjectorMeta im = new InjectorMeta();
+ String injectorPid = registry.getPluginId(TransformPluginType.class, im);
+ TransformMeta injectorTransform = new TransformMeta(injectorPid,
injectorTransformName, im);
+ pipelineMeta.addTransform(injectorTransform);
+
+ // Get XML Data, reading the XML from the incoming "field1"
+ String getXMLDataName = "get xml data transform";
+ GetXmlDataMeta gxdm = new GetXmlDataMeta();
+ gxdm.setEncoding(Const.UTF_8);
+ gxdm.setAFile(false);
+ gxdm.setInFields(true);
+ gxdm.setLoopXPath("Level1/Level2/Props");
+ gxdm.setXmlField("field1");
+ gxdm.setInputFields(java.util.Arrays.asList(createXmlDataFields()));
+ String getXMLDataPid = registry.getPluginId(TransformPluginType.class,
gxdm);
+ TransformMeta getXMLDataTransform = new TransformMeta(getXMLDataPid,
getXMLDataName, gxdm);
+ pipelineMeta.addTransform(getXMLDataTransform);
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(injectorTransform,
getXMLDataTransform));
+
+ // Main output
+ String dummyMainName = "dummy main";
+ DummyMeta dmMain = new DummyMeta();
+ String dummyMainPid = registry.getPluginId(TransformPluginType.class,
dmMain);
+ TransformMeta dummyMain = new TransformMeta(dummyMainPid, dummyMainName,
dmMain);
+ pipelineMeta.addTransform(dummyMain);
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(getXMLDataTransform,
dummyMain));
+
+ // Error output
+ String dummyErrorName = "dummy error";
+ DummyMeta dmError = new DummyMeta();
+ String dummyErrorPid = registry.getPluginId(TransformPluginType.class,
dmError);
+ TransformMeta dummyError = new TransformMeta(dummyErrorPid,
dummyErrorName, dmError);
+ pipelineMeta.addTransform(dummyError);
+ pipelineMeta.addPipelineHop(new PipelineHopMeta(getXMLDataTransform,
dummyError));
+
+ // Enable error handling on Get XML Data, routing error rows to the error
dummy
+ TransformErrorMeta errorMeta = new TransformErrorMeta(getXMLDataTransform,
dummyError);
+ errorMeta.setEnabled(true);
+ getXMLDataTransform.setTransformErrorMeta(errorMeta);
+
+ Pipeline pipeline = new LocalPipelineEngine(pipelineMeta);
+ pipeline.prepareExecution();
+
+ // Capture the error rows emitted by the Get XML Data transform itself...
+ RowTransformCollector errorCollector = new RowTransformCollector();
+ pipeline.getTransform(getXMLDataName, 0).addRowListener(errorCollector);
+ // ...and the good rows arriving at the main output.
+ RowTransformCollector mainCollector = new RowTransformCollector();
+ pipeline.getTransform(dummyMainName, 0).addRowListener(mainCollector);
+
+ RowProducer rp = pipeline.addRowProducer(injectorTransformName, 0);
+ pipeline.startThreads();
+
+ // A good row, a row whose XML field is null (the bug trigger), then
another good row.
+ IRowMeta rm = createRowMetaInterface();
+ rp.putRow(rm, new Object[] {getXML1()});
+ rp.putRow(rm, new Object[] {null});
+ rp.putRow(rm, new Object[] {getXML2()});
+ rp.finished();
+
+ pipeline.waitUntilFinished();
+
+ // The transform must keep going: no errors reported, so the pipeline did
not abort.
+ assertEquals(0, pipeline.getResult().getNrErrors(), "transform should not
report errors");
+
+ // Both good rows still produce their output (2 from XML1 + 1 from XML2).
+ assertEquals(3, mainCollector.getRowsWritten().size(), "good rows still
flow through");
+
+ // The single bad row is diverted to error handling and carries the
original (null) field.
+ assertEquals(1, errorCollector.getRowsError().size(), "bad row goes to
error handling");
+ assertEquals(
+ null,
+ errorCollector.getRowsError().getFirst().getData()[0],
+ "error row keeps the offending input value");
+ }
}
