This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 00c37a87f4 XML join and XSD validator create their XML factories for
every row, … (#9345)
00c37a87f4 is described below
commit 00c37a87f4f8ce3a84327269210346c3ba154d2f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Nov 16 09:58:16 2026 +0100
XML join and XSD validator create their XML factories for every row, …
(#9345)
* XML join and XSD validator create their XML factories for every row,
fixes #9342
* fix remarks
---
.../pipeline/transforms/xml/xmljoin/XmlJoin.java | 27 +-
.../transforms/xml/xmljoin/XmlJoinData.java | 8 +
.../transforms/xml/xsdvalidator/XsdValidator.java | 130 ++++++---
.../xml/xsdvalidator/XsdValidatorData.java | 34 +++
.../xml/xmljoin/XmlJoinSeveralRowsTest.java | 174 +++++++++++
.../xsdvalidator/XsdValidatorSchemaCacheTest.java | 323 +++++++++++++++++++++
6 files changed, 657 insertions(+), 39 deletions(-)
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
index 4a192fafdb..f6b7c2edc5 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.xml.xmljoin;
import java.io.StringReader;
import java.io.StringWriter;
import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.dom.DOMSource;
@@ -67,7 +68,11 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta,
XmlJoinData> {
@Override
public boolean processRow() throws HopException {
- XPath xpath = XPathFactory.newInstance().newXPath();
+ // Created once per copy: creating an XPathFactory scans the classpath.
+ if (data.xpath == null) {
+ data.xpath = XPathFactory.newInstance().newXPath();
+ }
+ XPath xpath = data.xpath;
// if first row we do some initializing and process the first row of the
target XML Transform
if (first) {
@@ -235,9 +240,8 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta,
XmlJoinData> {
String strJoinXML = (String) rJoinSource[data.iSourceXMLField];
try {
- DocumentBuilder builder =
-
XmlParserFactoryProducer.createSecureDocBuilderFactory().newDocumentBuilder();
- Document joinDocument = builder.parse(new InputSource(new
StringReader(strJoinXML)));
+ Document joinDocument =
+ getSourceDocumentBuilder().parse(new InputSource(new
StringReader(strJoinXML)));
Node node =
data.targetDom.importNode(joinDocument.getDocumentElement(), true);
@@ -261,6 +265,21 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta,
XmlJoinData> {
return true;
}
+ /**
+ * The parser for the source XML fragments, one per copy. Creating the
secure factory scans the
+ * classpath for every security feature it sets, so it is not done per row.
reset() returns the
+ * builder to the state the factory gave it, security features included.
+ */
+ private DocumentBuilder getSourceDocumentBuilder() throws
ParserConfigurationException {
+ if (data.sourceDocumentBuilder == null) {
+ data.sourceDocumentBuilder =
+
XmlParserFactoryProducer.createSecureDocBuilderFactory().newDocumentBuilder();
+ } else {
+ data.sourceDocumentBuilder.reset();
+ }
+ return data.sourceDocumentBuilder;
+ }
+
@Override
public boolean init() {
if (!super.init()) {
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
index bcd1ffcc4f..eda97c17a3 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
@@ -17,6 +17,8 @@
package org.apache.hop.pipeline.transforms.xml.xmljoin;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.xpath.XPath;
import org.apache.hop.core.IRowSet;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -45,6 +47,12 @@ public class XmlJoinData extends BaseTransformData
implements ITransformData {
public int iSourceXMLField = -1;
public int iCompareFieldID = -1;
+ /** Created on the first row, reused for every row of this copy. */
+ public XPath xpath;
+
+ /** Parses the source XML fragments; created on the first source row, reused
afterwards. */
+ public DocumentBuilder sourceDocumentBuilder;
+
/** */
public XmlJoinData() {
super();
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
index 4aa6fbc132..c28cbb81f7 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
@@ -127,51 +127,34 @@ public class XsdValidator extends
BaseTransform<XsdValidatorMeta, XsdValidatorDa
xsdfilename = getInputRowMeta().getString(row, data.xsdindex);
}
- // Get XSD file
+ // A compiled schema is kept per copy, by XSD file name. Only resolve
the file when the
+ // schema still has to be compiled.
+ Schema cachedSchema = getCachedSchema(xsdfilename);
FileObject xsdfile =
- StringUtils.isNotEmpty(xsdfilename) ?
HopVfs.getFileObject(xsdfilename, variables) : null;
+ cachedSchema == null && StringUtils.isNotEmpty(xsdfilename)
+ ? HopVfs.getFileObject(xsdfilename, variables)
+ : null;
try {
- // The factory resolves the schema document itself, so it needs the
same external-access
- // restrictions as the validator it produces. Honour the transform's
own opt-in so a
- // pipeline that deliberately relies on remote schemas keeps working.
- SchemaFactory factoryXSDValidator =
- meta.isAllowExternalEntities()
- ? SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
- : XmlParserFactoryProducer.createSecureSchemaFactory(
- XMLConstants.W3C_XML_SCHEMA_NS_URI);
-
// Get XML stream
Source sourceXML = getSourceXML(getInputRowMeta().getString(row,
data.xmlindex));
- // create the schema
+ // The compiled schema: from the cache, or compiled now and kept
Schema schematXSD =
- xsdfile != null
- ? getXsdSchemaFromFile(xsdfile, factoryXSDValidator)
- : getXsdSchema(xsdfilename, factoryXSDValidator);
-
- // Create XSDValidator
- Validator xsdValidator = schematXSD.newValidator();
-
- // Prevent against XML Entity Expansion (XEE) attacks.
- //
https://www.owasp.org/index.php/XML_Security_Cheat_Sheet#XML_Entity_Expansion
- if (!meta.isAllowExternalEntities()) {
-
xsdValidator.setFeature("http://apache.org/xml/features/disallow-doctype-decl",
true);
-
xsdValidator.setFeature("http://xml.org/sax/features/external-general-entities",
false);
-
xsdValidator.setFeature("http://xml.org/sax/features/external-parameter-entities",
false);
- xsdValidator.setProperty(
- "http://apache.org/xml/properties/internal/entity-resolver",
- (XMLEntityResolver)
- xmlResourceIdentifier -> {
- String message =
- BaseMessages.getString(PKG,
"XsdValidator.Exception.DisallowedDocType");
- throw new IOException(message);
- });
- }
+ cachedSchema != null ? cachedSchema : compileSchema(xsdfilename,
xsdfile);
+
+ // The validator of this schema, configured once
+ Validator xsdValidator = getValidator(schematXSD);
// Validate XML / XSD
- xsdValidator.validate(sourceXML);
+ try {
+ xsdValidator.validate(sourceXML);
+ } catch (SAXException | IOException e) {
+ // Don't reuse a validator that failed; the next row gets a new one.
+ data.validators.remove(schematXSD);
+ throw e;
+ }
isvalid = true;
@@ -246,6 +229,83 @@ public class XsdValidator extends
BaseTransform<XsdValidatorMeta, XsdValidatorDa
return true;
}
+ /**
+ * The schema factory of this copy, created on first use. The factory
resolves the schema document
+ * itself, so it needs the same external-access restrictions as the
validator it produces. Honour
+ * the transform's own opt-in so a pipeline that deliberately relies on
remote schemas keeps
+ * working. A SchemaFactory is not thread-safe, which is why it is kept per
copy.
+ */
+ private SchemaFactory getSchemaFactory() throws SAXException {
+ if (data.schemaFactory == null) {
+ data.schemaFactory =
+ meta.isAllowExternalEntities()
+ ? SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
+ : XmlParserFactoryProducer.createSecureSchemaFactory(
+ XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ }
+ return data.schemaFactory;
+ }
+
+ /** The schema compiled earlier for this XSD file name, or null. */
+ private Schema getCachedSchema(String xsdfilename) {
+ if (StringUtils.isEmpty(xsdfilename)) {
+ return data.schemaFromDocuments;
+ }
+ return data.schemas.get(xsdfilename);
+ }
+
+ /**
+ * Compiles the schema and keeps it for the next rows. A schema that fails
to compile is not kept,
+ * so every row that uses it reports the error.
+ */
+ private Schema compileSchema(String xsdfilename, FileObject xsdfile)
+ throws SAXException, HopTransformException, FileSystemException {
+ if (xsdfile == null) {
+ Schema schema = getXsdSchema(xsdfilename, getSchemaFactory());
+ data.schemaFromDocuments = schema;
+ return schema;
+ }
+ Schema schema = getXsdSchemaFromFile(xsdfile, getSchemaFactory());
+ data.schemas.put(xsdfilename, schema);
+ return schema;
+ }
+
+ /**
+ * The validator of this schema for this copy, configured once. A Validator
is not thread-safe,
+ * which is why it is kept per copy. reset() is not used: it would drop the
features set here.
+ *
+ * <p>The schema without an XSD file is the exception: its validator keeps
the schemas it loaded
+ * from the location hints of earlier documents and would validate later
documents against them,
+ * whatever their own hints say. That schema gets a new validator for every
row.
+ */
+ private Validator getValidator(Schema schema) throws SAXException {
+ Validator xsdValidator = data.validators.get(schema);
+ if (xsdValidator != null) {
+ return xsdValidator;
+ }
+ xsdValidator = schema.newValidator();
+
+ // Prevent against XML Entity Expansion (XEE) attacks.
+ //
https://www.owasp.org/index.php/XML_Security_Cheat_Sheet#XML_Entity_Expansion
+ if (!meta.isAllowExternalEntities()) {
+
xsdValidator.setFeature("http://apache.org/xml/features/disallow-doctype-decl",
true);
+
xsdValidator.setFeature("http://xml.org/sax/features/external-general-entities",
false);
+
xsdValidator.setFeature("http://xml.org/sax/features/external-parameter-entities",
false);
+ xsdValidator.setProperty(
+ "http://apache.org/xml/properties/internal/entity-resolver",
+ (XMLEntityResolver)
+ xmlResourceIdentifier -> {
+ String message =
+ BaseMessages.getString(PKG,
"XsdValidator.Exception.DisallowedDocType");
+ throw new IOException(message);
+ });
+ }
+ if (schema != data.schemaFromDocuments) {
+ data.validators.put(schema, xsdValidator);
+ }
+ return xsdValidator;
+ }
+
private Schema getXsdSchema(String xsdfilename, SchemaFactory
factoryXSDValidator)
throws SAXException, HopTransformException {
Schema schematXSD;
diff --git
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
index d6ee5e12d4..65622269e7 100644
---
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
+++
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
@@ -17,6 +17,12 @@
package org.apache.hop.pipeline.transforms.xml.xsdvalidator;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
@@ -27,6 +33,34 @@ public class XsdValidatorData extends BaseTransformData
implements ITransformDat
public int xsdindex;
public IRowMeta outputRowMeta;
+ /** The most XSD file names whose compiled schema is kept per copy. */
+ public static final int MAX_CACHED_SCHEMAS = 100;
+
+ /** Created on first use and reused: creating a SchemaFactory scans the
classpath. */
+ public SchemaFactory schemaFactory;
+
+ /** The schema without an XSD file, which uses the location hints in each
document. */
+ public Schema schemaFromDocuments;
+
+ /**
+ * One validator per compiled schema, configured once. Not for {@link
#schemaFromDocuments}: a
+ * validator keeps the schemas loaded from earlier documents' location hints.
+ */
+ public final Map<Schema, Validator> validators = new IdentityHashMap<>();
+
+ /** Compiled schemas by XSD file name, the least recently used dropped
beyond the limit. */
+ public final Map<String, Schema> schemas =
+ new LinkedHashMap<>(16, 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, Schema> eldest) {
+ if (size() > MAX_CACHED_SCHEMAS) {
+ validators.remove(eldest.getValue());
+ return true;
+ }
+ return false;
+ }
+ };
+
public XsdValidatorData() {
super();
xmlindex = -1;
diff --git
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
new file mode 100644
index 0000000000..110981b23f
--- /dev/null
+++
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.xml.xmljoin;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.xml.XmlParserFactoryProducer;
+import org.apache.hop.pipeline.transform.ITransformIOMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.stream.IStream;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+
+/**
+ * XML join of several source rows into one target document. The transform
runs in the test thread,
+ * so the parser factory calls can be counted: one for the target, one for all
source rows.
+ */
+class XmlJoinSeveralRowsTest {
+
+ private TransformMockHelper<XmlJoinMeta, XmlJoinData> tmh;
+
+ @BeforeEach
+ void setUp() {
+ tmh = new TransformMockHelper<>("XmlJoin", XmlJoinMeta.class,
XmlJoinData.class);
+ when(tmh.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(tmh.iLogChannel);
+ when(tmh.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ tmh.cleanUp();
+ }
+
+ @Test
+ void simpleJoinOfSeveralSourceRows() throws Exception {
+ XmlJoinMeta meta = tmh.iTransformMeta;
+ when(meta.getTargetXPath()).thenReturn("//root");
+
+ try (MockedStatic<XmlParserFactoryProducer> producer =
+ mockStatic(XmlParserFactoryProducer.class, CALLS_REAL_METHODS)) {
+ String xml =
+ join(
+ "<root/>",
+ sourceRowMeta("sourceField"),
+ List.of(
+ new Object[] {"<a>1</a>"}, new Object[] {"<a>2</a>"}, new
Object[] {"<a>3</a>"}));
+
+ assertTrue(xml.endsWith("<root><a>1</a><a>2</a><a>3</a></root>"), xml);
+ // One parser factory for the target document, one for all source rows.
+ producer.verify(XmlParserFactoryProducer::createSecureDocBuilderFactory,
times(2));
+ }
+ }
+
+ @Test
+ void complexJoinPlacesEachSourceRowByItsCompareValue() throws Exception {
+ XmlJoinMeta meta = tmh.iTransformMeta;
+ when(meta.getTargetXPath()).thenReturn("//item[@id='?']");
+ when(meta.isComplexJoin()).thenReturn(true);
+ when(meta.getJoinCompareField()).thenReturn("id");
+
+ String xml =
+ join(
+ "<root><item id=\"1\"/><item id=\"2\"/></root>",
+ sourceRowMeta("sourceField", "id"),
+ List.of(
+ new Object[] {"<v>x</v>", "2"},
+ new Object[] {"<v>y</v>", "1"},
+ new Object[] {"<v>z</v>", "2"}));
+
+ assertTrue(
+ xml.endsWith(
+ "<root><item id=\"1\"><v>y</v></item><item
id=\"2\"><v>x</v><v>z</v></item></root>"),
+ xml);
+ }
+
+ private static IRowMeta sourceRowMeta(String... fields) {
+ IRowMeta rowMeta = new RowMeta();
+ for (String field : fields) {
+ rowMeta.addValueMeta(new ValueMetaString(field));
+ }
+ return rowMeta;
+ }
+
+ /** Runs the join of one target row and the source rows; returns the XML of
the output row. */
+ private String join(String targetXml, IRowMeta sourceRowMeta, List<Object[]>
sourceRows)
+ throws Exception {
+ XmlJoinMeta meta = tmh.iTransformMeta;
+ when(meta.getSourceXmlTransform()).thenReturn("source");
+ when(meta.getTargetXmlTransform()).thenReturn("target");
+ when(meta.getSourceXmlField()).thenReturn("sourceField");
+ when(meta.getTargetXmlField()).thenReturn("targetField");
+ when(meta.getValueXmlField()).thenReturn("resultField");
+
+ ITransformIOMeta transformIOMeta = mock(ITransformIOMeta.class);
+ when(meta.getTransformIOMeta()).thenReturn(transformIOMeta);
+ IStream targetStream = mock(IStream.class);
+ IStream sourceStream = mock(IStream.class);
+ when(targetStream.getTransformName()).thenReturn("target");
+ when(sourceStream.getTransformName()).thenReturn("source");
+ when(transformIOMeta.getInfoStreams()).thenReturn(List.of(targetStream,
sourceStream));
+
+ XmlJoin transform =
+ spy(
+ new XmlJoin(
+ tmh.transformMeta, meta, new XmlJoinData(), 0,
tmh.pipelineMeta, tmh.pipeline));
+
+ IRowSet targetRowSet = tmh.getMockInputRowSet(new Object[][]
{{targetXml}});
+ IRowMeta targetRowMeta = new RowMeta();
+ targetRowMeta.addValueMeta(new ValueMetaString("targetField"));
+ when(targetRowSet.getRowMeta()).thenReturn(targetRowMeta);
+ IRowSet sourceRowSet = tmh.getMockInputRowSet(sourceRows);
+ when(sourceRowSet.getRowMeta()).thenReturn(sourceRowMeta);
+
+ List<IRowSet> inputRowSets = new ArrayList<>();
+ inputRowSets.add(sourceRowSet);
+ inputRowSets.add(targetRowSet);
+ transform.setInputRowSets(inputRowSets);
+ doReturn(sourceRowSet).when(transform).findInputRowSet("source");
+ doReturn(targetRowSet).when(transform).findInputRowSet("target");
+
+ transform.init();
+ List<String> output = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws
HopTransformException {
+ output.add((String) row[0]);
+ }
+ });
+
+ while (transform.processRow()) {
+ // one call per source row
+ }
+ assertEquals(0, transform.getErrors());
+ assertEquals(1, output.size());
+ return output.get(0);
+ }
+}
diff --git
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
new file mode 100644
index 0000000000..bc6bab8429
--- /dev/null
+++
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.xml.xsdvalidator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.xml.XmlParserFactoryProducer;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.MockedStatic;
+
+/**
+ * XSD validator over several rows: the schema factory is created and each XSD
compiled once per
+ * copy, and every row still gets the same result and message as before. The
transform runs in the
+ * test thread, so the factory calls can be counted.
+ */
+class XsdValidatorSchemaCacheTest {
+
+ private static final String XSD_ORDER =
+ "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'><xs:element
name='order'>"
+ + "<xs:complexType><xs:sequence><xs:element name='id'
type='xs:int'/></xs:sequence>"
+ + "</xs:complexType></xs:element></xs:schema>";
+ private static final String XSD_ITEM = XSD_ORDER.replace("'order'",
"'item'");
+ private static final String VALID_ORDER = "<order><id>1</id></order>";
+ private static final String INVALID_ORDER = "<order><id>x</id></order>";
+ private static final String VALID_ITEM = "<item><id>2</id></item>";
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<XsdValidatorMeta, XsdValidatorData> tmh;
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ tmh =
+ new TransformMockHelper<>("XSD validator", XsdValidatorMeta.class,
XsdValidatorData.class);
+ when(tmh.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(tmh.iLogChannel);
+ when(tmh.pipeline.isRunning()).thenReturn(true);
+ }
+
+ @AfterEach
+ void tearDown() {
+ tmh.cleanUp();
+ }
+
+ @Test
+ void fixedXsdFileCreatesTheFactoryOnce() throws Exception {
+ String xsd = write("order.xsd", XSD_ORDER);
+ XsdValidatorMeta meta = meta(false);
+ meta.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+ meta.setXsdFilename(xsd);
+
+ try (MockedStatic<XmlParserFactoryProducer> producer =
+ mockStatic(XmlParserFactoryProducer.class, CALLS_REAL_METHODS)) {
+ Run run = new Run(meta, xmlOnly(), rows(VALID_ORDER, INVALID_ORDER,
VALID_ORDER));
+ run.all();
+
+ assertEquals(List.of(true, false, true), run.results());
+ assertNull(run.messages().get(0));
+ assertNotNull(run.messages().get(1));
+ assertNull(run.messages().get(2));
+ producer.verify(
+ () ->
XmlParserFactoryProducer.createSecureSchemaFactory(anyString()), times(1));
+ }
+ }
+
+ /**
+ * Each XSD from the field is compiled the first time it is used.
Overwriting the files after that
+ * no longer changes the results: the compiled schemas are kept for the run.
+ */
+ @Test
+ void xsdFromFieldIsCompiledOncePerFile() throws Exception {
+ Path orderXsd = tempDir.resolve("order.xsd");
+ Path itemXsd = tempDir.resolve("item.xsd");
+ Files.writeString(orderXsd, XSD_ORDER, StandardCharsets.UTF_8);
+ Files.writeString(itemXsd, XSD_ITEM, StandardCharsets.UTF_8);
+ XsdValidatorMeta meta = meta(false);
+ meta.setXsdSource(XsdValidatorMeta.SPECIFY_FIELDNAME);
+ meta.setXsdDefinedField("xsd");
+
+ Run run =
+ new Run(
+ meta,
+ xmlAndXsd(),
+ List.of(
+ new Object[] {VALID_ORDER, orderXsd.toString()},
+ new Object[] {VALID_ITEM, itemXsd.toString()},
+ new Object[] {INVALID_ORDER, orderXsd.toString()},
+ new Object[] {VALID_ITEM, itemXsd.toString()},
+ new Object[] {VALID_ORDER, orderXsd.toString()}));
+ run.next();
+ run.next();
+ Files.writeString(orderXsd, "not a schema", StandardCharsets.UTF_8);
+ Files.writeString(itemXsd, "not a schema", StandardCharsets.UTF_8);
+ run.all();
+
+ assertEquals(List.of(true, true, false, true, true), run.results());
+ assertNotNull(run.messages().get(2));
+ }
+
+ /** A schema that doesn't compile is not kept: every row that uses it
reports the error. */
+ @Test
+ void xsdThatFailsToCompileReportsTheErrorOnEveryRow() throws Exception {
+ String broken = write("broken.xsd", "<xs:schema
xmlns:xs='http://www.w3.org/2001/XMLSchema'>");
+ String order = write("order.xsd", XSD_ORDER);
+ XsdValidatorMeta meta = meta(false);
+ meta.setXsdSource(XsdValidatorMeta.SPECIFY_FIELDNAME);
+ meta.setXsdDefinedField("xsd");
+
+ Run run =
+ new Run(
+ meta,
+ xmlAndXsd(),
+ List.of(
+ new Object[] {VALID_ORDER, broken},
+ new Object[] {VALID_ORDER, order},
+ new Object[] {VALID_ORDER, broken}));
+ run.all();
+
+ assertEquals(List.of(false, true, false), run.results());
+ assertNotNull(run.messages().get(0));
+ assertEquals(run.messages().get(0), run.messages().get(2));
+ }
+
+ /**
+ * Without an XSD file each document names its own schema. A row is
validated against the schema
+ * in its own hint, never against one loaded for an earlier row.
+ */
+ @Test
+ void schemaFromDocumentHintsIsResolvedForEveryRow() throws Exception {
+ String intXsd = Path.of(write("int.xsd", XSD_ORDER)).toUri().toString();
+ String stringXsd =
+ Path.of(write("string.xsd", XSD_ORDER.replace("xs:int",
"xs:string"))).toUri().toString();
+ XsdValidatorMeta meta = meta(true);
+ meta.setXsdSource(XsdValidatorMeta.NO_NEED);
+
+ Run run =
+ new Run(
+ meta,
+ xmlOnly(),
+ rows(
+ withHint(stringXsd, "x"),
+ withHint(intXsd, "x"),
+ withHint(stringXsd, "x"),
+ INVALID_ORDER,
+ withHint(intXsd, "1"),
+ withHint(stringXsd, "y"),
+ withHint(intXsd, "y")));
+ run.all();
+
+ assertEquals(List.of(true, false, true, false, true, true, false),
run.results());
+ }
+
+ private static String withHint(String xsdUri, String id) {
+ return "<order xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'"
+ + " xsi:noNamespaceSchemaLocation='"
+ + xsdUri
+ + "'><id>"
+ + id
+ + "</id></order>";
+ }
+
+ @Test
+ void externalEntitiesAreStillRefusedOnEveryRow() throws Exception {
+ String xsd = write("order.xsd", XSD_ORDER);
+ String value = write("value.txt", "5");
+ String withEntity =
+ "<?xml version='1.0'?><!DOCTYPE order [<!ENTITY v SYSTEM '"
+ + Path.of(value).toUri()
+ + "'>]><order><id>&v;</id></order>";
+
+ XsdValidatorMeta secure = meta(false);
+ secure.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+ secure.setXsdFilename(xsd);
+ Run refused = new Run(secure, xmlOnly(), rows(withEntity, VALID_ORDER,
withEntity));
+ refused.all();
+ assertEquals(List.of(false, true, false), refused.results());
+ assertNotNull(refused.messages().get(0));
+ assertEquals(refused.messages().get(0), refused.messages().get(2));
+
+ tearDown();
+ setUp();
+ XsdValidatorMeta allowed = meta(true);
+ allowed.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+ allowed.setXsdFilename(xsd);
+ Run accepted = new Run(allowed, xmlOnly(), rows(withEntity, withEntity));
+ accepted.all();
+ assertEquals(List.of(true, true), accepted.results());
+ }
+
+ private XsdValidatorMeta meta(boolean allowExternalEntities) {
+ XsdValidatorMeta meta = new XsdValidatorMeta();
+ meta.setDefault();
+ meta.setXmlSourceFile(false);
+ meta.setXmlStream("xml");
+ meta.setAddValidationMessage(true);
+ meta.setAllowExternalEntities(allowExternalEntities);
+ return meta;
+ }
+
+ private String write(String name, String content) throws Exception {
+ Path file = tempDir.resolve(name);
+ Files.writeString(file, content, StandardCharsets.UTF_8);
+ return file.toString();
+ }
+
+ private static IRowMeta xmlOnly() {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("xml"));
+ return rowMeta;
+ }
+
+ private static IRowMeta xmlAndXsd() {
+ IRowMeta rowMeta = xmlOnly();
+ rowMeta.addValueMeta(new ValueMetaString("xsd"));
+ return rowMeta;
+ }
+
+ private static List<Object[]> rows(String... xml) {
+ List<Object[]> rows = new ArrayList<>();
+ for (String value : xml) {
+ rows.add(new Object[] {value});
+ }
+ return rows;
+ }
+
+ /** One transform copy, fed row by row; collects the result flag and message
of each row. */
+ private class Run {
+ private final XsdValidator transform;
+ private final int resultIndex;
+ private final List<Boolean> results = new ArrayList<>();
+ private final List<String> messages = new ArrayList<>();
+ private boolean done;
+
+ Run(XsdValidatorMeta meta, IRowMeta rowMeta, List<Object[]> rows) {
+ transform =
+ new XsdValidator(
+ tmh.transformMeta, meta, new XsdValidatorData(), 0,
tmh.pipelineMeta, tmh.pipeline);
+ transform.init();
+ IRowSet rowSet = tmh.getMockInputRowSet(rows);
+ when(rowSet.getRowMeta()).thenReturn(rowMeta);
+ transform.addRowSetToInputRowSets(rowSet);
+ resultIndex = rowMeta.size();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta outputRowMeta, Object[] row)
+ throws HopTransformException {
+ results.add((Boolean) row[resultIndex]);
+ messages.add((String) row[resultIndex + 1]);
+ }
+ });
+ }
+
+ void next() throws Exception {
+ done = !transform.processRow();
+ }
+
+ void all() throws Exception {
+ while (!done) {
+ next();
+ }
+ assertEquals(0, transform.getErrors());
+ assertFalse(results.contains(null));
+ assertTrue(results.size() == messages.size());
+ }
+
+ List<Boolean> results() {
+ return results;
+ }
+
+ List<String> messages() {
+ return messages;
+ }
+ }
+}