This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new 00c37a87f4 XML join and XSD validator create their XML factories for 
every row, … (#9345)
00c37a87f4 is described below

commit 00c37a87f4f8ce3a84327269210346c3ba154d2f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Mon Nov 16 09:58:16 2026 +0100

    XML join and XSD validator create their XML factories for every row, … 
(#9345)
    
    * XML join and XSD validator create their XML factories for every row, 
fixes #9342
    
    * fix remarks
---
 .../pipeline/transforms/xml/xmljoin/XmlJoin.java   |  27 +-
 .../transforms/xml/xmljoin/XmlJoinData.java        |   8 +
 .../transforms/xml/xsdvalidator/XsdValidator.java  | 130 ++++++---
 .../xml/xsdvalidator/XsdValidatorData.java         |  34 +++
 .../xml/xmljoin/XmlJoinSeveralRowsTest.java        | 174 +++++++++++
 .../xsdvalidator/XsdValidatorSchemaCacheTest.java  | 323 +++++++++++++++++++++
 6 files changed, 657 insertions(+), 39 deletions(-)

diff --git 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
index 4a192fafdb..f6b7c2edc5 100644
--- 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
+++ 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoin.java
@@ -20,6 +20,7 @@ package org.apache.hop.pipeline.transforms.xml.xmljoin;
 import java.io.StringReader;
 import java.io.StringWriter;
 import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
 import javax.xml.transform.dom.DOMSource;
@@ -67,7 +68,11 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta, 
XmlJoinData> {
   @Override
   public boolean processRow() throws HopException {
 
-    XPath xpath = XPathFactory.newInstance().newXPath();
+    // Created once per copy: creating an XPathFactory scans the classpath.
+    if (data.xpath == null) {
+      data.xpath = XPathFactory.newInstance().newXPath();
+    }
+    XPath xpath = data.xpath;
 
     // if first row we do some initializing and process the first row of the 
target XML Transform
     if (first) {
@@ -235,9 +240,8 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta, 
XmlJoinData> {
       String strJoinXML = (String) rJoinSource[data.iSourceXMLField];
 
       try {
-        DocumentBuilder builder =
-            
XmlParserFactoryProducer.createSecureDocBuilderFactory().newDocumentBuilder();
-        Document joinDocument = builder.parse(new InputSource(new 
StringReader(strJoinXML)));
+        Document joinDocument =
+            getSourceDocumentBuilder().parse(new InputSource(new 
StringReader(strJoinXML)));
 
         Node node = 
data.targetDom.importNode(joinDocument.getDocumentElement(), true);
 
@@ -261,6 +265,21 @@ public class XmlJoin extends BaseTransform<XmlJoinMeta, 
XmlJoinData> {
     return true;
   }
 
+  /**
+   * The parser for the source XML fragments, one per copy. Creating the 
secure factory scans the
+   * classpath for every security feature it sets, so it is not done per row. 
reset() returns the
+   * builder to the state the factory gave it, security features included.
+   */
+  private DocumentBuilder getSourceDocumentBuilder() throws 
ParserConfigurationException {
+    if (data.sourceDocumentBuilder == null) {
+      data.sourceDocumentBuilder =
+          
XmlParserFactoryProducer.createSecureDocBuilderFactory().newDocumentBuilder();
+    } else {
+      data.sourceDocumentBuilder.reset();
+    }
+    return data.sourceDocumentBuilder;
+  }
+
   @Override
   public boolean init() {
     if (!super.init()) {
diff --git 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
index bcd1ffcc4f..eda97c17a3 100644
--- 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
+++ 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinData.java
@@ -17,6 +17,8 @@
 
 package org.apache.hop.pipeline.transforms.xml.xmljoin;
 
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.xpath.XPath;
 import org.apache.hop.core.IRowSet;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.pipeline.transform.BaseTransformData;
@@ -45,6 +47,12 @@ public class XmlJoinData extends BaseTransformData 
implements ITransformData {
   public int iSourceXMLField = -1;
   public int iCompareFieldID = -1;
 
+  /** Created on the first row, reused for every row of this copy. */
+  public XPath xpath;
+
+  /** Parses the source XML fragments; created on the first source row, reused 
afterwards. */
+  public DocumentBuilder sourceDocumentBuilder;
+
   /** */
   public XmlJoinData() {
     super();
diff --git 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
index 4aa6fbc132..c28cbb81f7 100644
--- 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
+++ 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidator.java
@@ -127,51 +127,34 @@ public class XsdValidator extends 
BaseTransform<XsdValidatorMeta, XsdValidatorDa
         xsdfilename = getInputRowMeta().getString(row, data.xsdindex);
       }
 
-      // Get XSD file
+      // A compiled schema is kept per copy, by XSD file name. Only resolve 
the file when the
+      // schema still has to be compiled.
+      Schema cachedSchema = getCachedSchema(xsdfilename);
       FileObject xsdfile =
-          StringUtils.isNotEmpty(xsdfilename) ? 
HopVfs.getFileObject(xsdfilename, variables) : null;
+          cachedSchema == null && StringUtils.isNotEmpty(xsdfilename)
+              ? HopVfs.getFileObject(xsdfilename, variables)
+              : null;
 
       try {
 
-        // The factory resolves the schema document itself, so it needs the 
same external-access
-        // restrictions as the validator it produces. Honour the transform's 
own opt-in so a
-        // pipeline that deliberately relies on remote schemas keeps working.
-        SchemaFactory factoryXSDValidator =
-            meta.isAllowExternalEntities()
-                ? SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
-                : XmlParserFactoryProducer.createSecureSchemaFactory(
-                    XMLConstants.W3C_XML_SCHEMA_NS_URI);
-
         // Get XML stream
         Source sourceXML = getSourceXML(getInputRowMeta().getString(row, 
data.xmlindex));
 
-        // create the schema
+        // The compiled schema: from the cache, or compiled now and kept
         Schema schematXSD =
-            xsdfile != null
-                ? getXsdSchemaFromFile(xsdfile, factoryXSDValidator)
-                : getXsdSchema(xsdfilename, factoryXSDValidator);
-
-        // Create XSDValidator
-        Validator xsdValidator = schematXSD.newValidator();
-
-        // Prevent against XML Entity Expansion (XEE) attacks.
-        // 
https://www.owasp.org/index.php/XML_Security_Cheat_Sheet#XML_Entity_Expansion
-        if (!meta.isAllowExternalEntities()) {
-          
xsdValidator.setFeature("http://apache.org/xml/features/disallow-doctype-decl";, 
true);
-          
xsdValidator.setFeature("http://xml.org/sax/features/external-general-entities";,
 false);
-          
xsdValidator.setFeature("http://xml.org/sax/features/external-parameter-entities";,
 false);
-          xsdValidator.setProperty(
-              "http://apache.org/xml/properties/internal/entity-resolver";,
-              (XMLEntityResolver)
-                  xmlResourceIdentifier -> {
-                    String message =
-                        BaseMessages.getString(PKG, 
"XsdValidator.Exception.DisallowedDocType");
-                    throw new IOException(message);
-                  });
-        }
+            cachedSchema != null ? cachedSchema : compileSchema(xsdfilename, 
xsdfile);
+
+        // The validator of this schema, configured once
+        Validator xsdValidator = getValidator(schematXSD);
 
         // Validate XML / XSD
-        xsdValidator.validate(sourceXML);
+        try {
+          xsdValidator.validate(sourceXML);
+        } catch (SAXException | IOException e) {
+          // Don't reuse a validator that failed; the next row gets a new one.
+          data.validators.remove(schematXSD);
+          throw e;
+        }
 
         isvalid = true;
 
@@ -246,6 +229,83 @@ public class XsdValidator extends 
BaseTransform<XsdValidatorMeta, XsdValidatorDa
     return true;
   }
 
+  /**
+   * The schema factory of this copy, created on first use. The factory 
resolves the schema document
+   * itself, so it needs the same external-access restrictions as the 
validator it produces. Honour
+   * the transform's own opt-in so a pipeline that deliberately relies on 
remote schemas keeps
+   * working. A SchemaFactory is not thread-safe, which is why it is kept per 
copy.
+   */
+  private SchemaFactory getSchemaFactory() throws SAXException {
+    if (data.schemaFactory == null) {
+      data.schemaFactory =
+          meta.isAllowExternalEntities()
+              ? SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
+              : XmlParserFactoryProducer.createSecureSchemaFactory(
+                  XMLConstants.W3C_XML_SCHEMA_NS_URI);
+    }
+    return data.schemaFactory;
+  }
+
+  /** The schema compiled earlier for this XSD file name, or null. */
+  private Schema getCachedSchema(String xsdfilename) {
+    if (StringUtils.isEmpty(xsdfilename)) {
+      return data.schemaFromDocuments;
+    }
+    return data.schemas.get(xsdfilename);
+  }
+
+  /**
+   * Compiles the schema and keeps it for the next rows. A schema that fails 
to compile is not kept,
+   * so every row that uses it reports the error.
+   */
+  private Schema compileSchema(String xsdfilename, FileObject xsdfile)
+      throws SAXException, HopTransformException, FileSystemException {
+    if (xsdfile == null) {
+      Schema schema = getXsdSchema(xsdfilename, getSchemaFactory());
+      data.schemaFromDocuments = schema;
+      return schema;
+    }
+    Schema schema = getXsdSchemaFromFile(xsdfile, getSchemaFactory());
+    data.schemas.put(xsdfilename, schema);
+    return schema;
+  }
+
+  /**
+   * The validator of this schema for this copy, configured once. A Validator 
is not thread-safe,
+   * which is why it is kept per copy. reset() is not used: it would drop the 
features set here.
+   *
+   * <p>The schema without an XSD file is the exception: its validator keeps 
the schemas it loaded
+   * from the location hints of earlier documents and would validate later 
documents against them,
+   * whatever their own hints say. That schema gets a new validator for every 
row.
+   */
+  private Validator getValidator(Schema schema) throws SAXException {
+    Validator xsdValidator = data.validators.get(schema);
+    if (xsdValidator != null) {
+      return xsdValidator;
+    }
+    xsdValidator = schema.newValidator();
+
+    // Prevent against XML Entity Expansion (XEE) attacks.
+    // 
https://www.owasp.org/index.php/XML_Security_Cheat_Sheet#XML_Entity_Expansion
+    if (!meta.isAllowExternalEntities()) {
+      
xsdValidator.setFeature("http://apache.org/xml/features/disallow-doctype-decl";, 
true);
+      
xsdValidator.setFeature("http://xml.org/sax/features/external-general-entities";,
 false);
+      
xsdValidator.setFeature("http://xml.org/sax/features/external-parameter-entities";,
 false);
+      xsdValidator.setProperty(
+          "http://apache.org/xml/properties/internal/entity-resolver";,
+          (XMLEntityResolver)
+              xmlResourceIdentifier -> {
+                String message =
+                    BaseMessages.getString(PKG, 
"XsdValidator.Exception.DisallowedDocType");
+                throw new IOException(message);
+              });
+    }
+    if (schema != data.schemaFromDocuments) {
+      data.validators.put(schema, xsdValidator);
+    }
+    return xsdValidator;
+  }
+
   private Schema getXsdSchema(String xsdfilename, SchemaFactory 
factoryXSDValidator)
       throws SAXException, HopTransformException {
     Schema schematXSD;
diff --git 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
index d6ee5e12d4..65622269e7 100644
--- 
a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
+++ 
b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorData.java
@@ -17,6 +17,12 @@
 
 package org.apache.hop.pipeline.transforms.xml.xsdvalidator;
 
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.pipeline.transform.BaseTransformData;
 import org.apache.hop.pipeline.transform.ITransformData;
@@ -27,6 +33,34 @@ public class XsdValidatorData extends BaseTransformData 
implements ITransformDat
   public int xsdindex;
   public IRowMeta outputRowMeta;
 
+  /** The most XSD file names whose compiled schema is kept per copy. */
+  public static final int MAX_CACHED_SCHEMAS = 100;
+
+  /** Created on first use and reused: creating a SchemaFactory scans the 
classpath. */
+  public SchemaFactory schemaFactory;
+
+  /** The schema without an XSD file, which uses the location hints in each 
document. */
+  public Schema schemaFromDocuments;
+
+  /**
+   * One validator per compiled schema, configured once. Not for {@link 
#schemaFromDocuments}: a
+   * validator keeps the schemas loaded from earlier documents' location hints.
+   */
+  public final Map<Schema, Validator> validators = new IdentityHashMap<>();
+
+  /** Compiled schemas by XSD file name, the least recently used dropped 
beyond the limit. */
+  public final Map<String, Schema> schemas =
+      new LinkedHashMap<>(16, 0.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, Schema> eldest) {
+          if (size() > MAX_CACHED_SCHEMAS) {
+            validators.remove(eldest.getValue());
+            return true;
+          }
+          return false;
+        }
+      };
+
   public XsdValidatorData() {
     super();
     xmlindex = -1;
diff --git 
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
 
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
new file mode 100644
index 0000000000..110981b23f
--- /dev/null
+++ 
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xmljoin/XmlJoinSeveralRowsTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.xml.xmljoin;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.xml.XmlParserFactoryProducer;
+import org.apache.hop.pipeline.transform.ITransformIOMeta;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.stream.IStream;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+
+/**
+ * XML join of several source rows into one target document. The transform 
runs in the test thread,
+ * so the parser factory calls can be counted: one for the target, one for all 
source rows.
+ */
+class XmlJoinSeveralRowsTest {
+
+  private TransformMockHelper<XmlJoinMeta, XmlJoinData> tmh;
+
+  @BeforeEach
+  void setUp() {
+    tmh = new TransformMockHelper<>("XmlJoin", XmlJoinMeta.class, 
XmlJoinData.class);
+    when(tmh.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(tmh.iLogChannel);
+    when(tmh.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    tmh.cleanUp();
+  }
+
+  @Test
+  void simpleJoinOfSeveralSourceRows() throws Exception {
+    XmlJoinMeta meta = tmh.iTransformMeta;
+    when(meta.getTargetXPath()).thenReturn("//root");
+
+    try (MockedStatic<XmlParserFactoryProducer> producer =
+        mockStatic(XmlParserFactoryProducer.class, CALLS_REAL_METHODS)) {
+      String xml =
+          join(
+              "<root/>",
+              sourceRowMeta("sourceField"),
+              List.of(
+                  new Object[] {"<a>1</a>"}, new Object[] {"<a>2</a>"}, new 
Object[] {"<a>3</a>"}));
+
+      assertTrue(xml.endsWith("<root><a>1</a><a>2</a><a>3</a></root>"), xml);
+      // One parser factory for the target document, one for all source rows.
+      producer.verify(XmlParserFactoryProducer::createSecureDocBuilderFactory, 
times(2));
+    }
+  }
+
+  @Test
+  void complexJoinPlacesEachSourceRowByItsCompareValue() throws Exception {
+    XmlJoinMeta meta = tmh.iTransformMeta;
+    when(meta.getTargetXPath()).thenReturn("//item[@id='?']");
+    when(meta.isComplexJoin()).thenReturn(true);
+    when(meta.getJoinCompareField()).thenReturn("id");
+
+    String xml =
+        join(
+            "<root><item id=\"1\"/><item id=\"2\"/></root>",
+            sourceRowMeta("sourceField", "id"),
+            List.of(
+                new Object[] {"<v>x</v>", "2"},
+                new Object[] {"<v>y</v>", "1"},
+                new Object[] {"<v>z</v>", "2"}));
+
+    assertTrue(
+        xml.endsWith(
+            "<root><item id=\"1\"><v>y</v></item><item 
id=\"2\"><v>x</v><v>z</v></item></root>"),
+        xml);
+  }
+
+  private static IRowMeta sourceRowMeta(String... fields) {
+    IRowMeta rowMeta = new RowMeta();
+    for (String field : fields) {
+      rowMeta.addValueMeta(new ValueMetaString(field));
+    }
+    return rowMeta;
+  }
+
+  /** Runs the join of one target row and the source rows; returns the XML of 
the output row. */
+  private String join(String targetXml, IRowMeta sourceRowMeta, List<Object[]> 
sourceRows)
+      throws Exception {
+    XmlJoinMeta meta = tmh.iTransformMeta;
+    when(meta.getSourceXmlTransform()).thenReturn("source");
+    when(meta.getTargetXmlTransform()).thenReturn("target");
+    when(meta.getSourceXmlField()).thenReturn("sourceField");
+    when(meta.getTargetXmlField()).thenReturn("targetField");
+    when(meta.getValueXmlField()).thenReturn("resultField");
+
+    ITransformIOMeta transformIOMeta = mock(ITransformIOMeta.class);
+    when(meta.getTransformIOMeta()).thenReturn(transformIOMeta);
+    IStream targetStream = mock(IStream.class);
+    IStream sourceStream = mock(IStream.class);
+    when(targetStream.getTransformName()).thenReturn("target");
+    when(sourceStream.getTransformName()).thenReturn("source");
+    when(transformIOMeta.getInfoStreams()).thenReturn(List.of(targetStream, 
sourceStream));
+
+    XmlJoin transform =
+        spy(
+            new XmlJoin(
+                tmh.transformMeta, meta, new XmlJoinData(), 0, 
tmh.pipelineMeta, tmh.pipeline));
+
+    IRowSet targetRowSet = tmh.getMockInputRowSet(new Object[][] 
{{targetXml}});
+    IRowMeta targetRowMeta = new RowMeta();
+    targetRowMeta.addValueMeta(new ValueMetaString("targetField"));
+    when(targetRowSet.getRowMeta()).thenReturn(targetRowMeta);
+    IRowSet sourceRowSet = tmh.getMockInputRowSet(sourceRows);
+    when(sourceRowSet.getRowMeta()).thenReturn(sourceRowMeta);
+
+    List<IRowSet> inputRowSets = new ArrayList<>();
+    inputRowSets.add(sourceRowSet);
+    inputRowSets.add(targetRowSet);
+    transform.setInputRowSets(inputRowSets);
+    doReturn(sourceRowSet).when(transform).findInputRowSet("source");
+    doReturn(targetRowSet).when(transform).findInputRowSet("target");
+
+    transform.init();
+    List<String> output = new ArrayList<>();
+    transform.addRowListener(
+        new RowAdapter() {
+          @Override
+          public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws 
HopTransformException {
+            output.add((String) row[0]);
+          }
+        });
+
+    while (transform.processRow()) {
+      // one call per source row
+    }
+    assertEquals(0, transform.getErrors());
+    assertEquals(1, output.size());
+    return output.get(0);
+  }
+}
diff --git 
a/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
 
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
new file mode 100644
index 0000000000..bc6bab8429
--- /dev/null
+++ 
b/plugins/transforms/xml/src/test/java/org/apache/hop/pipeline/transforms/xml/xsdvalidator/XsdValidatorSchemaCacheTest.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.xml.xsdvalidator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.IRowSet;
+import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.xml.XmlParserFactoryProducer;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.MockedStatic;
+
+/**
+ * XSD validator over several rows: the schema factory is created and each XSD 
compiled once per
+ * copy, and every row still gets the same result and message as before. The 
transform runs in the
+ * test thread, so the factory calls can be counted.
+ */
+class XsdValidatorSchemaCacheTest {
+
+  private static final String XSD_ORDER =
+      "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'><xs:element 
name='order'>"
+          + "<xs:complexType><xs:sequence><xs:element name='id' 
type='xs:int'/></xs:sequence>"
+          + "</xs:complexType></xs:element></xs:schema>";
+  private static final String XSD_ITEM = XSD_ORDER.replace("'order'", 
"'item'");
+  private static final String VALID_ORDER = "<order><id>1</id></order>";
+  private static final String INVALID_ORDER = "<order><id>x</id></order>";
+  private static final String VALID_ITEM = "<item><id>2</id></item>";
+
+  @TempDir Path tempDir;
+
+  private TransformMockHelper<XsdValidatorMeta, XsdValidatorData> tmh;
+
+  @BeforeAll
+  static void initHop() throws Exception {
+    HopEnvironment.init();
+  }
+
+  @BeforeEach
+  void setUp() {
+    tmh =
+        new TransformMockHelper<>("XSD validator", XsdValidatorMeta.class, 
XsdValidatorData.class);
+    when(tmh.logChannelFactory.create(any(), any(ILoggingObject.class)))
+        .thenReturn(tmh.iLogChannel);
+    when(tmh.pipeline.isRunning()).thenReturn(true);
+  }
+
+  @AfterEach
+  void tearDown() {
+    tmh.cleanUp();
+  }
+
+  @Test
+  void fixedXsdFileCreatesTheFactoryOnce() throws Exception {
+    String xsd = write("order.xsd", XSD_ORDER);
+    XsdValidatorMeta meta = meta(false);
+    meta.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+    meta.setXsdFilename(xsd);
+
+    try (MockedStatic<XmlParserFactoryProducer> producer =
+        mockStatic(XmlParserFactoryProducer.class, CALLS_REAL_METHODS)) {
+      Run run = new Run(meta, xmlOnly(), rows(VALID_ORDER, INVALID_ORDER, 
VALID_ORDER));
+      run.all();
+
+      assertEquals(List.of(true, false, true), run.results());
+      assertNull(run.messages().get(0));
+      assertNotNull(run.messages().get(1));
+      assertNull(run.messages().get(2));
+      producer.verify(
+          () -> 
XmlParserFactoryProducer.createSecureSchemaFactory(anyString()), times(1));
+    }
+  }
+
+  /**
+   * Each XSD from the field is compiled the first time it is used. 
Overwriting the files after that
+   * no longer changes the results: the compiled schemas are kept for the run.
+   */
+  @Test
+  void xsdFromFieldIsCompiledOncePerFile() throws Exception {
+    Path orderXsd = tempDir.resolve("order.xsd");
+    Path itemXsd = tempDir.resolve("item.xsd");
+    Files.writeString(orderXsd, XSD_ORDER, StandardCharsets.UTF_8);
+    Files.writeString(itemXsd, XSD_ITEM, StandardCharsets.UTF_8);
+    XsdValidatorMeta meta = meta(false);
+    meta.setXsdSource(XsdValidatorMeta.SPECIFY_FIELDNAME);
+    meta.setXsdDefinedField("xsd");
+
+    Run run =
+        new Run(
+            meta,
+            xmlAndXsd(),
+            List.of(
+                new Object[] {VALID_ORDER, orderXsd.toString()},
+                new Object[] {VALID_ITEM, itemXsd.toString()},
+                new Object[] {INVALID_ORDER, orderXsd.toString()},
+                new Object[] {VALID_ITEM, itemXsd.toString()},
+                new Object[] {VALID_ORDER, orderXsd.toString()}));
+    run.next();
+    run.next();
+    Files.writeString(orderXsd, "not a schema", StandardCharsets.UTF_8);
+    Files.writeString(itemXsd, "not a schema", StandardCharsets.UTF_8);
+    run.all();
+
+    assertEquals(List.of(true, true, false, true, true), run.results());
+    assertNotNull(run.messages().get(2));
+  }
+
+  /** A schema that doesn't compile is not kept: every row that uses it 
reports the error. */
+  @Test
+  void xsdThatFailsToCompileReportsTheErrorOnEveryRow() throws Exception {
+    String broken = write("broken.xsd", "<xs:schema 
xmlns:xs='http://www.w3.org/2001/XMLSchema'>");
+    String order = write("order.xsd", XSD_ORDER);
+    XsdValidatorMeta meta = meta(false);
+    meta.setXsdSource(XsdValidatorMeta.SPECIFY_FIELDNAME);
+    meta.setXsdDefinedField("xsd");
+
+    Run run =
+        new Run(
+            meta,
+            xmlAndXsd(),
+            List.of(
+                new Object[] {VALID_ORDER, broken},
+                new Object[] {VALID_ORDER, order},
+                new Object[] {VALID_ORDER, broken}));
+    run.all();
+
+    assertEquals(List.of(false, true, false), run.results());
+    assertNotNull(run.messages().get(0));
+    assertEquals(run.messages().get(0), run.messages().get(2));
+  }
+
+  /**
+   * Without an XSD file each document names its own schema. A row is 
validated against the schema
+   * in its own hint, never against one loaded for an earlier row.
+   */
+  @Test
+  void schemaFromDocumentHintsIsResolvedForEveryRow() throws Exception {
+    String intXsd = Path.of(write("int.xsd", XSD_ORDER)).toUri().toString();
+    String stringXsd =
+        Path.of(write("string.xsd", XSD_ORDER.replace("xs:int", 
"xs:string"))).toUri().toString();
+    XsdValidatorMeta meta = meta(true);
+    meta.setXsdSource(XsdValidatorMeta.NO_NEED);
+
+    Run run =
+        new Run(
+            meta,
+            xmlOnly(),
+            rows(
+                withHint(stringXsd, "x"),
+                withHint(intXsd, "x"),
+                withHint(stringXsd, "x"),
+                INVALID_ORDER,
+                withHint(intXsd, "1"),
+                withHint(stringXsd, "y"),
+                withHint(intXsd, "y")));
+    run.all();
+
+    assertEquals(List.of(true, false, true, false, true, true, false), 
run.results());
+  }
+
+  private static String withHint(String xsdUri, String id) {
+    return "<order xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'"
+        + " xsi:noNamespaceSchemaLocation='"
+        + xsdUri
+        + "'><id>"
+        + id
+        + "</id></order>";
+  }
+
+  @Test
+  void externalEntitiesAreStillRefusedOnEveryRow() throws Exception {
+    String xsd = write("order.xsd", XSD_ORDER);
+    String value = write("value.txt", "5");
+    String withEntity =
+        "<?xml version='1.0'?><!DOCTYPE order [<!ENTITY v SYSTEM '"
+            + Path.of(value).toUri()
+            + "'>]><order><id>&v;</id></order>";
+
+    XsdValidatorMeta secure = meta(false);
+    secure.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+    secure.setXsdFilename(xsd);
+    Run refused = new Run(secure, xmlOnly(), rows(withEntity, VALID_ORDER, 
withEntity));
+    refused.all();
+    assertEquals(List.of(false, true, false), refused.results());
+    assertNotNull(refused.messages().get(0));
+    assertEquals(refused.messages().get(0), refused.messages().get(2));
+
+    tearDown();
+    setUp();
+    XsdValidatorMeta allowed = meta(true);
+    allowed.setXsdSource(XsdValidatorMeta.SPECIFY_FILENAME);
+    allowed.setXsdFilename(xsd);
+    Run accepted = new Run(allowed, xmlOnly(), rows(withEntity, withEntity));
+    accepted.all();
+    assertEquals(List.of(true, true), accepted.results());
+  }
+
+  private XsdValidatorMeta meta(boolean allowExternalEntities) {
+    XsdValidatorMeta meta = new XsdValidatorMeta();
+    meta.setDefault();
+    meta.setXmlSourceFile(false);
+    meta.setXmlStream("xml");
+    meta.setAddValidationMessage(true);
+    meta.setAllowExternalEntities(allowExternalEntities);
+    return meta;
+  }
+
+  private String write(String name, String content) throws Exception {
+    Path file = tempDir.resolve(name);
+    Files.writeString(file, content, StandardCharsets.UTF_8);
+    return file.toString();
+  }
+
+  private static IRowMeta xmlOnly() {
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaString("xml"));
+    return rowMeta;
+  }
+
+  private static IRowMeta xmlAndXsd() {
+    IRowMeta rowMeta = xmlOnly();
+    rowMeta.addValueMeta(new ValueMetaString("xsd"));
+    return rowMeta;
+  }
+
+  private static List<Object[]> rows(String... xml) {
+    List<Object[]> rows = new ArrayList<>();
+    for (String value : xml) {
+      rows.add(new Object[] {value});
+    }
+    return rows;
+  }
+
+  /** One transform copy, fed row by row; collects the result flag and message 
of each row. */
+  private class Run {
+    private final XsdValidator transform;
+    private final int resultIndex;
+    private final List<Boolean> results = new ArrayList<>();
+    private final List<String> messages = new ArrayList<>();
+    private boolean done;
+
+    Run(XsdValidatorMeta meta, IRowMeta rowMeta, List<Object[]> rows) {
+      transform =
+          new XsdValidator(
+              tmh.transformMeta, meta, new XsdValidatorData(), 0, 
tmh.pipelineMeta, tmh.pipeline);
+      transform.init();
+      IRowSet rowSet = tmh.getMockInputRowSet(rows);
+      when(rowSet.getRowMeta()).thenReturn(rowMeta);
+      transform.addRowSetToInputRowSets(rowSet);
+      resultIndex = rowMeta.size();
+      transform.addRowListener(
+          new RowAdapter() {
+            @Override
+            public void rowWrittenEvent(IRowMeta outputRowMeta, Object[] row)
+                throws HopTransformException {
+              results.add((Boolean) row[resultIndex]);
+              messages.add((String) row[resultIndex + 1]);
+            }
+          });
+    }
+
+    void next() throws Exception {
+      done = !transform.processRow();
+    }
+
+    void all() throws Exception {
+      while (!done) {
+        next();
+      }
+      assertEquals(0, transform.getErrors());
+      assertFalse(results.contains(null));
+      assertTrue(results.size() == messages.size());
+    }
+
+    List<Boolean> results() {
+      return results;
+    }
+
+    List<String> messages() {
+      return messages;
+    }
+  }
+}

Reply via email to