This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 8b6612adac Fix#7000 JSON Input not reading filename from upstream
field after XML load (#7329)
8b6612adac is described below
commit 8b6612adac89450a816416713bec61a526ccc321
Author: Lance <[email protected]>
AuthorDate: Mon Jun 29 17:34:41 2026 +0800
Fix#7000 JSON Input not reading filename from upstream field after XML load
(#7329)
* Fix JSON Input not reading filename from upstream field after XML load
Signed-off-by: lance <[email protected]>
* Fix JSON Input not reading filename from upstream field after XML load
Signed-off-by: lance <[email protected]>
* Fix JSON Input not reading filename from upstream field after XML load
Signed-off-by: lance <[email protected]>
* add integration test
---------
Signed-off-by: lance <[email protected]>
Co-authored-by: Hans Van Akelyen <[email protected]>
---
.../json/0012-json-input-file-in-field.hpl | 168 +++++++++++++++++++++
.../datasets/golden-json-input-file-in-field.csv | 4 +
.../json/main-0012-json-input-file-in-field.hwf | 80 ++++++++++
.../dataset/golden-json-input-file-in-field.json | 25 +++
.../0012-json-input-file-in-field UNIT.json | 33 ++++
.../resources/json-input-file-in-field-it.json | 7 +
.../transforms/jsoninput/JsonInputMeta.java | 21 +++
.../transforms/jsoninput/reader/InputsReader.java | 3 +-
.../jsonnormalize/JsonNormalizeInputMeta.java | 16 ++
.../jsonnormalize/NormalizeInputsReader.java | 3 +-
.../transforms/jsoninput/JsonInputMetaTest.java | 25 +++
.../jsoninput/reader/InputsReaderTests.java | 49 ++++--
.../src/test/resources/json-input-in-fields.xml | 31 ++++
13 files changed, 454 insertions(+), 11 deletions(-)
diff --git a/integration-tests/json/0012-json-input-file-in-field.hpl
b/integration-tests/json/0012-json-input-file-in-field.hpl
new file mode 100644
index 0000000000..08e8aecc04
--- /dev/null
+++ b/integration-tests/json/0012-json-input-file-in-field.hpl
@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0012-json-input-file-in-field</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/06/29 10:00:00.000</created_date>
+ <modified_date>2026/06/29 10:00:00.000</modified_date>
+ </info>
+ <transform>
+ <type>DataGrid</type>
+ <name>Generate filename</name>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <group/>
+ <name>filename</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+
<item>${HOP_UNIT_TESTS_FOLDER}/resources/json-input-file-in-field-it.json</item>
+ </line>
+ </data>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>JsonInput</type>
+ <name>Read JSON from file in field</name>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <limit>0</limit>
+ <valueField>filename</valueField>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>Y</IsAFile>
+ <addresultfile>N</addresultfile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <ignoreMissingPath>Y</ignoreMissingPath>
+ <readurl>N</readurl>
+ <removeSourceField>Y</removeSourceField>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <files>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ </files>
+ <fields>
+ <field>
+ <name>name</name>
+ <length>-1</length>
+ <type>String</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <repeat>N</repeat>
+ <path>$.fighters[*].name</path>
+ </field>
+ <field>
+ <name>power</name>
+ <length>-1</length>
+ <type>Integer</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <repeat>N</repeat>
+ <path>$.fighters[*].power</path>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>Validate</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>560</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Generate filename</from>
+ <to>Read JSON from file in field</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Read JSON from file in field</from>
+ <to>Validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git
a/integration-tests/json/datasets/golden-json-input-file-in-field.csv
b/integration-tests/json/datasets/golden-json-input-file-in-field.csv
new file mode 100644
index 0000000000..f028ddfa23
--- /dev/null
+++ b/integration-tests/json/datasets/golden-json-input-file-in-field.csv
@@ -0,0 +1,4 @@
+name,power
+Goku,9500
+Vegeta,9000
+Gohan,8500
diff --git a/integration-tests/json/main-0012-json-input-file-in-field.hwf
b/integration-tests/json/main-0012-json-input-file-in-field.hwf
new file mode 100644
index 0000000000..5dbf501106
--- /dev/null
+++ b/integration-tests/json/main-0012-json-input-file-in-field.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0012-json-input-file-in-field</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/06/29 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/06/29 10:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0012-json-input-file-in-field UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/json/metadata/dataset/golden-json-input-file-in-field.json
b/integration-tests/json/metadata/dataset/golden-json-input-file-in-field.json
new file mode 100644
index 0000000000..9af32db826
--- /dev/null
+++
b/integration-tests/json/metadata/dataset/golden-json-input-file-in-field.json
@@ -0,0 +1,25 @@
+{
+ "base_filename": "golden-json-input-file-in-field.csv",
+ "virtualPath": "",
+ "name": "golden-json-input-file-in-field",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": -1,
+ "field_name": "power",
+ "field_format": ""
+ }
+ ],
+ "folder_name": "${HOP_DATASETS_FOLDER}"
+}
diff --git
a/integration-tests/json/metadata/unit-test/0012-json-input-file-in-field
UNIT.json
b/integration-tests/json/metadata/unit-test/0012-json-input-file-in-field
UNIT.json
new file mode 100644
index 0000000000..af9ad3abd7
--- /dev/null
+++ b/integration-tests/json/metadata/unit-test/0012-json-input-file-in-field
UNIT.json
@@ -0,0 +1,33 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "name",
+ "data_set_field": "name"
+ },
+ {
+ "transform_field": "power",
+ "data_set_field": "power"
+ }
+ ],
+ "field_order": [
+ "name",
+ "power"
+ ],
+ "data_set_name": "golden-json-input-file-in-field",
+ "transform_name": "Validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0012-json-input-file-in-field UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0012-json-input-file-in-field.hpl"
+}
diff --git a/integration-tests/json/resources/json-input-file-in-field-it.json
b/integration-tests/json/resources/json-input-file-in-field-it.json
new file mode 100644
index 0000000000..3b30d95998
--- /dev/null
+++ b/integration-tests/json/resources/json-input-file-in-field-it.json
@@ -0,0 +1,7 @@
+{
+ "fighters": [
+ { "name": "Goku", "power": 9500 },
+ { "name": "Vegeta", "power": 9000 },
+ { "name": "Gohan", "power": 8500 }
+ ]
+}
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
index 5777950433..e94a14bec4 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
@@ -240,6 +240,7 @@ public class JsonInputMeta extends
BaseFileInputMeta<JsonInput, JsonInputData, B
this.additionalOutputFields = new
BaseFileInputAdditionalFields(m.additionalOutputFields);
this.fileInput = new BaseFileInput(m.fileInput);
m.inputFields.forEach(f -> inputFields.add(new JsonInputField(f)));
+ syncFieldSourceFlags();
}
@Override
@@ -376,6 +377,11 @@ public class JsonInputMeta extends
BaseFileInputMeta<JsonInput, JsonInputData, B
fileInput.setAcceptingField(value);
}
+ /**
+ * Source is from a previous transform
+ *
+ * @param inFields checked
+ */
public void setInFields(boolean inFields) {
this.inFields = inFields;
fileInput.setAcceptingFilenames(inFields);
@@ -404,6 +410,21 @@ public class JsonInputMeta extends
BaseFileInputMeta<JsonInput, JsonInputData, B
@Override
public void convertLegacyXml(Node node) {
convertLegacyXml(getFileInput().getInputFiles(), node);
+
+ // sync field(acceptingFilenames, acceptingField)
+ syncFieldSourceFlags();
+ }
+
+ /**
+ * XML deserialization sets {@link #inFields} and {@link #valueField}
directly on fields, not
+ * through {@link #setInFields(boolean)} / {@link #setFieldValue(String)}.
Keep {@link
+ * BaseFileInput} in sync so runtime file reading uses the upstream field
when configured.
+ */
+ private void syncFieldSourceFlags() {
+ fileInput.setAcceptingFilenames(inFields);
+ if (!Utils.isEmpty(valueField)) {
+ fileInput.setAcceptingField(valueField);
+ }
}
public String getRequiredFilesDesc(String tt) {
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReader.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReader.java
index a5749b6033..a29b9f2e18 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReader.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReader.java
@@ -57,7 +57,8 @@ public class InputsReader implements Iterable<InputStream> {
public Iterator<InputStream> iterator() {
if (!meta.isInFields() || meta.getIsAFile()) {
Iterator<FileObject> files;
- if (meta.getFileInput().isAcceptingFilenames()) {
+ // Source is from a previous transform
+ if (meta.isInFields()) {
// paths from input
files = new FileNamesIterator(transform, errorHandler,
getFieldIterator());
} else {
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java
index 979f632226..bc99846f7a 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputMeta.java
@@ -264,6 +264,7 @@ public class JsonNormalizeInputMeta
this.additionalOutputFields = new
BaseFileInputAdditionalFields(m.additionalOutputFields);
this.fileInput = new BaseFileInput(m.fileInput);
m.inputFields.forEach(f -> inputFields.add(new JsonInputField(f)));
+ syncFieldSourceFlags();
}
@Override
@@ -372,6 +373,21 @@ public class JsonNormalizeInputMeta
@Override
public void convertLegacyXml(Node node) {
convertLegacyXml(getFileInput().getInputFiles(), node);
+
+ // sync field(acceptingFilenames, acceptingField)
+ syncFieldSourceFlags();
+ }
+
+ /**
+ * XML deserialization sets {@link #inFields} and {@link #valueField}
directly on fields, not
+ * through {@link #setInFields(boolean)} / {@link #setFieldValue(String)}.
Keep {@link
+ * BaseFileInput} in sync so runtime file reading uses the upstream field
when configured.
+ */
+ private void syncFieldSourceFlags() {
+ fileInput.setAcceptingFilenames(inFields);
+ if (!Utils.isEmpty(valueField)) {
+ fileInput.setAcceptingField(valueField);
+ }
}
public String getRequiredFilesDesc(String tt) {
diff --git
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/NormalizeInputsReader.java
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/NormalizeInputsReader.java
index 89b1972a29..d4420b57e9 100644
---
a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/NormalizeInputsReader.java
+++
b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/NormalizeInputsReader.java
@@ -62,7 +62,8 @@ public class NormalizeInputsReader implements
Iterable<InputStream> {
public Iterator<InputStream> iterator() {
if (!meta.isInFields() || meta.getIsAFile()) {
Iterator<FileObject> files;
- if (meta.getFileInput().isAcceptingFilenames()) {
+ // Source is from a previous transform
+ if (meta.isInFields()) {
files = new FileNamesIterator(transform, errorHandler,
getFieldIterator());
} else {
if (data.files == null) {
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
index 391d3f282f..b731a86c96 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMetaTest.java
@@ -17,7 +17,9 @@
package org.apache.hop.pipeline.transforms.jsoninput;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -83,6 +85,29 @@ class JsonInputMetaTest {
validate(metaCopy);
}
+ @Test
+ void xmlLoadSyncsAcceptingFilenamesWhenSourceIsInField() throws Exception {
+ Path path =
+ Paths.get(
+
Objects.requireNonNull(getClass().getResource("/json-input-in-fields.xml")).toURI());
+ JsonInputMeta meta = new JsonInputMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString(Files.readString(path),
TransformMeta.XML_TAG),
+ JsonInputMeta.class,
+ meta,
+ new MemoryMetadataProvider());
+
+ assertTrue(meta.isInFields());
+ assertTrue(meta.getIsAFile());
+ assertTrue(
+ meta.getFileInput().isAcceptingFilenames(),
+ "IsInFields=Y must sync acceptingFilenames after XML load");
+ assertEquals(
+ "filename",
+ meta.getFileInput().getAcceptingField(),
+ "valueField must sync acceptingField after XML load");
+ }
+
private static void validate(JsonInputMeta meta) {
assertNotNull(meta.getInputFields());
assertNotNull(meta.getFileInput().getInputFiles());
diff --git
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReaderTests.java
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReaderTests.java
index 2e5d9cb919..e5f1b86ef1 100644
---
a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReaderTests.java
+++
b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsoninput/reader/InputsReaderTests.java
@@ -539,7 +539,8 @@ class InputsReaderTests {
}
@Test
- void iterator_inFieldsAndSourceIsFile_usesFileBranch() throws Exception {
+ void
iterator_inFieldsAndSourceIsFile_readsPathFromFieldEvenWithoutAcceptingFilenamesFlag()
+ throws Exception {
final String path = BASE_RAM_DIR + "isFile.json";
try (FileObject writeTo = HopVfs.getFileObject(path)) {
writeTo.createFile();
@@ -548,25 +549,27 @@ class InputsReaderTests {
}
}
- // New FileObject for the list so size/exists match what onNewFile +
getInputStream see (RAM
- // VFS).
try (FileObject f1 = HopVfs.getFileObject(path)) {
assertTrue(f1.exists(), "file must exist before iterator");
assertTrue(f1.getContent().getSize() > 0, "file must be non-empty before
iterator");
- JsonInputMeta meta = createFileListMeta(List.of(f1));
+ JsonInputMeta meta = new JsonInputMeta();
+ meta.setDefault();
+ ensureMinimalInputField(meta);
meta.setInFields(true);
- // Use setIsAFile (not only Lombok setSourceAFile) so sourceAFile is set
on anonymous
- // JsonInputMeta subclasses.
meta.setIsAFile(true);
+ // Simulate XML deserialization: inFields set without syncing
acceptingFilenames.
meta.getFileInput().setAcceptingFilenames(false);
- meta.setFieldValue("_");
+ meta.setFieldValue("pathcol");
JsonInputData data = new JsonInputData();
- JsonInput jsonInput = newJsonInputWithoutInputRows(meta, data);
+ data.indexSourceField = 0;
+ JsonInput jsonInput =
+ newJsonInputWithStringRows(meta, data, "pathcol", new Object[]
{f1.getName().getURI()});
+
InputsReader reader = new InputsReader(jsonInput, meta, data, new
RecordingErrorHandler());
var it = reader.iterator();
- assertTrue(it.hasNext(), "!inFields || getIsAFile must be true to use
file list iterator");
+ assertTrue(it.hasNext(), "inFields+isAFile must read file path from the
input field");
try (InputStream in = it.next()) {
assertNotNull(in, "expected stream after onNewFile accepted the file");
assertTrue(IOUtils.toString(in,
StandardCharsets.UTF_8).contains("store"));
@@ -575,6 +578,34 @@ class InputsReaderTests {
}
}
+ @Test
+ void iterator_staticFileList_whenNotInFields() throws Exception {
+ final String path = BASE_RAM_DIR + "staticList.json";
+ try (FileObject writeTo = HopVfs.getFileObject(path)) {
+ writeTo.createFile();
+ try (OutputStream os = writeTo.getContent().getOutputStream()) {
+ os.write(BASIC_JSON.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ try (FileObject f1 = HopVfs.getFileObject(path)) {
+ JsonInputMeta meta = createFileListMeta(List.of(f1));
+ meta.setInFields(false);
+ meta.setIsAFile(true);
+
+ JsonInputData data = new JsonInputData();
+ JsonInput jsonInput = newJsonInputWithoutInputRows(meta, data);
+ InputsReader reader = new InputsReader(jsonInput, meta, data, new
RecordingErrorHandler());
+ var it = reader.iterator();
+ assertTrue(it.hasNext(), "static file list must be used when source is
not from a field");
+ try (InputStream in = it.next()) {
+ assertNotNull(in);
+ assertTrue(IOUtils.toString(in,
StandardCharsets.UTF_8).contains("store"));
+ }
+ assertFalse(it.hasNext());
+ }
+ }
+
@Test
void iterator_whenNoFiles_hasNextFalse() {
JsonInputMeta meta = createFileListMeta(Collections.emptyList());
diff --git
a/plugins/transforms/json/src/test/resources/json-input-in-fields.xml
b/plugins/transforms/json/src/test/resources/json-input-in-fields.xml
new file mode 100644
index 0000000000..a7f19a057d
--- /dev/null
+++ b/plugins/transforms/json/src/test/resources/json-input-in-fields.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<transform>
+ <include>N</include>
+ <rownum>N</rownum>
+ <readurl>N</readurl>
+ <removeSourceField>N</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>Y</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <files/>
+ <fields>
+ <field>
+ <name>test_source</name>
+ <path>$.test[*].test_source</path>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>Y</IsAFile>
+ <valueField>filename</valueField>
+</transform>
