This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 98f9c7f8b1 Four transforms repeat per-row work that only needs doing
once, fixes #9455 (#9456)
98f9c7f8b1 is described below
commit 98f9c7f8b1f6abb59b7b5ece117e3f1effbb3a10
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 12 20:49:11 2026 +0100
Four transforms repeat per-row work that only needs doing once, fixes #9455
(#9456)
---
.../hop/avro/transforms/avrooutput/AvroOutput.java | 98 +++++++++--
.../avrooutput/AvroOutputRecordLayoutTest.java | 194 +++++++++++++++++++++
.../transforms/loadfileinput/LoadFileInput.java | 13 +-
.../pipeline/transforms/metainject/MetaInject.java | 14 +-
.../StandardizePhoneNumber.java | 175 +++++++++++--------
.../StandardizePhoneNumberData.java | 29 +++
6 files changed, 431 insertions(+), 92 deletions(-)
diff --git
a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
index 2420d6ff6c..b3fd678010 100644
---
a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
+++
b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avrooutput/AvroOutput.java
@@ -57,6 +57,9 @@ public class AvroOutput extends BaseTransform<AvroOutputMeta,
AvroOutputData> {
private List<AvroOutputField> avroOutputFields;
private int outputFieldIndex;
+ /** Where the values of a row go in the Avro record; made on the first row.
*/
+ private RecordLayout recordLayout;
+
public AvroOutput(
TransformMeta transformMeta,
AvroOutputMeta meta,
@@ -67,8 +70,39 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
}
- private GenericRecord getRecord(Object[] r, String parentPath, Schema
inputSchema)
- throws HopException {
+ /**
+ * Where the values of a record go: worked out once from the Avro names of
the output fields and
+ * the schema, so that a row only has to put its values in.
+ */
+ private static final class RecordLayout {
+ private final Schema recordSchema;
+ private final List<Entry> entries = new ArrayList<>();
+
+ private RecordLayout(Schema recordSchema) {
+ this.recordSchema = recordSchema;
+ }
+
+ /** A value of the row, or a nested record. */
+ private static final class Entry {
+ /** The name of the field in the record: used when the schema does not
have the field. */
+ private String name;
+
+ /** The field in the schema, null when the schema does not have it. */
+ private Field avroField;
+
+ /** For a value: the position of the output field it comes from. */
+ private int fieldIndex;
+
+ /** For a nested record: its layout. Null for a value. */
+ private RecordLayout record;
+ }
+ }
+
+ /**
+ * Works out the layout of a record and of the records nested in it. Nested
fields have a dot in
+ * their Avro name; the fields of one record follow each other in the list
of output fields.
+ */
+ private RecordLayout createLayout(String parentPath, Schema inputSchema) {
String parentName = "";
if (parentPath != null) {
parentName = parentPath;
@@ -89,7 +123,7 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
}
}
- GenericRecord result = new GenericData.Record(recordSchema);
+ RecordLayout layout = new RecordLayout(recordSchema);
while (outputFieldIndex < avroOutputFields.size()) {
AvroOutputField aof = avroOutputFields.get(outputFieldIndex);
@@ -102,33 +136,57 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
if (!parentName.isEmpty()) {
avroName = avroName.substring(parentName.length() + 1);
}
+ RecordLayout.Entry entry = new RecordLayout.Entry();
if (avroName.contains(".")) {
String currentAvroPath = avroName.substring(0,
avroName.indexOf("."));
- Schema childSchema = recordSchema.getField(currentAvroPath).schema();
+ Field childField = recordSchema.getField(currentAvroPath);
+ Schema childSchema = childField.schema();
String childPath = parentName + "." + currentAvroPath;
if (parentName.isEmpty()) {
childPath = currentAvroPath;
}
- GenericRecord fieldRecord = getRecord(r, childPath, childSchema);
- result.put(currentAvroPath, fieldRecord);
+ entry.name = currentAvroPath;
+ entry.avroField = childField;
+ entry.record = createLayout(childPath, childSchema);
} else {
- Field avroField = recordSchema.getField(avroName);
- Object value =
- getValue(
- r,
- meta.getOutputFields().get(outputFieldIndex),
- data.fieldNrs[outputFieldIndex],
- avroField);
- if (value != null) {
- result.put(avroName, value);
- }
+ entry.name = avroName;
+ entry.avroField = recordSchema.getField(avroName);
+ entry.fieldIndex = outputFieldIndex;
outputFieldIndex++;
}
+ layout.entries.add(entry);
} else {
break;
}
}
+ return layout;
+ }
+
+ private GenericRecord getRecord(Object[] r, RecordLayout layout) throws
HopException {
+ GenericRecord result = new GenericData.Record(layout.recordSchema);
+ for (RecordLayout.Entry entry : layout.entries) {
+ Object value;
+ if (entry.record != null) {
+ value = getRecord(r, entry.record);
+ } else {
+ value =
+ getValue(
+ r,
+ meta.getOutputFields().get(entry.fieldIndex),
+ data.fieldNrs[entry.fieldIndex],
+ entry.avroField);
+ if (value == null) {
+ continue;
+ }
+ }
+ if (entry.avroField != null) {
+ result.put(entry.avroField.pos(), value);
+ } else {
+ // not in the schema: Avro reports it by name, as it always did
+ result.put(entry.name, value);
+ }
+ }
return result;
}
@@ -350,8 +408,12 @@ public class AvroOutput extends
BaseTransform<AvroOutputMeta, AvroOutputData> {
return false;
}
- outputFieldIndex = 0;
- GenericRecord row = getRecord(r, null, data.avroSchema);
+ if (recordLayout == null) {
+ // The Avro names and the schema do not change from row to row
+ outputFieldIndex = 0;
+ recordLayout = createLayout(null, data.avroSchema);
+ }
+ GenericRecord row = getRecord(r, recordLayout);
try {
if
(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_BINARY_FILE].equals(
diff --git
a/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputRecordLayoutTest.java
b/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputRecordLayoutTest.java
new file mode 100644
index 0000000000..9bf988c9c3
--- /dev/null
+++
b/plugins/tech/avro/src/test/java/org/apache/hop/avro/transforms/avrooutput/AvroOutputRecordLayoutTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.avro.transforms.avrooutput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.logging.ILoggingObject;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Where each value goes in the Avro record is worked out once, on the first
row. Every row after
+ * that has to end up with its own values in the right fields, nested records
included.
+ */
+class AvroOutputRecordLayoutTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @TempDir Path tempDir;
+
+ private TransformMockHelper<AvroOutputMeta, AvroOutputData> mockHelper;
+
+ @BeforeAll
+ static void initEnv() throws Exception {
+ HopEnvironment.init();
+ }
+
+ @BeforeEach
+ void setUp() {
+ mockHelper =
+ new TransformMockHelper<>("AvroOutput", AvroOutputMeta.class,
AvroOutputData.class);
+ when(mockHelper.logChannelFactory.create(any(), any(ILoggingObject.class)))
+ .thenReturn(mockHelper.iLogChannel);
+ }
+
+ @AfterEach
+ void tearDown() {
+ mockHelper.cleanUp();
+ }
+
+ private static IRowMeta rowMeta() {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaInteger("id"));
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaString("city"));
+ rowMeta.addValueMeta(new ValueMetaString("zip"));
+ rowMeta.addValueMeta(new ValueMetaNumber("lat"));
+ rowMeta.addValueMeta(new ValueMetaNumber("amount"));
+ return rowMeta;
+ }
+
+ private List<GenericRecord> write(List<AvroOutputField> fields, Object[]...
rows)
+ throws Exception {
+ Path file = tempDir.resolve("out.avro");
+ AvroOutputMeta meta = new AvroOutputMeta();
+ meta.setDefault();
+
meta.setOutputType(AvroOutputMeta.OUTPUT_TYPES[AvroOutputMeta.OUTPUT_TYPE_BINARY_FILE]);
+ meta.setCreateSchemaFile(true);
+ meta.setWriteSchemaFile(false);
+ meta.setRecordName("customer");
+ meta.setNamespace("org.apache.hop.test");
+ meta.setFileName(file.toString());
+ meta.setOutputFields(fields);
+
+ AvroOutput transform =
+ spy(
+ new AvroOutput(
+ mockHelper.transformMeta,
+ meta,
+ new AvroOutputData(),
+ 0,
+ mockHelper.pipelineMeta,
+ mockHelper.pipeline));
+ doReturn(rowMeta()).when(transform).getInputRowMeta();
+ Object[][] rest = new Object[rows.length][];
+ System.arraycopy(rows, 1, rest, 0, rows.length - 1);
+ doReturn(rows[0], (Object[]) rest).when(transform).getRow();
+ doAnswer(inv -> null).when(transform).putRow(any(), any());
+
+ assertTrue(transform.init());
+ for (int i = 0; i < rows.length; i++) {
+ assertTrue(transform.processRow());
+ }
+ assertFalse(transform.processRow());
+ transform.dispose();
+
+ List<GenericRecord> records = new ArrayList<>();
+ try (InputStream inputStream = Files.newInputStream(file);
+ DataFileStream<GenericRecord> reader =
+ new DataFileStream<>(inputStream, new GenericDatumReader<>())) {
+ reader.forEach(records::add);
+ }
+ return records;
+ }
+
+ @Test
+ void flatFieldsKeepTheirValuesRowAfterRow() throws Exception {
+ List<AvroOutputField> fields = new ArrayList<>();
+ fields.add(new AvroOutputField("id", "id", AvroOutputField.AVRO_TYPE_LONG,
true));
+ fields.add(new AvroOutputField("name", "full_name",
AvroOutputField.AVRO_TYPE_STRING, true));
+ fields.add(new AvroOutputField("amount", "amount",
AvroOutputField.AVRO_TYPE_DOUBLE, true));
+
+ List<GenericRecord> records =
+ write(
+ fields,
+ new Object[] {1L, "Alice", "Ghent", "9000", 51.05, 10.5},
+ new Object[] {2L, null, "Lier", "2500", 51.13, null},
+ new Object[] {3L, "Carol", "Mol", "2400", 51.19, 30.25});
+
+ assertEquals(3, records.size());
+ assertEquals(1L, records.get(0).get("id"));
+ assertEquals("Alice", records.get(0).get("full_name").toString());
+ assertEquals(10.5, records.get(0).get("amount"));
+ // a null value leaves the field empty
+ assertEquals(2L, records.get(1).get("id"));
+ assertNull(records.get(1).get("full_name"));
+ assertNull(records.get(1).get("amount"));
+ assertEquals("Carol", records.get(2).get("full_name").toString());
+ assertEquals(30.25, records.get(2).get("amount"));
+ }
+
+ @Test
+ void nestedRecordsGetTheirOwnValues() throws Exception {
+ List<AvroOutputField> fields = new ArrayList<>();
+ fields.add(new AvroOutputField("id", "id", AvroOutputField.AVRO_TYPE_LONG,
true));
+ fields.add(new AvroOutputField("city", "address.city",
AvroOutputField.AVRO_TYPE_STRING, true));
+ fields.add(new AvroOutputField("zip", "address.zip",
AvroOutputField.AVRO_TYPE_STRING, true));
+ fields.add(
+ new AvroOutputField("lat", "address.geo.lat",
AvroOutputField.AVRO_TYPE_DOUBLE, true));
+ fields.add(new AvroOutputField("name", "$.name",
AvroOutputField.AVRO_TYPE_STRING, true));
+
+ List<GenericRecord> records =
+ write(
+ fields,
+ new Object[] {1L, "Alice", "Ghent", "9000", 51.05, 10.5},
+ new Object[] {2L, "Bob", "Lier", null, 51.13, 20.0});
+
+ assertEquals(2, records.size());
+ for (int i = 0; i < 2; i++) {
+ GenericRecord record = records.get(i);
+ GenericRecord address = (GenericRecord) record.get("address");
+ GenericRecord geo = (GenericRecord) address.get("geo");
+ assertEquals((long) i + 1, record.get("id"));
+ assertEquals(i == 0 ? "Alice" : "Bob", record.get("name").toString());
+ assertEquals(i == 0 ? "Ghent" : "Lier", address.get("city").toString());
+ assertEquals(i == 0 ? 51.05 : 51.13, geo.get("lat"));
+ }
+ assertEquals("9000", ((GenericRecord)
records.get(0).get("address")).get("zip").toString());
+ assertNull(((GenericRecord) records.get(1).get("address")).get("zip"));
+ }
+}
diff --git
a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInput.java
b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInput.java
index 8be427bd14..a652506df4 100644
---
a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInput.java
+++
b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInput.java
@@ -279,7 +279,8 @@ public class LoadFileInput extends
BaseTransform<LoadFileInputMeta, LoadFileInpu
void getFileContent() throws HopException {
try {
- data.filecontent = getFileBinaryContent(data.file.toString(), variables);
+ // The file is resolved already: read it, do not look it up again by its
name
+ data.filecontent = readContent(data.file);
if (data.filecontent != null) {
dataVolumeIn = (dataVolumeIn != null ? dataVolumeIn : 0L) +
data.filecontent.length;
}
@@ -294,6 +295,16 @@ public class LoadFileInput extends
BaseTransform<LoadFileInputMeta, LoadFileInpu
}
}
+ private static byte[] readContent(FileObject file) throws HopException {
+ try (InputStream inputStream = HopVfs.getInputStream(file)) {
+ return inputStream.readAllBytes();
+ } catch (Exception e) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "LoadFileInput.Error.GettingFileContent", file.toString(),
e.toString()));
+ }
+ }
+
/**
* Read a file.
*
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
index 7833a0f909..a63267fdef 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
@@ -28,8 +28,10 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
@@ -221,6 +223,12 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
handleReadingFromSourceTransform(injectPipeline);
}
+ // Told when the template is done, so that the wait below ends at that
moment and not up to
+ // 50 ms later.
+ CountDownLatch templateFinished = new CountDownLatch(1);
+ injectPipeline.addExecutionFinishedListener(e ->
templateFinished.countDown());
+ injectPipeline.addExecutionStoppedListener(e ->
templateFinished.countDown());
+
injectPipeline.startThreads();
Future<HopTransformException> steamingFuture =
handleStreamingToTargetTransform(rowProducer);
@@ -230,9 +238,11 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData> {
while (!injectPipeline.isFinished() && !injectPipeline.isStopped() &&
!isStopped()) {
copyResult(injectPipeline);
- // Wait a little bit.
+ // Wait a little bit: until the template is done, or to bring the line
counters up to date.
try {
- Thread.sleep(50);
+ if (templateFinished.await(50, TimeUnit.MILLISECONDS)) {
+ break;
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
diff --git
a/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumber.java
b/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumber.java
index 3c9f313539..4a66f9f004 100644
---
a/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumber.java
+++
b/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumber.java
@@ -21,6 +21,7 @@ import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.PhoneNumberUtil.PhoneNumberFormat;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
import java.util.Set;
@@ -81,6 +82,11 @@ public class StandardizePhoneNumber
// use meta.getFields() to change it, so it reflects the output row
// structure
meta.getFields(data.outputRowMeta, this.getTransformName(), null, null,
this, null);
+
+ // The settings of every field do not change from row to row: work them
out once
+ if (!prepareFields()) {
+ return false;
+ }
}
IRowMeta inputRowMeta = getInputRowMeta();
@@ -89,62 +95,35 @@ public class StandardizePhoneNumber
// the output values
Object[] outputRow = Arrays.copyOf(row, data.outputRowMeta.size());
- for (StandardizePhoneField standardize : meta.getFields()) {
- String inputField = resolve(standardize.getInputField());
- String outputField = resolve(standardize.getOutputField());
- String countryField = resolve(standardize.getCountryField());
- String numberTypeField = resolve(standardize.getNumberTypeField());
- String isValidNumberField = resolve(standardize.getIsValidNumberField());
- String numberFormat = resolve(standardize.getNumberFormat());
-
- String region = resolveRegion(standardize, countryField, inputRowMeta,
row);
-
- int inputIndex = inputRowMeta.indexOfValue(inputField);
-
- // if input field not found
- if (inputIndex < 0) {
- this.logError(
- BaseMessages.getString(
- PKG, "StandardizePhoneNumber.Log.InputFieldNotFound",
inputField));
- this.setErrors(1);
- return false;
- }
+ for (StandardizePhoneNumberData.Field field : data.fields) {
+ String region = region(field, inputRowMeta, row);
- int outputIndex = inputIndex;
- if (!Utils.isEmpty(outputField)) {
- int resolvedOutputIndex = data.outputRowMeta.indexOfValue(outputField);
- if (resolvedOutputIndex >= 0) {
- outputIndex = resolvedOutputIndex;
- }
- }
-
- String originalValue = inputRowMeta.getString(row, inputIndex);
+ String originalValue = inputRowMeta.getString(row, field.inputIndex);
if (!Utils.isEmpty(originalValue)) {
PhoneNumber phoneNumber = null;
try {
// Replace unsupported character with blank
String value = originalValue.replace(',', ' ');
- PhoneNumberFormat format = getPhoneNumberFormat(numberFormat);
phoneNumber = phoneNumberService.parse(value, region);
- outputRow[outputIndex] = phoneNumberService.format(phoneNumber,
format);
+ outputRow[field.outputIndex] =
phoneNumberService.format(phoneNumber, format(field));
} catch (NumberParseException e) {
- outputRow[outputIndex] = originalValue;
+ outputRow[field.outputIndex] = originalValue;
if (isRowLevel()) {
logRowlevel(
BaseMessages.getString(
PKG,
"StandardizePhoneNumber.Log.ProcessPhoneNumberError",
- inputField,
+ field.inputField,
originalValue));
}
}
- setNumberType(outputRow, numberTypeField, phoneNumber);
- setIsValid(outputRow, isValidNumberField, phoneNumber);
+ setNumberType(outputRow, field.numberTypeIndex, phoneNumber);
+ setIsValid(outputRow, field.isValidIndex, phoneNumber);
} else {
- setNumberType(outputRow, numberTypeField, null);
- setIsValid(outputRow, isValidNumberField, null);
+ setNumberType(outputRow, field.numberTypeIndex, null);
+ setIsValid(outputRow, field.isValidIndex, null);
}
}
@@ -180,65 +159,119 @@ public class StandardizePhoneNumber
return normalized.isEmpty() ? null : normalized;
}
- private String resolveRegion(
- StandardizePhoneField standardize, String countryField, IRowMeta
inputRowMeta, Object[] row)
- throws HopException {
- String region = normalizeRegion(resolve(standardize.getDefaultCountry()));
- if (Utils.isEmpty(countryField)) {
- return region;
- }
+ /**
+ * Resolves the settings of every field: the names, where the fields are in
the row, the default
+ * region and the number format. Called on the first row.
+ *
+ * @return false when an input field is not in the row: the error is logged
+ */
+ private boolean prepareFields() throws HopException {
+ IRowMeta inputRowMeta = getInputRowMeta();
+ data.fields = new ArrayList<>();
+ for (StandardizePhoneField standardize : meta.getFields()) {
+ StandardizePhoneNumberData.Field field = new
StandardizePhoneNumberData.Field();
+ field.inputField = resolve(standardize.getInputField());
+ String outputField = resolve(standardize.getOutputField());
+ String countryField = resolve(standardize.getCountryField());
+ String numberTypeField = resolve(standardize.getNumberTypeField());
+ String isValidNumberField = resolve(standardize.getIsValidNumberField());
+ field.numberFormat = resolve(standardize.getNumberFormat());
+ field.defaultRegion =
normalizeRegion(resolve(standardize.getDefaultCountry()));
+
+ field.countryIndex = -1;
+ if (!Utils.isEmpty(countryField)) {
+ field.countryIndex = inputRowMeta.indexOfValue(countryField);
+
+ // if country field not found
+ if (field.countryIndex < 0) {
+ String message =
+ BaseMessages.getString(
+ PKG, "StandardizePhoneNumber.Log.CountryFieldNotFound",
countryField);
+ logError(message);
+ this.setErrors(1);
+ throw new HopException(message);
+ }
+ }
- int index = inputRowMeta.indexOfValue(countryField);
+ field.inputIndex = inputRowMeta.indexOfValue(field.inputField);
- // if country field not found
- if (index < 0) {
- String message =
- BaseMessages.getString(
- PKG, "StandardizePhoneNumber.Log.CountryFieldNotFound",
countryField);
- logError(message);
- this.setErrors(1);
- throw new HopException(message);
+ // if input field not found
+ if (field.inputIndex < 0) {
+ this.logError(
+ BaseMessages.getString(
+ PKG, "StandardizePhoneNumber.Log.InputFieldNotFound",
field.inputField));
+ this.setErrors(1);
+ return false;
+ }
+
+ field.outputIndex = field.inputIndex;
+ if (!Utils.isEmpty(outputField)) {
+ int resolvedOutputIndex = data.outputRowMeta.indexOfValue(outputField);
+ if (resolvedOutputIndex >= 0) {
+ field.outputIndex = resolvedOutputIndex;
+ }
+ }
+
+ field.numberTypeIndex =
+ Utils.isEmpty(numberTypeField) ? -1 :
data.outputRowMeta.indexOfValue(numberTypeField);
+ field.isValidIndex =
+ Utils.isEmpty(isValidNumberField)
+ ? -1
+ : data.outputRowMeta.indexOfValue(isValidNumberField);
+
+ data.fields.add(field);
+ }
+ return true;
+ }
+
+ /**
+ * The region to parse a number with: the country of the row when it is
known, else the default.
+ */
+ private String region(StandardizePhoneNumberData.Field field, IRowMeta
inputRowMeta, Object[] row)
+ throws HopException {
+ if (field.countryIndex < 0) {
+ return field.defaultRegion;
}
- String country = inputRowMeta.getString(row, index);
+ String country = inputRowMeta.getString(row, field.countryIndex);
String normalized = normalizeRegion(country);
if (normalized == null) {
- return region;
+ return field.defaultRegion;
}
if (supportedRegions.contains(normalized)) {
return normalized;
}
logError(BaseMessages.getString(PKG,
"StandardizePhoneNumber.Log.RegionNotSupported", country));
- return region;
+ return field.defaultRegion;
}
- private void setNumberType(Object[] outputRow, String numberTypeField,
PhoneNumber phoneNumber) {
- if (Utils.isEmpty(numberTypeField)) {
- return;
+ /** The number format of a field, looked up the first time a number has to
be formatted. */
+ private PhoneNumberFormat format(StandardizePhoneNumberData.Field field) {
+ if (field.format == null) {
+ field.format = getPhoneNumberFormat(field.numberFormat);
}
- int i = data.outputRowMeta.indexOfValue(numberTypeField);
- if (i < 0) {
+ return field.format;
+ }
+
+ private void setNumberType(Object[] outputRow, int index, PhoneNumber
phoneNumber) {
+ if (index < 0) {
return;
}
if (phoneNumber != null) {
- outputRow[i] = phoneNumberService.getNumberType(phoneNumber).toString();
+ outputRow[index] =
phoneNumberService.getNumberType(phoneNumber).toString();
} else {
- outputRow[i] = NUMBER_TYPE_ERROR;
+ outputRow[index] = NUMBER_TYPE_ERROR;
}
}
- private void setIsValid(Object[] outputRow, String isValidNumberField,
PhoneNumber phoneNumber) {
- if (Utils.isEmpty(isValidNumberField)) {
- return;
- }
- int i = data.outputRowMeta.indexOfValue(isValidNumberField);
- if (i < 0) {
+ private void setIsValid(Object[] outputRow, int index, PhoneNumber
phoneNumber) {
+ if (index < 0) {
return;
}
if (phoneNumber != null) {
- outputRow[i] = phoneNumberService.isValidNumber(phoneNumber);
+ outputRow[index] = phoneNumberService.isValidNumber(phoneNumber);
} else {
- outputRow[i] = false;
+ outputRow[index] = false;
}
}
diff --git
a/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumberData.java
b/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumberData.java
index c3253a14de..49b3047e72 100644
---
a/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumberData.java
+++
b/plugins/transforms/standardizephonenumber/src/main/java/org/apache/hop/pipeline/transforms/standardizephonenumber/StandardizePhoneNumberData.java
@@ -17,6 +17,8 @@
package org.apache.hop.pipeline.transforms.standardizephonenumber;
+import com.google.i18n.phonenumbers.PhoneNumberUtil.PhoneNumberFormat;
+import java.util.List;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.pipeline.transform.BaseTransformData;
import org.apache.hop.pipeline.transform.ITransformData;
@@ -25,6 +27,33 @@ public class StandardizePhoneNumberData extends
BaseTransformData implements ITr
protected IRowMeta outputRowMeta;
+ /** The settings of each configured field, resolved on the first row. */
+ protected List<Field> fields;
+
+ /** One configured field: names resolved, positions in the row looked up. */
+ static final class Field {
+ String inputField;
+ int inputIndex;
+ int outputIndex;
+
+ /** Position of the country field in the input row, -1 when there is none.
*/
+ int countryIndex;
+
+ /** The region to use when the row has no country, or an unknown one. */
+ String defaultRegion;
+
+ String numberFormat;
+
+ /** The format for {@link #numberFormat}, set when the first number is
formatted. */
+ PhoneNumberFormat format;
+
+ /** Position of the number type field in the output row, -1 when there is
none. */
+ int numberTypeIndex;
+
+ /** Position of the validity field in the output row, -1 when there is
none. */
+ int isValidIndex;
+ }
+
public StandardizePhoneNumberData() {
super();
}