mattcasters commented on code in PR #8783:
URL: https://github.com/apache/hop/pull/8783#discussion_r4210061863
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMeta.java:
##########
@@ -104,6 +138,14 @@ public void getFields(
v.setOrigin(name);
inputRowMeta.addValueMeta(v);
}
+ for (BinOutputField outputField : outputFields) {
+ String realName = variables.resolve(outputField.getName());
+ if (!Utils.isEmpty(realName)) {
+ IValueMeta v = new ValueMetaString(realName);
Review Comment:
**[bug]** Extra BIN output fields are always added as `ValueMetaString`. The
dialog stores type, length, precision, and format on `BinOutputField`, and
`convertValue` can return a `Long`, `Double`, or `Date`, but none of those
properties are copied onto the row meta. Downstream steps then cast that object
to `String` (`ValueMetaString.getBoolean` / `getInteger`) or create the wrong
column type.
**Suggestion:** Create the meta with
`ValueMetaFactory.createValueMeta(realName, outputField.getType(),
outputField.getLength(), outputField.getPrecision())` and copy the conversion
mask, decimal/group/currency symbols, trim type, and origin. Cover it with a
`getFields` test that an Integer mapping is not a String.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java:
##########
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.creditcardvalidator;
+
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvValidationException;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.i18n.BaseMessages;
+
+/** In-memory lookup table for Bank Identification Numbers (BIN) loaded from
an external CSV. */
+public class BinDatabase {
+
+ private static final Class<?> PKG = CreditCardValidatorMeta.class;
+
+ private static final int MIN_BIN_LENGTH = 1;
+ private static final int MAX_BIN_LENGTH = 8;
+ private static final int EXPECTED_RECORDS = 1_000_000;
+
+ private final Long2ObjectOpenHashMap<BinRecord> records;
+ private final Map<String, String> dedup = new HashMap<>();
+
+ /** Bitmask of the BIN lengths present in the dataset (bit {@code 1 <<
length}). */
+ private int binLengths;
+
+ public BinDatabase() {
+ this.records = new Long2ObjectOpenHashMap<>(EXPECTED_RECORDS);
+ }
+
+ /** Load a user-supplied CSV file with an explicit column mapping and read
configuration. */
+ public void load(
+ IVariables variables,
+ String vfsFilename,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ String encoding,
+ boolean headerPresent)
+ throws HopException {
+ Charset charset;
+ try {
+ charset = Charset.forName(encoding == null ? "UTF-8" : encoding);
+ } catch (Exception e) {
+ charset = StandardCharsets.UTF_8;
+ }
+ try (InputStream in = HopVfs.getInputStream(vfsFilename, variables)) {
+ load(in, binCsvColumn, outputFields, delimiter, enclosure, charset,
headerPresent);
+ } catch (IOException e) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseRead"), e);
+ }
+ }
+
+ private void load(
+ InputStream in,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ Charset charset,
+ boolean headerPresent)
+ throws HopException {
+ try (CSVReader reader =
+ new CSVReaderBuilder(new InputStreamReader(in, charset))
+ .withCSVParser(parser(delimiter, enclosure))
+ .build()) {
+ int binIdx;
+ int cardTypeIdx;
+ int issuerIdx;
+ int countryIdx;
+ Map<String, Integer> extraIdx = new HashMap<>();
+
+ if (headerPresent) {
+ String[] header = reader.readNext();
+ if (header == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseEmpty"));
+ }
+ binIdx = resolveColumn(header, binCsvColumn, 0, "bin");
+ if (binIdx < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "CreditCardValidator.Error.BinColumnNotFound",
binCsvColumn));
+ }
+ cardTypeIdx = resolveColumn(header, null, 1, "cardtype", "card_type",
"card type");
+ issuerIdx = resolveColumn(header, null, 2, "issuer", "issuers");
+ countryIdx = resolveColumn(header, null, 3, "country");
+ for (BinOutputField outputField : outputFields) {
+ if (Utils.isEmpty(outputField.getColumn())) {
+ continue;
+ }
+ for (int i = 0; i < header.length; i++) {
+ if (outputField.getColumn().equals(header[i])) {
+ extraIdx.put(outputField.getName(), i);
+ break;
+ }
+ }
+ }
+ } else {
Review Comment:
**[bug]** With "Header row present" unchecked, `extraIdx` stays empty, so
every extra output field is null for the whole run. The dialog still accepts
CSV-column mappings, and `readHeader` always treats the first line as a header,
ignoring that checkbox. Mappings created from Get columns therefore never load.
**Suggestion:** When there is no header, map extra fields by column index,
or fail `init` with a clear message if a mapping needs a header name. Make Get
columns honor the header checkbox.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java:
##########
@@ -167,13 +181,57 @@ public boolean processRow() throws HopException {
return true;
}
+ private Object convertValue(BinOutputField field, String value) {
+ if (value == null) {
+ return null;
+ }
+ try {
+ IValueMeta valueMeta =
+ ValueMetaFactory.createValueMeta(
+ field.getName(), field.getType(), field.getLength(),
field.getPrecision());
+ valueMeta.setConversionMask(field.getFormat());
+ valueMeta.setDecimalSymbol(field.getDecimal());
+ valueMeta.setGroupingSymbol(field.getGroup());
+ valueMeta.setCurrencySymbol(field.getCurrency());
+ valueMeta.setTrimType(field.getTrimType());
+ return valueMeta.convertDataFromString(
+ value, new ValueMetaString(field.getName()), null, null,
ValueMetaString.TRIM_TYPE_NONE);
Review Comment:
**[bug]** The Trim column never affects the value. `setTrimType` is set on
the target meta, but `convertDataFromString` trims with its last argument,
which is hard-coded to `TRIM_TYPE_NONE`. The `catch` then returns the original
`String`, so one row can hold a `Long` and the next row a `String` in the same
field.
**Suggestion:** Pass `field.getTrimType()` into `convertDataFromString`. On
a conversion failure, return null or route the row to error handling instead of
a value of the wrong runtime type. Build the `IValueMeta` once per field in
`init`, not on every row.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java:
##########
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.creditcardvalidator;
+
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvValidationException;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.i18n.BaseMessages;
+
+/** In-memory lookup table for Bank Identification Numbers (BIN) loaded from
an external CSV. */
+public class BinDatabase {
+
+ private static final Class<?> PKG = CreditCardValidatorMeta.class;
+
+ private static final int MIN_BIN_LENGTH = 1;
+ private static final int MAX_BIN_LENGTH = 8;
+ private static final int EXPECTED_RECORDS = 1_000_000;
+
+ private final Long2ObjectOpenHashMap<BinRecord> records;
+ private final Map<String, String> dedup = new HashMap<>();
+
+ /** Bitmask of the BIN lengths present in the dataset (bit {@code 1 <<
length}). */
+ private int binLengths;
+
+ public BinDatabase() {
+ this.records = new Long2ObjectOpenHashMap<>(EXPECTED_RECORDS);
+ }
+
+ /** Load a user-supplied CSV file with an explicit column mapping and read
configuration. */
+ public void load(
+ IVariables variables,
+ String vfsFilename,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ String encoding,
+ boolean headerPresent)
+ throws HopException {
+ Charset charset;
+ try {
+ charset = Charset.forName(encoding == null ? "UTF-8" : encoding);
+ } catch (Exception e) {
+ charset = StandardCharsets.UTF_8;
+ }
+ try (InputStream in = HopVfs.getInputStream(vfsFilename, variables)) {
+ load(in, binCsvColumn, outputFields, delimiter, enclosure, charset,
headerPresent);
+ } catch (IOException e) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseRead"), e);
+ }
+ }
+
+ private void load(
+ InputStream in,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ Charset charset,
+ boolean headerPresent)
+ throws HopException {
+ try (CSVReader reader =
+ new CSVReaderBuilder(new InputStreamReader(in, charset))
+ .withCSVParser(parser(delimiter, enclosure))
+ .build()) {
+ int binIdx;
+ int cardTypeIdx;
+ int issuerIdx;
+ int countryIdx;
+ Map<String, Integer> extraIdx = new HashMap<>();
+
+ if (headerPresent) {
+ String[] header = reader.readNext();
+ if (header == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseEmpty"));
+ }
+ binIdx = resolveColumn(header, binCsvColumn, 0, "bin");
+ if (binIdx < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "CreditCardValidator.Error.BinColumnNotFound",
binCsvColumn));
+ }
+ cardTypeIdx = resolveColumn(header, null, 1, "cardtype", "card_type",
"card type");
+ issuerIdx = resolveColumn(header, null, 2, "issuer", "issuers");
+ countryIdx = resolveColumn(header, null, 3, "country");
+ for (BinOutputField outputField : outputFields) {
+ if (Utils.isEmpty(outputField.getColumn())) {
+ continue;
+ }
+ for (int i = 0; i < header.length; i++) {
+ if (outputField.getColumn().equals(header[i])) {
+ extraIdx.put(outputField.getName(), i);
+ break;
+ }
+ }
+ }
+ } else {
+ binIdx = 0;
+ cardTypeIdx = 1;
+ issuerIdx = 2;
+ countryIdx = 3;
+ }
+
+ String[] fields;
+ while ((fields = reader.readNext()) != null) {
+ String rawBin = field(fields, binIdx).trim();
+ if (rawBin.isEmpty() || rawBin.length() > MAX_BIN_LENGTH) {
Review Comment:
**[suggestion]** A BIN longer than 8 characters, or one that is not all
digits, is skipped with no log. Those cards later fail lookup as "not valid"
even though the row was in the file. Eight digits matches the current IIN
length, but several BIN extracts use a longer prefix.
**Suggestion:** Count skipped rows and log that count from `init`. Either
keep the prefix as a digit string so it can be longer than 8, or state the
8-digit limit on the BIN file control.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java:
##########
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.creditcardvalidator;
+
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvValidationException;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.i18n.BaseMessages;
+
+/** In-memory lookup table for Bank Identification Numbers (BIN) loaded from
an external CSV. */
+public class BinDatabase {
+
+ private static final Class<?> PKG = CreditCardValidatorMeta.class;
+
+ private static final int MIN_BIN_LENGTH = 1;
+ private static final int MAX_BIN_LENGTH = 8;
+ private static final int EXPECTED_RECORDS = 1_000_000;
+
+ private final Long2ObjectOpenHashMap<BinRecord> records;
+ private final Map<String, String> dedup = new HashMap<>();
+
+ /** Bitmask of the BIN lengths present in the dataset (bit {@code 1 <<
length}). */
+ private int binLengths;
+
+ public BinDatabase() {
+ this.records = new Long2ObjectOpenHashMap<>(EXPECTED_RECORDS);
+ }
+
+ /** Load a user-supplied CSV file with an explicit column mapping and read
configuration. */
+ public void load(
+ IVariables variables,
+ String vfsFilename,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ String encoding,
+ boolean headerPresent)
+ throws HopException {
+ Charset charset;
+ try {
+ charset = Charset.forName(encoding == null ? "UTF-8" : encoding);
+ } catch (Exception e) {
+ charset = StandardCharsets.UTF_8;
+ }
+ try (InputStream in = HopVfs.getInputStream(vfsFilename, variables)) {
+ load(in, binCsvColumn, outputFields, delimiter, enclosure, charset,
headerPresent);
+ } catch (IOException e) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseRead"), e);
+ }
+ }
+
+ private void load(
+ InputStream in,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ Charset charset,
+ boolean headerPresent)
+ throws HopException {
+ try (CSVReader reader =
+ new CSVReaderBuilder(new InputStreamReader(in, charset))
+ .withCSVParser(parser(delimiter, enclosure))
+ .build()) {
+ int binIdx;
+ int cardTypeIdx;
+ int issuerIdx;
+ int countryIdx;
+ Map<String, Integer> extraIdx = new HashMap<>();
+
+ if (headerPresent) {
+ String[] header = reader.readNext();
+ if (header == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseEmpty"));
+ }
+ binIdx = resolveColumn(header, binCsvColumn, 0, "bin");
+ if (binIdx < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "CreditCardValidator.Error.BinColumnNotFound",
binCsvColumn));
+ }
+ cardTypeIdx = resolveColumn(header, null, 1, "cardtype", "card_type",
"card type");
Review Comment:
**[suggestion]** If the header has no `cardtype`, `card_type`, or `card
type` column, `resolveColumn` returns index 1. The sample in the issue uses
`brand`, which is not one of those names, so the card type becomes whatever is
in column 2. The same positional fallback is applied to issuer (index 2) and
country (index 3), so a different column order silently mis-labels the card.
**Suggestion:** Also accept `brand`, `scheme`, and `type`. When nothing
matches, return -1 and leave the value empty instead of reading another column.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java:
##########
@@ -127,10 +135,16 @@ public boolean processRow() throws HopException {
if (!Utils.isEmpty(data.realNotValidMsgFieldname)) {
outputRow[rowIndex++] = unValid;
}
+ // add extra output fields?
+ for (BinOutputField outputField : meta.getOutputFields()) {
+ if (!Utils.isEmpty(outputField.getName())) {
Review Comment:
**[bug]** `getFields` adds a column only when the resolved field name is
non-empty, but this loop writes a column whenever the raw name is non-empty. A
name such as `${BIN_FIELD}` that resolves to empty still increments `rowIndex`
and writes past the array allocated from `data.outputRowMeta.size()`. The same
loop also runs when "Use BIN database" is off, so the extra columns stay in the
stream as nulls.
**Suggestion:** Resolve the name once and use that both here and in
`getFields`. Skip the loop unless `meta.isUseBinDatabase()` is true, in both
places, so the schema and the row stay aligned.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java:
##########
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.creditcardvalidator;
+
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvValidationException;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.i18n.BaseMessages;
+
+/** In-memory lookup table for Bank Identification Numbers (BIN) loaded from
an external CSV. */
+public class BinDatabase {
+
+ private static final Class<?> PKG = CreditCardValidatorMeta.class;
+
+ private static final int MIN_BIN_LENGTH = 1;
+ private static final int MAX_BIN_LENGTH = 8;
+ private static final int EXPECTED_RECORDS = 1_000_000;
+
+ private final Long2ObjectOpenHashMap<BinRecord> records;
+ private final Map<String, String> dedup = new HashMap<>();
+
+ /** Bitmask of the BIN lengths present in the dataset (bit {@code 1 <<
length}). */
+ private int binLengths;
+
+ public BinDatabase() {
+ this.records = new Long2ObjectOpenHashMap<>(EXPECTED_RECORDS);
+ }
+
+ /** Load a user-supplied CSV file with an explicit column mapping and read
configuration. */
+ public void load(
+ IVariables variables,
+ String vfsFilename,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ String encoding,
+ boolean headerPresent)
+ throws HopException {
+ Charset charset;
+ try {
+ charset = Charset.forName(encoding == null ? "UTF-8" : encoding);
+ } catch (Exception e) {
+ charset = StandardCharsets.UTF_8;
+ }
+ try (InputStream in = HopVfs.getInputStream(vfsFilename, variables)) {
+ load(in, binCsvColumn, outputFields, delimiter, enclosure, charset,
headerPresent);
+ } catch (IOException e) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseRead"), e);
+ }
+ }
+
+ private void load(
+ InputStream in,
+ String binCsvColumn,
+ List<BinOutputField> outputFields,
+ String delimiter,
+ String enclosure,
+ Charset charset,
+ boolean headerPresent)
+ throws HopException {
+ try (CSVReader reader =
+ new CSVReaderBuilder(new InputStreamReader(in, charset))
+ .withCSVParser(parser(delimiter, enclosure))
+ .build()) {
+ int binIdx;
+ int cardTypeIdx;
+ int issuerIdx;
+ int countryIdx;
+ Map<String, Integer> extraIdx = new HashMap<>();
+
+ if (headerPresent) {
+ String[] header = reader.readNext();
+ if (header == null) {
+ throw new HopException(
+ BaseMessages.getString(PKG,
"CreditCardValidator.Error.BinDatabaseEmpty"));
+ }
+ binIdx = resolveColumn(header, binCsvColumn, 0, "bin");
+ if (binIdx < 0) {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "CreditCardValidator.Error.BinColumnNotFound",
binCsvColumn));
+ }
+ cardTypeIdx = resolveColumn(header, null, 1, "cardtype", "card_type",
"card type");
+ issuerIdx = resolveColumn(header, null, 2, "issuer", "issuers");
+ countryIdx = resolveColumn(header, null, 3, "country");
+ for (BinOutputField outputField : outputFields) {
+ if (Utils.isEmpty(outputField.getColumn())) {
+ continue;
+ }
+ for (int i = 0; i < header.length; i++) {
+ if (outputField.getColumn().equals(header[i])) {
+ extraIdx.put(outputField.getName(), i);
+ break;
+ }
+ }
+ }
+ } else {
+ binIdx = 0;
+ cardTypeIdx = 1;
+ issuerIdx = 2;
+ countryIdx = 3;
+ }
+
+ String[] fields;
+ while ((fields = reader.readNext()) != null) {
+ String rawBin = field(fields, binIdx).trim();
+ if (rawBin.isEmpty() || rawBin.length() > MAX_BIN_LENGTH) {
+ continue;
+ }
+ int bin = parseDigits(rawBin);
+ if (bin < 0) {
+ continue;
+ }
+ int length = rawBin.length();
+ String cardType = dedup(field(fields, cardTypeIdx));
+ String issuer = field(fields, issuerIdx).trim();
+ String country = dedup(field(fields, countryIdx));
+ Map<String, String> extraValues = new HashMap<>();
Review Comment:
**[suggestion]** Every CSV row allocates its own `HashMap`, even when no
extra columns are mapped, and the lookup table is constructed for 1,000,000
entries before the file is read. Issuer and country are stored on every record
but are never written to the row unless the user maps those columns again as
extra fields. Each transform copy loads its own copy during `init`.
**Suggestion:** Share one empty map when `extraIdx` is empty, size the
lookup from the file instead of pre-allocating a million slots, and skip issuer
and country unless an output field needs them.
##########
plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorDialog.java:
##########
@@ -277,12 +671,158 @@ private void ok() {
input.setResultFieldName(wResult.getText());
input.setCardType(wFileType.getText());
input.setNotValidMessage(wNotValidMsg.getText());
+ input.setUseBinDatabase(wUseBinDatabase.getSelection());
+ input.setBinFileName(wBinFileName.getText());
+ input.setBinDelimiter(wBinDelimiter.getText());
+ input.setBinEnclosure(wBinEnclosure.getText());
+ input.setBinEncoding(wBinEncoding.getText());
+ input.setBinHeaderPresent(wBinHeaderPresent.getSelection());
+ input.setBinCsvColumn(wBinCsvColumn.getText());
+ readOutputFieldsTable();
// return value
transformName = wTransformName.getText();
dispose();
}
+ private void populateOutputFieldsTable() {
+ wOutputFields.clearAll();
+ for (BinOutputField outputField : input.getOutputFields()) {
+ wOutputFields.add(
+ Const.NVL(outputField.getName(), ""),
+ Const.NVL(outputField.getColumn(), ""),
+ ValueMetaFactory.getValueMetaName(outputField.getType()),
+ Const.NVL(outputField.getFormat(), ""),
+ Integer.toString(outputField.getLength()),
+ Integer.toString(outputField.getPrecision()),
+ Const.NVL(outputField.getCurrency(), ""),
+ Const.NVL(outputField.getDecimal(), ""),
+ Const.NVL(outputField.getGroup(), ""),
+ ValueMetaBase.getTrimTypeDesc(outputField.getTrimType()));
+ }
+ wOutputFields.removeEmptyRows();
+ wOutputFields.setRowNums();
+ wOutputFields.optWidth(true);
+ }
+
+ private void readOutputFieldsTable() {
+ List<BinOutputField> outputFields = new ArrayList<>();
+ int rows = wOutputFields.nrNonEmpty();
+ for (int i = 0; i < rows; i++) {
+ String[] row = wOutputFields.getItem(i);
Review Comment:
**[bug]** `nrNonEmpty()` skips blank rows, but the loop reads `getItem(i)`,
which is the raw table index. `TableView` only fills the non-empty index in
`nrNonEmpty()` for a following `getNonEmpty(i)` call. A blank row above a
filled one makes the last mapping fall outside `0..nrNonEmpty-1`, so OK drops
it.
**Suggestion:** Keep the `nrNonEmpty()` call, then read `TableItem item =
wOutputFields.getNonEmpty(i)` and take the cells with `item.getText(1)` through
`item.getText(10)`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
