This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 65aab02f75 Issue #6323 (SAS Input improvements) (#6359)
65aab02f75 is described below
commit 65aab02f75479cee35e190a23c42dc952c8aaa80
Author: Matt Casters <[email protected]>
AuthorDate: Tue Jan 13 19:17:21 2026 +0100
Issue #6323 (SAS Input improvements) (#6359)
---
.../ROOT/pages/pipeline/transforms/sasinput.adoc | 10 +-
.../hop/pipeline/transforms/sasinput/SasInput.java | 79 +++++--
.../pipeline/transforms/sasinput/SasInputData.java | 1 +
.../transforms/sasinput/SasInputDialog.java | 58 ++++-
.../transforms/sasinput/SasInputField.java | 237 +++------------------
.../pipeline/transforms/sasinput/SasInputMeta.java | 151 ++++++-------
.../sasinput/messages/messages_en_US.properties | 7 +-
.../transforms/sasinput/SasInputMetaTest.java | 112 ++++------
.../sasinput/src/test/resources/transform.xml | 60 ++++++
9 files changed, 325 insertions(+), 390 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sasinput.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sasinput.adoc
index b706b7fb4d..9caf597cd1 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sasinput.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/sasinput.adoc
@@ -34,9 +34,9 @@ The functionality is backed by the
https://github.com/epam/parso[Parso java libr
[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
!===
!Hop Engine! image:check_mark.svg[Supported, 24]
-!Spark! image:question_mark.svg[Maybe Supported, 24]
-!Flink! image:question_mark.svg[Maybe Supported, 24]
-!Dataflow! image:question_mark.svg[Maybe Supported, 24]
+!Spark! image:check_mark.svg[Supported, 24]
+!Flink! image:check_mark.svg[Supported, 24]
+!Dataflow! image:check_mark.svg[Supported, 24]
!===
|===
@@ -53,6 +53,10 @@ The functionality is backed by the
https://github.com/epam/parso[Parso java libr
|Select the input field that will contain the filename at runtime.
For example, you can use a "Get file names" transform to drive the content of
this field.
+|Limit
+|Stop reading after the specified number or rows. If nothing is specified or a
number smaller than or equal to zero, all rows in the SAS files will be read.
+This can be used to read a single row from a large SAS file when all you want
to do is add a xref:pipeline/transforms/metastructure.adoc[Metadata Structure
From Stream] transform to determine the file layout.
+
|The selected fields from the files
|If you use the "Get Fields" button you can populate this data grid.
Please note that even though the sas7bdat file format only contains certain
formats, that you can specify any desired data type and that Apache Hop will
convert for you.
diff --git
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java
index 8e012c62f8..9f32106b33 100644
---
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java
+++
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java
@@ -18,7 +18,6 @@
package org.apache.hop.pipeline.transforms.sasinput;
import com.epam.parso.Column;
-import com.epam.parso.ColumnFormat;
import com.epam.parso.SasFileProperties;
import com.epam.parso.impl.SasFileReaderImpl;
import java.io.InputStream;
@@ -35,6 +34,7 @@ import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaNumber;
import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.pipeline.Pipeline;
@@ -88,6 +88,8 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
//
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this,
metadataProvider);
+
+ data.limit = Const.toLong(resolve(meta.getLimit()), -1);
}
String rawFilename = getInputRowMeta().getString(fileRowData,
meta.getAcceptingField(), null);
@@ -116,37 +118,32 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
//
List<Column> columns = sasFileReader.getColumns();
- // Map this to the columns we want...
+ // Map this to the columns we want.
//
- List<Integer> indexes = new ArrayList<>();
- for (SasInputField field : meta.getOutputFields()) {
-
- int index = -1;
- for (int c = 0; c < columns.size(); c++) {
- if (columns.get(c).getName().equalsIgnoreCase(field.getName())) {
- index = c;
- break;
- }
- }
- if (index < 0) {
- throw new HopException(
- "Field '" + field.getName() + " could not be found in input file
'" + filename);
- }
- indexes.add(index);
- }
+ String metaFilename = resolve(meta.getMetadataFilename());
+ List<Integer> indexes = getColumnIndexes(columns, filename,
metaFilename);
// Now we have the indexes of the output fields to grab.
// Let's grab them...
//
Object[] sasRow;
while ((sasRow = sasFileReader.readNext()) != null) {
+ incrementLinesInput();
Object[] outputRow = RowDataUtil.createResizedCopy(fileRowData,
data.outputRowMeta.size());
- for (int i = 0; i < meta.getOutputFields().size(); i++) {
- SasInputField field = meta.getOutputFields().get(i);
+ for (int i = 0; i < indexes.size(); i++) {
int index = indexes.get(i);
Column column = columns.get(index);
- ColumnFormat columnFormat = column.getFormat();
+ SasInputField field;
+ if (StringUtil.isEmpty(metaFilename)) {
+ field = meta.getOutputFields().get(i);
+ } else {
+ field = new SasInputField();
+ field.setName(column.getName());
+ field.setLength(column.getFormat().getWidth());
+ field.setPrecision(column.getFormat().getPrecision());
+ field.setType(SasUtil.getHopDataType(column.getType()));
+ }
Object sasValue = sasRow[index];
Object value = null;
IValueMeta inputValueMeta = null;
@@ -157,7 +154,6 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
if (sasFileProperties.getEncoding() != null) {
value = new String(bytes, sasFileProperties.getEncoding());
} else {
- // TODO: user defined encoding.
value = new String(bytes);
}
}
@@ -171,7 +167,7 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
}
if (sasValue instanceof Float) {
inputValueMeta = new ValueMetaNumber(fieldName);
- value = (double) sasValue;
+ value = sasValue;
}
if (sasValue instanceof Long) {
inputValueMeta = new ValueMetaInteger(fieldName);
@@ -197,6 +193,13 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
// Send the row on its way...
//
putRow(data.outputRowMeta, outputRow);
+
+ // One extra row is handled. Do we need to get more?
+ //
+ if (data.limit > 0 && getLinesInput() >= data.limit) {
+ // Stop the while loop reading lines from the SAS file.
+ break;
+ }
}
} catch (Exception e) {
throw new HopException("Error reading from file " + filename, e);
@@ -204,4 +207,34 @@ public class SasInput extends BaseTransform<SasInputMeta,
SasInputData> {
return true;
}
+
+ private List<Integer> getColumnIndexes(List<Column> columns, String
filename, String metaFilename)
+ throws HopException {
+ List<Integer> indexes = new ArrayList<>();
+
+ if (StringUtil.isEmpty(metaFilename)) {
+ for (SasInputField field : meta.getOutputFields()) {
+
+ int index = -1;
+ for (int c = 0; c < columns.size(); c++) {
+ if (columns.get(c).getName().equalsIgnoreCase(field.getName())) {
+ index = c;
+ break;
+ }
+ }
+ if (index < 0) {
+ throw new HopException(
+ "Field '" + field.getName() + " could not be found in input file
'" + filename);
+ }
+ indexes.add(index);
+ }
+ } else {
+ // Get the column indexes in the same order as in the file.
+ //
+ for (int c = 0; c < columns.size(); c++) {
+ indexes.add(c);
+ }
+ }
+ return indexes;
+ }
}
diff --git
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputData.java
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputData.java
index f65fdef9d9..ff1601f834 100644
---
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputData.java
+++
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputData.java
@@ -24,6 +24,7 @@ import org.apache.hop.pipeline.transform.ITransformData;
@SuppressWarnings("java:S1104")
public class SasInputData extends BaseTransformData implements ITransformData {
public IRowMeta outputRowMeta;
+ public long limit;
public SasInputData() {
super();
diff --git
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java
index ad8d723e02..742ec0ec1e 100644
---
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java
+++
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java
@@ -41,6 +41,7 @@ import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.dialog.MessageBox;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
@@ -58,10 +59,11 @@ import org.eclipse.swt.widgets.Text;
public class SasInputDialog extends BaseTransformDialog {
private static final Class<?> PKG = SasInputMeta.class;
- private CCombo wAccField;
-
private final SasInputMeta input;
- private boolean backupChanged;
+
+ private CCombo wAccField;
+ private TextVar wMetadataFilename;
+ private TextVar wLimit;
private TableView wFields;
public SasInputDialog(
@@ -132,6 +134,48 @@ public class SasInputDialog extends BaseTransformDialog {
wAccField.setLayoutData(fdAccField);
lastControl = wAccField;
+ // Do we use a file as a reference for metadata?
+ //
+ Label wlMetadataFilename = new Label(shell, SWT.RIGHT);
+ wlMetadataFilename.setText(
+ BaseMessages.getString(PKG, "SASInputDialog.MetadataFilename.Label"));
+ PropsUi.setLook(wlMetadataFilename);
+ FormData fdlMetadataFilename = new FormData();
+ fdlMetadataFilename.top = new FormAttachment(lastControl, margin);
+ fdlMetadataFilename.left = new FormAttachment(0, 0);
+ fdlMetadataFilename.right = new FormAttachment(middle, -margin);
+ wlMetadataFilename.setLayoutData(fdlMetadataFilename);
+ wMetadataFilename = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ wMetadataFilename.setToolTipText(
+ BaseMessages.getString(PKG,
"SASInputDialog.MetadataFilename.Tooltip"));
+ PropsUi.setLook(wMetadataFilename);
+ FormData fdMetadataFilename = new FormData();
+ fdMetadataFilename.top = new FormAttachment(lastControl, margin);
+ fdMetadataFilename.left = new FormAttachment(middle, 0);
+ fdMetadataFilename.right = new FormAttachment(100, 0);
+ wMetadataFilename.setLayoutData(fdMetadataFilename);
+ lastControl = wMetadataFilename;
+
+ // Do we use a file as a reference for metadata?
+ //
+ Label wlLimit = new Label(shell, SWT.RIGHT);
+ wlLimit.setText(BaseMessages.getString(PKG, "SASInputDialog.Limit.Label"));
+ PropsUi.setLook(wlLimit);
+ FormData fdlLimit = new FormData();
+ fdlLimit.top = new FormAttachment(lastControl, margin);
+ fdlLimit.left = new FormAttachment(0, 0);
+ fdlLimit.right = new FormAttachment(middle, -margin);
+ wlLimit.setLayoutData(fdlLimit);
+ wLimit = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ wLimit.setToolTipText(BaseMessages.getString(PKG,
"SASInputDialog.Limit.Tooltip"));
+ PropsUi.setLook(wLimit);
+ FormData fdLimit = new FormData();
+ fdLimit.top = new FormAttachment(lastControl, margin);
+ fdLimit.left = new FormAttachment(middle, 0);
+ fdLimit.right = new FormAttachment(100, 0);
+ wLimit.setLayoutData(fdLimit);
+ lastControl = wLimit;
+
// Fill in the source fields...
//
try {
@@ -229,7 +273,8 @@ public class SasInputDialog extends BaseTransformDialog {
/** Copy information from the meta-data input to the dialog fields. */
public void getData() {
wAccField.setText(Const.NVL(input.getAcceptingField(), ""));
-
+ wMetadataFilename.setText(Const.NVL(input.getMetadataFilename(), ""));
+ wLimit.setText(Const.NVL(input.getLimit(), ""));
for (int i = 0; i < input.getOutputFields().size(); i++) {
SasInputField field = input.getOutputFields().get(i);
@@ -244,7 +289,7 @@ public class SasInputDialog extends BaseTransformDialog {
colnr++, field.getPrecision() >= 0 ?
Integer.toString(field.getPrecision()) : "");
item.setText(colnr++, Const.NVL(field.getDecimalSymbol(), ""));
item.setText(colnr++, Const.NVL(field.getGroupingSymbol(), ""));
- item.setText(colnr++, Const.NVL(field.getTrimTypeDesc(), ""));
+ item.setText(colnr, ValueMetaBase.getTrimTypeDesc(field.getTrimType()));
}
wFields.removeEmptyRows();
wFields.setRowNums();
@@ -263,6 +308,8 @@ public class SasInputDialog extends BaseTransformDialog {
public void getInfo(SasInputMeta meta) throws HopTransformException {
// copy info to Meta class (input)
meta.setAcceptingField(wAccField.getText());
+ meta.setMetadataFilename(wMetadataFilename.getText());
+ meta.setLimit(wLimit.getText());
int nrNonEmptyFields = wFields.nrNonEmpty();
meta.getOutputFields().clear();
@@ -300,6 +347,7 @@ public class SasInputDialog extends BaseTransformDialog {
try {
transformName = wTransformName.getText(); // return value
getInfo(input);
+ transformMeta.setChanged(changed);
} catch (HopTransformException e) {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
mb.setMessage(e.toString());
diff --git
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputField.java
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputField.java
index 94bfb6a821..b75bc62feb 100644
---
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputField.java
+++
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputField.java
@@ -17,223 +17,52 @@
package org.apache.hop.pipeline.transforms.sasinput;
-import org.apache.hop.core.Const;
-import org.apache.hop.core.exception.HopXmlException;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.row.value.ValueMetaBase;
-import org.apache.hop.core.row.value.ValueMetaFactory;
-import org.apache.hop.core.xml.XmlHandler;
-import org.w3c.dom.Node;
+import org.apache.hop.metadata.api.HopMetadataProperty;
/** This defines a selected list of fields from the input files */
+@Getter
+@Setter
public class SasInputField implements Cloneable {
- private String name;
- private String rename;
- private int type;
- private int length;
- private int precision;
- private String conversionMask;
- private String decimalSymbol;
- private String groupingSymbol;
- private int trimType;
-
- /**
- * @param name
- * @param rename
- * @param type
- * @param conversionMask
- * @param decimalSymbol
- * @param groupingSymbol
- * @param trimType
- */
- public SasInputField(
- String name,
- String rename,
- int type,
- String conversionMask,
- String decimalSymbol,
- String groupingSymbol,
- int trimType) {
- this.name = name;
- this.rename = rename;
- this.type = type;
- this.conversionMask = conversionMask;
- this.decimalSymbol = decimalSymbol;
- this.groupingSymbol = groupingSymbol;
- this.trimType = trimType;
- }
-
- public SasInputField() {}
-
- @Override
- protected SasInputField clone() {
- try {
- return (SasInputField) super.clone();
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- }
-
- public String getXml() {
+ @HopMetadataProperty private String name;
+ @HopMetadataProperty private String rename;
- return " "
- + XmlHandler.addTagValue("name", name)
- + " "
- + XmlHandler.addTagValue("rename", rename)
- + " "
- + XmlHandler.addTagValue("type",
ValueMetaFactory.getValueMetaName(type))
- + " "
- + XmlHandler.addTagValue("length", length)
- + " "
- + XmlHandler.addTagValue("precision", precision)
- + " "
- + XmlHandler.addTagValue("conversion_mask", conversionMask)
- + " "
- + XmlHandler.addTagValue("decimal", decimalSymbol)
- + " "
- + XmlHandler.addTagValue("grouping", groupingSymbol)
- + " "
- + XmlHandler.addTagValue("trim_type",
ValueMetaBase.getTrimTypeCode(trimType));
- }
-
- public SasInputField(Node node) throws HopXmlException {
- name = XmlHandler.getTagValue(node, "name");
- rename = XmlHandler.getTagValue(node, "rename");
- type = ValueMetaFactory.getIdForValueMeta(XmlHandler.getTagValue(node,
"type"));
- length = Const.toInt(XmlHandler.getTagValue(node, "length"), -1);
- precision = Const.toInt(XmlHandler.getTagValue(node, "precision"), -1);
- conversionMask = XmlHandler.getTagValue(node, "conversion_mask");
- decimalSymbol = XmlHandler.getTagValue(node, "decimal");
- groupingSymbol = XmlHandler.getTagValue(node, "grouping");
- trimType = ValueMetaBase.getTrimTypeByCode(XmlHandler.getTagValue(node,
"trim_type"));
- }
-
- /**
- * @return the name
- */
- public String getName() {
- return name;
- }
-
- /**
- * @param name the name to set
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * @return the rename
- */
- public String getRename() {
- return rename;
- }
-
- /**
- * @param rename the rename to set
- */
- public void setRename(String rename) {
- this.rename = rename;
- }
-
- /**
- * @return the type
- */
- public int getType() {
- return type;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(int type) {
- this.type = type;
- }
-
- /**
- * @return the conversionMask
- */
- public String getConversionMask() {
- return conversionMask;
- }
-
- /**
- * @param conversionMask the conversionMask to set
- */
- public void setConversionMask(String conversionMask) {
- this.conversionMask = conversionMask;
- }
-
- /**
- * @return the decimalSymbol
- */
- public String getDecimalSymbol() {
- return decimalSymbol;
- }
-
- /**
- * @param decimalSymbol the decimalSymbol to set
- */
- public void setDecimalSymbol(String decimalSymbol) {
- this.decimalSymbol = decimalSymbol;
- }
+ @HopMetadataProperty(intCodeConverter =
ValueMetaBase.ValueTypeCodeConverter.class)
+ private int type;
- /**
- * @return the groupingSymbol
- */
- public String getGroupingSymbol() {
- return groupingSymbol;
- }
+ @HopMetadataProperty private int length;
+ @HopMetadataProperty private int precision;
- /**
- * @param groupingSymbol the groupingSymbol to set
- */
- public void setGroupingSymbol(String groupingSymbol) {
- this.groupingSymbol = groupingSymbol;
- }
-
- /**
- * @return the trimType
- */
- public int getTrimType() {
- return trimType;
- }
+ @HopMetadataProperty(key = "conversion_mask")
+ private String conversionMask;
- /**
- * @param trimType the trimType to set
- */
- public void setTrimType(int trimType) {
- this.trimType = trimType;
- }
+ @HopMetadataProperty(key = "decimal")
+ private String decimalSymbol;
- /**
- * @return the precision
- */
- public int getPrecision() {
- return precision;
- }
+ @HopMetadataProperty(key = "grouping")
+ private String groupingSymbol;
- /**
- * @param precision the precision to set
- */
- public void setPrecision(int precision) {
- this.precision = precision;
- }
+ @HopMetadataProperty(
+ key = "trim_type",
+ intCodeConverter = ValueMetaBase.TrimTypeCodeConverter.class)
+ private int trimType;
- /**
- * @return the length
- */
- public int getLength() {
- return length;
- }
+ public SasInputField() {}
- /**
- * @param length the length to set
- */
- public void setLength(int length) {
- this.length = length;
+ public SasInputField(SasInputField field) {
+ this.name = field.name;
+ this.rename = field.rename;
+ this.type = field.type;
+ this.conversionMask = field.conversionMask;
+ this.decimalSymbol = field.decimalSymbol;
+ this.groupingSymbol = field.groupingSymbol;
+ this.trimType = field.trimType;
}
- public String getTrimTypeDesc() {
- return ValueMetaBase.getTrimTypeDesc(trimType);
+ @Override
+ public SasInputField clone() throws CloneNotSupportedException {
+ return new SasInputField(this);
}
}
diff --git
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMeta.java
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMeta.java
index d1b1d11174..8095b91efe 100644
---
a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMeta.java
+++
b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMeta.java
@@ -17,25 +17,31 @@
package org.apache.hop.pipeline.transforms.sasinput;
+import com.epam.parso.Column;
+import com.epam.parso.ColumnFormat;
+import com.epam.parso.impl.SasFileReaderImpl;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.w3c.dom.Node;
@Transform(
id = "SASInput",
@@ -45,52 +51,41 @@ import org.w3c.dom.Node;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input",
keywords = "i18n::SasInputMeta.keyword",
documentationUrl = "/pipeline/transforms/sasinput.html")
+@Getter
+@Setter
public class SasInputMeta extends BaseTransformMeta<SasInput, SasInputData> {
private static final Class<?> PKG = SasInputMeta.class; // for i18n purposes,
- public static final String XML_TAG_FIELD = "field";
-
/** The field in which the filename is placed */
+ @HopMetadataProperty(key = "accept_field")
private String acceptingField;
+ @HopMetadataProperty(key = "field")
private List<SasInputField> outputFields;
+ @HopMetadataProperty(key = "meta_filename")
+ private String metadataFilename;
+
+ @HopMetadataProperty private String limit;
+
public SasInputMeta() {
super(); // allocate BaseTransformMeta
- }
-
- @Override
- public void setDefault() {
outputFields = new ArrayList<>();
}
- @Override
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- try {
- acceptingField = XmlHandler.getTagValue(transformNode, "accept_field");
- int nrFields = XmlHandler.countNodes(transformNode, XML_TAG_FIELD);
- outputFields = new ArrayList<>();
- for (int i = 0; i < nrFields; i++) {
- Node fieldNode = XmlHandler.getSubNodeByNr(transformNode,
XML_TAG_FIELD, i);
- outputFields.add(new SasInputField(fieldNode));
- }
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(
- PKG,
"SASInputMeta.Exception.UnableToReadTransformInformationFromXml"),
- e);
+ public SasInputMeta(SasInputMeta m) {
+ this();
+ this.acceptingField = m.acceptingField;
+ this.metadataFilename = m.metadataFilename;
+ this.limit = m.limit;
+ for (SasInputField field : m.outputFields) {
+ outputFields.add(new SasInputField(field));
}
}
@Override
- public Object clone() {
- SasInputMeta retval = (SasInputMeta) super.clone();
- retval.setOutputFields(new ArrayList<>());
- for (SasInputField field : outputFields) {
- retval.getOutputFields().add(field.clone());
- }
- return retval;
+ public SasInputMeta clone() {
+ return new SasInputMeta(this);
}
@Override
@@ -103,37 +98,49 @@ public class SasInputMeta extends
BaseTransformMeta<SasInput, SasInputData> {
IHopMetadataProvider metadataProvider)
throws HopTransformException {
- for (SasInputField field : outputFields) {
- try {
- IValueMeta valueMeta =
ValueMetaFactory.createValueMeta(field.getRename(), field.getType());
- valueMeta.setLength(field.getLength(), field.getPrecision());
- valueMeta.setDecimalSymbol(field.getDecimalSymbol());
- valueMeta.setGroupingSymbol(field.getGroupingSymbol());
- valueMeta.setConversionMask(field.getConversionMask());
- valueMeta.setTrimType(field.getTrimType());
- valueMeta.setOrigin(name);
-
- inputRowMeta.addValueMeta(valueMeta);
+ String metaFilename = variables.resolve(metadataFilename);
+ if (StringUtils.isEmpty(metaFilename)) {
+
+ for (SasInputField field : outputFields) {
+ try {
+ IValueMeta valueMeta =
+ ValueMetaFactory.createValueMeta(field.getRename(),
field.getType());
+ valueMeta.setLength(field.getLength(), field.getPrecision());
+ valueMeta.setDecimalSymbol(field.getDecimalSymbol());
+ valueMeta.setGroupingSymbol(field.getGroupingSymbol());
+ valueMeta.setConversionMask(field.getConversionMask());
+ valueMeta.setTrimType(field.getTrimType());
+ valueMeta.setOrigin(name);
+
+ inputRowMeta.addValueMeta(valueMeta);
+ } catch (Exception e) {
+ throw new HopTransformException(e);
+ }
+ }
+ } else {
+ // We need to get the file metadata from a reference file to get the row
layout.
+ try (InputStream inputStream = HopVfs.getInputStream(metaFilename,
variables)) {
+ SasFileReaderImpl sasFileReader = new SasFileReaderImpl(inputStream);
+
+ List<Column> columns = sasFileReader.getColumns();
+ for (Column column : columns) {
+ ColumnFormat format = column.getFormat();
+
+ String columnName = column.getName();
+ int length = format.getWidth() == 0 ? -1 : format.getWidth();
+ int precision = format.getPrecision() == 0 ? -1 : format.getWidth();
+ int columnType = SasUtil.getHopDataType(column.getType());
+ IValueMeta valueMeta =
+ ValueMetaFactory.createValueMeta(columnName, columnType, length,
precision);
+ valueMeta.setOrigin(name);
+ inputRowMeta.addValueMeta(valueMeta);
+ }
} catch (Exception e) {
- throw new HopTransformException(e);
+ throw new HopTransformException("Error reading from metadata file: " +
metaFilename);
}
}
}
- @Override
- public String getXml() {
- StringBuilder retval = new StringBuilder();
-
- retval.append(" " + XmlHandler.addTagValue("accept_field",
acceptingField));
- for (SasInputField field : outputFields) {
- retval.append(XmlHandler.openTag(XML_TAG_FIELD));
- retval.append(field.getXml());
- retval.append(XmlHandler.closeTag(XML_TAG_FIELD));
- }
-
- return retval.toString();
- }
-
@Override
public void check(
List<ICheckResult> remarks,
@@ -157,32 +164,4 @@ public class SasInputMeta extends
BaseTransformMeta<SasInput, SasInputData> {
remarks.add(cr);
}
}
-
- /**
- * @return Returns the acceptingField.
- */
- public String getAcceptingField() {
- return acceptingField;
- }
-
- /**
- * @param acceptingField The acceptingField to set.
- */
- public void setAcceptingField(String acceptingField) {
- this.acceptingField = acceptingField;
- }
-
- /**
- * @return the outputFields
- */
- public List<SasInputField> getOutputFields() {
- return outputFields;
- }
-
- /**
- * @param outputFields the outputFields to set
- */
- public void setOutputFields(List<SasInputField> outputFields) {
- this.outputFields = outputFields;
- }
}
diff --git
a/plugins/transforms/sasinput/src/main/resources/org/apache/hop/pipeline/transforms/sasinput/messages/messages_en_US.properties
b/plugins/transforms/sasinput/src/main/resources/org/apache/hop/pipeline/transforms/sasinput/messages/messages_en_US.properties
index b44b52ebe0..02b4e99260 100644
---
a/plugins/transforms/sasinput/src/main/resources/org/apache/hop/pipeline/transforms/sasinput/messages/messages_en_US.properties
+++
b/plugins/transforms/sasinput/src/main/resources/org/apache/hop/pipeline/transforms/sasinput/messages/messages_en_US.properties
@@ -26,6 +26,9 @@ SASInputDialog.AcceptField.Label=Field in the input to use as
filename
SASInputDialog.AcceptField.Tooltip=Specify the field in the input rows to use
as filename
SASInputDialog.Dialog.Title=SAS Input
SASInputDialog.Fields.Label=The selected fields from the files:
+SASInputDialog.MetadataFilename.Label=The file to get metadata from
+SASInputDialog.MetadataFilename.Tooltip=When specified, Hop will read the
columns from this SAS7BDAT file instead of the outpt fields given below.
+
SASInputDialog.FileType.SAS7BAT=SAS7BAT files
SASInputDialog.OutputFieldColumn.Decimal=Decimal
SASInputDialog.OutputFieldColumn.Group=Group
@@ -36,4 +39,6 @@ SASInputDialog.OutputFieldColumn.Precision=Precision
SASInputDialog.OutputFieldColumn.Rename=New name
SASInputDialog.OutputFieldColumn.TrimType=Trim type
SASInputDialog.OutputFieldColumn.Type=Target type
-SasInputMeta.keyword=Sas, input
+SasInputMeta.keyword=SAS, input
+SASInputDialog.Limit.Label=Limit
+SASInputDialog.Limit.Tooltip=Stop reading after the specified number or rows
diff --git
a/plugins/transforms/sasinput/src/test/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMetaTest.java
b/plugins/transforms/sasinput/src/test/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMetaTest.java
index bdc1c2c0b4..9fdf640c6e 100644
---
a/plugins/transforms/sasinput/src/test/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMetaTest.java
+++
b/plugins/transforms/sasinput/src/test/java/org/apache/hop/pipeline/transforms/sasinput/SasInputMetaTest.java
@@ -17,90 +17,66 @@
package org.apache.hop.pipeline.transforms.sasinput;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.hop.core.HopEnvironment;
-import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.plugins.PluginRegistry;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import
org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
class SasInputMetaTest {
- LoadSaveTester loadSaveTester;
- Class<SasInputMeta> testMetaClass = SasInputMeta.class;
-
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
@BeforeEach
void setUpLoadSave() throws Exception {
HopEnvironment.init();
PluginRegistry.init();
- List<String> attributes = Arrays.asList("acceptingField", "outputFields");
-
- Map<String, String> gsMap = new HashMap<>();
-
- Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
- attrValidatorMap.put(
- "outputFields", new ListLoadSaveValidator<>(new
SasInputFieldLoadSaveValidator(), 5));
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
-
- loadSaveTester =
- new LoadSaveTester(
- testMetaClass, attributes, gsMap, gsMap, attrValidatorMap,
typeValidatorMap);
}
@Test
- void testSerialization() throws HopException {
- loadSaveTester.testSerialization();
- }
+ void testSerialization() throws Exception {
+ SasInputMeta meta =
+ TransformSerializationTestUtil.testSerialization("/transform.xml",
SasInputMeta.class);
+ Assertions.assertNotNull(meta);
+ Assertions.assertEquals("filename", meta.getAcceptingField());
+ Assertions.assertEquals("metaFilename", meta.getMetadataFilename());
+ Assertions.assertEquals("1", meta.getLimit());
+ Assertions.assertEquals(4, meta.getOutputFields().size());
+
+ SasInputField f1 = meta.getOutputFields().get(0);
+ Assertions.assertEquals("a", f1.getName());
+ Assertions.assertEquals("newA", f1.getRename());
+ Assertions.assertEquals(IValueMeta.TYPE_STRING, f1.getType());
+ Assertions.assertNull(f1.getConversionMask());
+ Assertions.assertEquals(50, f1.getLength());
+ Assertions.assertEquals(-1, f1.getPrecision());
+ Assertions.assertEquals(IValueMeta.TRIM_TYPE_NONE, f1.getTrimType());
- public class SasInputFieldLoadSaveValidator implements
IFieldLoadSaveValidator<SasInputField> {
- final Random rand = new Random();
+ SasInputField f2 = meta.getOutputFields().get(1);
+ Assertions.assertEquals("b", f2.getName());
+ Assertions.assertEquals("newB", f2.getRename());
+ Assertions.assertEquals(IValueMeta.TYPE_INTEGER, f2.getType());
+ Assertions.assertEquals("#", f2.getConversionMask());
+ Assertions.assertEquals(5, f2.getLength());
+ Assertions.assertEquals(-1, f2.getPrecision());
+ Assertions.assertEquals(IValueMeta.TRIM_TYPE_NONE, f2.getTrimType());
- @Override
- public SasInputField getTestObject() {
- SasInputField rtn = new SasInputField();
- rtn.setRename(UUID.randomUUID().toString());
- rtn.setDecimalSymbol(UUID.randomUUID().toString());
- rtn.setConversionMask(UUID.randomUUID().toString());
- rtn.setGroupingSymbol(UUID.randomUUID().toString());
- rtn.setName(UUID.randomUUID().toString());
- rtn.setTrimType(rand.nextInt(4));
- rtn.setPrecision(rand.nextInt(9));
- rtn.setType(rand.nextInt(7));
- rtn.setLength(rand.nextInt(50));
- return rtn;
- }
+ SasInputField f3 = meta.getOutputFields().get(2);
+ Assertions.assertEquals("c", f3.getName());
+ Assertions.assertEquals("newC", f3.getRename());
+ Assertions.assertEquals("yyyy/MM/dd HH:mm:ss", f3.getConversionMask());
+ Assertions.assertEquals(IValueMeta.TYPE_DATE, f3.getType());
+ Assertions.assertEquals(-1, f3.getLength());
+ Assertions.assertEquals(-1, f3.getPrecision());
+ Assertions.assertEquals(IValueMeta.TRIM_TYPE_NONE, f3.getTrimType());
- @Override
- public boolean validateTestObject(SasInputField testObject, Object actual)
{
- if (!(actual instanceof SasInputField)) {
- return false;
- }
- SasInputField another = (SasInputField) actual;
- return new EqualsBuilder()
- .append(testObject.getName(), another.getName())
- .append(testObject.getTrimType(), another.getTrimType())
- .append(testObject.getType(), another.getType())
- .append(testObject.getPrecision(), another.getPrecision())
- .append(testObject.getRename(), another.getRename())
- .append(testObject.getDecimalSymbol(), another.getDecimalSymbol())
- .append(testObject.getConversionMask(), another.getConversionMask())
- .append(testObject.getGroupingSymbol(), another.getGroupingSymbol())
- .append(testObject.getLength(), another.getLength())
- .isEquals();
- }
+ SasInputField f4 = meta.getOutputFields().get(3);
+ Assertions.assertEquals("d", f4.getName());
+ Assertions.assertEquals("newD", f4.getRename());
+ Assertions.assertEquals("0,000,000.00", f4.getConversionMask());
+ Assertions.assertEquals(IValueMeta.TYPE_NUMBER, f4.getType());
+ Assertions.assertEquals(9, f4.getLength());
+ Assertions.assertEquals(2, f4.getPrecision());
+ Assertions.assertEquals(IValueMeta.TRIM_TYPE_BOTH, f4.getTrimType());
}
}
diff --git a/plugins/transforms/sasinput/src/test/resources/transform.xml
b/plugins/transforms/sasinput/src/test/resources/transform.xml
new file mode 100644
index 0000000000..0b4395ae04
--- /dev/null
+++ b/plugins/transforms/sasinput/src/test/resources/transform.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<transform>
+ <accept_field>filename</accept_field>
+ <field>
+ <length>50</length>
+ <name>a</name>
+ <precision>-1</precision>
+ <rename>newA</rename>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ </field>
+ <field>
+ <conversion_mask>#</conversion_mask>
+ <length>5</length>
+ <name>b</name>
+ <precision>-1</precision>
+ <rename>newB</rename>
+ <trim_type>none</trim_type>
+ <type>Integer</type>
+ </field>
+ <field>
+ <conversion_mask>yyyy/MM/dd HH:mm:ss</conversion_mask>
+ <length>-1</length>
+ <name>c</name>
+ <precision>-1</precision>
+ <rename>newC</rename>
+ <trim_type>none</trim_type>
+ <type>Date</type>
+ </field>
+ <field>
+ <conversion_mask>0,000,000.00</conversion_mask>
+ <length>9</length>
+ <name>d</name>
+ <precision>2</precision>
+ <rename>newD</rename>
+ <trim_type>both</trim_type>
+ <type>Number</type>
+ </field>
+ <limit>1</limit>
+ <meta_filename>metaFilename</meta_filename>
+</transform>
+
