This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 4d0ac86 HOP-3092: move to database plugin
new 69b4cc5 Merge pull request #938 from hansva/HOP-3092
4d0ac86 is described below
commit 4d0ac86f4ebdcb98428acaad9eae38fb75bf1bd7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Jul 15 16:12:24 2021 +0200
HOP-3092: move to database plugin
- Add MDI support
- Bit of cleanup
- add Integration Test
---
.../databases/postgresql-assemblies/pom.xml | 2 +-
.../pgbulkloader/src/assembly/assembly.xml | 2 +-
.../ROOT/pages/pipeline/metadata-injection.adoc | 2 +-
.../database/0016-postgresql-bulkloader.hpl | 138 +++++++++
.../database/main-0016-postgresql-bulkloader.hwf | 116 ++++++++
plugins/databases/postgresql/pom.xml | 2 +-
plugins/transforms/pgbulkloader/pom.xml | 2 +-
.../transforms/pgbulkloader/PGBulkLoader.java | 50 ++--
.../transforms/pgbulkloader/PGBulkLoaderData.java | 19 +-
.../pgbulkloader/PGBulkLoaderDialog.java | 105 ++++---
.../pgbulkloader/PGBulkLoaderMappingMeta.java | 78 +++++
.../transforms/pgbulkloader/PGBulkLoaderMeta.java | 322 +++++++++++----------
.../messages/messages_en_US.properties | 12 +
.../pgbulkloader/PGBulkLoaderMetaTest.java | 9 +-
.../transforms/pgbulkloader/PGBulkLoaderTest.java | 4 +-
15 files changed, 595 insertions(+), 268 deletions(-)
diff --git a/assemblies/plugins/databases/postgresql-assemblies/pom.xml
b/assemblies/plugins/databases/postgresql-assemblies/pom.xml
index df7227e..6b0c0a1 100644
--- a/assemblies/plugins/databases/postgresql-assemblies/pom.xml
+++ b/assemblies/plugins/databases/postgresql-assemblies/pom.xml
@@ -37,7 +37,7 @@
<description></description>
<properties>
- <postgresql.version>42.2.8</postgresql.version>
+ <postgresql.version>42.2.23</postgresql.version>
</properties>
<dependencies>
diff --git
a/assemblies/plugins/transforms/pgbulkloader/src/assembly/assembly.xml
b/assemblies/plugins/transforms/pgbulkloader/src/assembly/assembly.xml
index 85bd372..4542d42 100644
--- a/assemblies/plugins/transforms/pgbulkloader/src/assembly/assembly.xml
+++ b/assemblies/plugins/transforms/pgbulkloader/src/assembly/assembly.xml
@@ -23,7 +23,7 @@
<formats>
<format>zip</format>
</formats>
- <baseDirectory>transforms/pgbulkloader</baseDirectory>
+ <baseDirectory>databases/postgresql</baseDirectory>
<files>
<file>
<source>${project.basedir}/src/main/resources/version.xml</source>
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
index 17eeb89..4bcc9cf 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
@@ -171,7 +171,7 @@ The current (1-july 2021) status is:
|Pipeline executor|N
|Pipeline Logging|N
|Pipeline Probe|N
-|PostgreSQL Bulk Loader|N
+|PostgreSQL Bulk Loader|Y
|Process files|N
|Properties input|N
|Properties output|N
diff --git a/integration-tests/database/0016-postgresql-bulkloader.hpl
b/integration-tests/database/0016-postgresql-bulkloader.hpl
new file mode 100644
index 0000000..87fef2c
--- /dev/null
+++ b/integration-tests/database/0016-postgresql-bulkloader.hpl
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0016-postgresql-bulkloader</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/07/14 15:27:01.196</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/07/14 15:27:01.196</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>test data</from>
+ <to>PostgreSQL Bulk Loader</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>PostgreSQL Bulk Loader</name>
+ <type>PGBulkLoader</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>unit-test-db</connection>
+ <schema>public</schema>
+ <table>testtable</table>
+ <load_action>TRUNCATE</load_action>
+ <dbname_override/>
+ <enclosure>"</enclosure>
+ <delimiter>;</delimiter>
+ <stop_on_error>N</stop_on_error>
+ <mapping>
+ <stream_name>key</stream_name>
+ <field_name>key_data</field_name>
+ <date_mask/>
+ </mapping>
+ <mapping>
+ <stream_name>value</stream_name>
+ <field_name>value_data</field_name>
+ <date_mask/>
+ </mapping>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>test data</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>key_data</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ <field>
+ <set_empty_string>N</set_empty_string>
+ <length>-1</length>
+ <name>value_data</name>
+ <precision>-1</precision>
+ <type>String</type>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>1</item>
+ <item>a</item>
+ </line>
+ <line>
+ <item>2</item>
+ <item>b</item>
+ </line>
+ <line>
+ <item>3</item>
+ <item>c</item>
+ </line>
+ <line>
+ <item>4</item>
+ <item>d</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/database/main-0016-postgresql-bulkloader.hwf
b/integration-tests/database/main-0016-postgresql-bulkloader.hwf
new file mode 100644
index 0000000..f193ec1
--- /dev/null
+++ b/integration-tests/database/main-0016-postgresql-bulkloader.hwf
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0016-postgresql-bulkloader</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2021/06/16 15:52:34.018</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/06/16 15:52:34.018</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>SQL</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql/>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>T</sqlfromfile>
+ <sqlfilename>${PROJECT_HOME}/scripts/script1.sql</sqlfilename>
+ <sendOneStatement>F</sendOneStatement>
+ <connection>unit-test-db</connection>
+ <parallel>N</parallel>
+ <xloc>272</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0016-postgresql-bulkloader.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0016-postgresql-bulkloader.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>transactional</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>464</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>SQL</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>SQL</from>
+ <to>0016-postgresql-bulkloader.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/plugins/databases/postgresql/pom.xml
b/plugins/databases/postgresql/pom.xml
index 29cbac4..1ffc484 100644
--- a/plugins/databases/postgresql/pom.xml
+++ b/plugins/databases/postgresql/pom.xml
@@ -47,7 +47,7 @@
<!-- Test running configuration -->
<maven-surefire-plugin.reuseForks>true</maven-surefire-plugin.reuseForks>
<maven-failsafe-plugin.reuseForks>false</maven-failsafe-plugin.reuseForks>
- <postgresql.version>42.2.8</postgresql.version>
+ <postgresql.version>42.2.23</postgresql.version>
</properties>
<dependencyManagement>
diff --git a/plugins/transforms/pgbulkloader/pom.xml
b/plugins/transforms/pgbulkloader/pom.xml
index 3e5a9e4..a89b825 100644
--- a/plugins/transforms/pgbulkloader/pom.xml
+++ b/plugins/transforms/pgbulkloader/pom.xml
@@ -35,7 +35,7 @@
<name>Hop Plugins Transforms PostgreSQL bulk loader</name>
<properties>
- <postgresql.version>42.2.8</postgresql.version>
+ <postgresql.version>42.2.23</postgresql.version>
</properties>
<dependencies>
diff --git
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java
index 36215b3..21895c9 100644
---
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java
+++
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java
@@ -35,8 +35,8 @@ import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.util.Utils;
import org.apache.hop.i18n.BaseMessages;
-import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.ITransform;
import org.apache.hop.pipeline.transform.TransformMeta;
@@ -49,15 +49,9 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.List;
-/**
- * Performs a bulk load to a postgres table.
- *
- * <p>Based on (copied from) Sven Boden's Oracle Bulk Loader transform
- *
- * @author matt
- * @since 28-mar-2008
- */
+/** Performs a bulk load to a postgres table. */
public class PGBulkLoader extends BaseTransform<PGBulkLoaderMeta,
PGBulkLoaderData>
implements ITransform<PGBulkLoaderMeta, PGBulkLoaderData> {
@@ -89,11 +83,6 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
String tableName =
dm.getQuotedSchemaTableCombination(this, meta.getSchemaName(),
meta.getTableName());
- // Set the date style...
- //
- // contents.append("SET DATESTYLE ISO;"); // This is the default but we
set it anyway...
- // contents.append(Const.CR);
-
// Create a Postgres / Greenplum COPY string for use with a psql client
contents.append("COPY ");
// Table name
@@ -104,18 +93,17 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
contents.append(" ( ");
- String[] streamFields = meta.getFieldStream();
- String[] tableFields = meta.getFieldTable();
+ List<PGBulkLoaderMappingMeta> mapping = meta.getMapping();
- if (streamFields == null || streamFields.length == 0) {
+ if (mapping == null || mapping.isEmpty()) {
throw new HopException("No fields defined to load to database");
}
- for (int i = 0; i < streamFields.length; i++) {
+ for (int i = 0; i < mapping.size(); i++) {
if (i != 0) {
contents.append(", ");
}
- contents.append(dm.quoteField(tableFields[i]));
+ contents.append(dm.quoteField(mapping.get(i).getFieldTable()));
}
contents.append(" ) ");
@@ -216,6 +204,7 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
}
}
+ @Override
public boolean processRow() throws HopException {
try {
@@ -230,6 +219,7 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
if (data != null && pgCopyOut != null) {
pgCopyOut.flush();
pgCopyOut.endCopy();
+ pgCopyOut.close();
}
return false;
@@ -240,9 +230,10 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
// Cache field indexes.
//
- data.keynrs = new int[meta.getFieldStream().length];
+ data.keynrs = new int[meta.getMapping().size()];
for (int i = 0; i < data.keynrs.length; i++) {
- data.keynrs[i] =
getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
+ data.keynrs[i] =
+
getInputRowMeta().indexOfValue(meta.getMapping().get(i).getFieldStream());
}
// execute the copy statement... pgCopyOut is setup there
@@ -293,7 +284,7 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
case IValueMeta.TYPE_STRING:
pgCopyOut.write(data.quote);
- // No longer dump the bytes for a Lazy Conversion;
+ // No longer dump the bytes for a Lazy Conversion
// We need to escape the quote characters in every string
String quoteStr = new String(data.quote);
String escapedString =
@@ -441,6 +432,7 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
}
}
+ @Override
public boolean init() {
String enclosure = resolve(meta.getEnclosure());
@@ -468,13 +460,19 @@ public class PGBulkLoader extends
BaseTransform<PGBulkLoaderMeta, PGBulkLoaderDa
}
data.newline = Const.CR.getBytes();
- data.dateFormatChoices = new int[meta.getFieldStream().length];
+ data.dateFormatChoices = new int[meta.getMapping().size()];
for (int i = 0; i < data.dateFormatChoices.length; i++) {
- if (Utils.isEmpty(meta.getDateMask()[i])) {
+ if (Utils.isEmpty(meta.getMapping().get(i).getDateMask())) {
data.dateFormatChoices[i] =
PGBulkLoaderMeta.NR_DATE_MASK_PASS_THROUGH;
- } else if
(meta.getDateMask()[i].equalsIgnoreCase(PGBulkLoaderMeta.DATE_MASK_DATE)) {
+ } else if (meta.getMapping()
+ .get(i)
+ .getDateMask()
+ .equalsIgnoreCase(PGBulkLoaderMeta.DATE_MASK_DATE)) {
data.dateFormatChoices[i] = PGBulkLoaderMeta.NR_DATE_MASK_DATE;
- } else if
(meta.getDateMask()[i].equalsIgnoreCase(PGBulkLoaderMeta.DATE_MASK_DATETIME)) {
+ } else if (meta.getMapping()
+ .get(i)
+ .getDateMask()
+ .equalsIgnoreCase(PGBulkLoaderMeta.DATE_MASK_DATETIME)) {
data.dateFormatChoices[i] = PGBulkLoaderMeta.NR_DATE_MASK_DATETIME;
} else { // The default : just pass it along...
data.dateFormatChoices[i] =
PGBulkLoaderMeta.NR_DATE_MASK_PASS_THROUGH;
diff --git
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderData.java
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderData.java
index 38af943..d5135e8 100644
---
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderData.java
+++
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderData.java
@@ -27,12 +27,7 @@ import org.postgresql.PGConnection;
import java.io.OutputStream;
-/**
- * Stores data for the GPBulkLoader transform.
- *
- * @author Sven Boden
- * @since 20-feb-2005
- */
+/** Stores data for the GPBulkLoader transform. */
public class PGBulkLoaderData extends BaseTransformData implements
ITransformData {
public Database db;
@@ -57,20 +52,18 @@ public class PGBulkLoaderData extends BaseTransformData
implements ITransformDat
public IValueMeta dateMeta;
public IValueMeta dateTimeMeta;
- /**
- * Default constructor.
- */
+ /** Default constructor. */
public PGBulkLoaderData() {
super();
db = null;
// Let's use ISO 8601 format. This in unambiguous with PostgreSQL
- dateMeta = new ValueMetaDate( "date" );
- dateMeta.setConversionMask( "yyyy-MM-dd" );
+ dateMeta = new ValueMetaDate("date");
+ dateMeta.setConversionMask("yyyy-MM-dd");
- dateTimeMeta = new ValueMetaDate( "date" );
+ dateTimeMeta = new ValueMetaDate("date");
// Let's keep milliseconds. Didn't find a way to keep microseconds (max
resolution with PG)
- dateTimeMeta.setConversionMask( "yyyy-MM-dd HH:mm:ss.SSS" );
+ dateTimeMeta.setConversionMask("yyyy-MM-dd HH:mm:ss.SSS");
}
}
diff --git
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderDialog.java
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderDialog.java
index a6fd0da..98f63e3 100644
---
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderDialog.java
+++
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderDialog.java
@@ -48,27 +48,14 @@ import
org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.apache.hop.ui.pipeline.transform.ITableItemInsertListener;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
-import org.eclipse.swt.events.FocusAdapter;
-import org.eclipse.swt.events.FocusEvent;
-import org.eclipse.swt.events.FocusListener;
-import org.eclipse.swt.events.ModifyListener;
-import org.eclipse.swt.events.SelectionAdapter;
-import org.eclipse.swt.events.SelectionEvent;
+import org.eclipse.swt.events.*;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
-import org.eclipse.swt.widgets.Button;
-import org.eclipse.swt.widgets.Label;
-import org.eclipse.swt.widgets.MessageBox;
-import org.eclipse.swt.widgets.Shell;
-import org.eclipse.swt.widgets.TableItem;
-import org.eclipse.swt.widgets.Text;
-
-import java.util.ArrayList;
-import java.util.HashMap;
+import org.eclipse.swt.widgets.*;
+
import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
public class PGBulkLoaderDialog extends BaseTransformDialog implements
ITransformDialog {
private static final Class<?> PKG = PGBulkLoaderMeta.class; // For Translator
@@ -93,9 +80,6 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
private final PGBulkLoaderMeta input;
- private static final String[] ALL_FILETYPES =
- new String[] {BaseMessages.getString(PKG,
"PGBulkLoaderDialog.Filetype.All")};
-
private ColumnInfo[] ciReturn;
private final Map<String, Integer> inputFields;
@@ -103,6 +87,12 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
/** List of ColumnInfo that should have the field names of the selected
database table */
private final List<ColumnInfo> tableFieldColumns = new ArrayList<>();
+ private static final String PASSTROUGH_LABEL =
"PGBulkLoaderDialog.PassThrough.Label";
+
+ private static final String DATEMASK_LABEL =
"PGBulkLoaderDialog.DateMask.Label";
+
+ private static final String DATETIMEMASK_LABEL =
"PGBulkLoaderDialog.DateTimeMask.Label";
+
public PGBulkLoaderDialog(
Shell parent, IVariables variables, Object in, PipelineMeta
pipelineMeta, String sname) {
super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname);
@@ -120,6 +110,7 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
ModifyListener lsMod = e -> input.setChanged();
FocusListener lsFocusLost =
new FocusAdapter() {
+ @Override
public void focusLost(FocusEvent arg0) {
setTableFieldCombo();
}
@@ -308,6 +299,7 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
wStopOnError.addSelectionListener(
new SelectionAdapter() {
+ @Override
public void widgetSelected(SelectionEvent e) {
input.setChanged();
}
@@ -331,10 +323,10 @@ public class PGBulkLoaderDialog extends
BaseTransformDialog implements ITransfor
fdlReturn.top = new FormAttachment(wStopOnError, margin);
wlReturn.setLayoutData(fdlReturn);
- int UpInsCols = 3;
- int UpInsRows = (input.getFieldTable() != null ?
input.getFieldTable().length : 1);
+ int upInsCols = 3;
+ int upInsRows = (input.getMapping() != null ? input.getMapping().size() :
1);
- ciReturn = new ColumnInfo[UpInsCols];
+ ciReturn = new ColumnInfo[upInsCols];
ciReturn[0] =
new ColumnInfo(
BaseMessages.getString(PKG,
"PGBulkLoaderDialog.ColumnInfo.TableField"),
@@ -349,13 +341,13 @@ public class PGBulkLoaderDialog extends
BaseTransformDialog implements ITransfor
false);
ciReturn[2] =
new ColumnInfo(
- BaseMessages.getString(PKG,
"PGBulkLoaderDialog.ColumnInfo.DateMask"),
+ BaseMessages.getString(PKG, DATEMASK_LABEL),
ColumnInfo.COLUMN_TYPE_CCOMBO,
new String[] {
"",
- BaseMessages.getString(PKG,
"PGBulkLoaderDialog.PassThrough.Label"),
- BaseMessages.getString(PKG, "PGBulkLoaderDialog.DateMask.Label"),
- BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateTimeMask.Label")
+ BaseMessages.getString(PKG, PASSTROUGH_LABEL),
+ BaseMessages.getString(PKG, DATEMASK_LABEL),
+ BaseMessages.getString(PKG, DATETIMEMASK_LABEL)
},
true);
tableFieldColumns.add(ciReturn[0]);
@@ -365,7 +357,7 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
shell,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL |
SWT.H_SCROLL,
ciReturn,
- UpInsRows,
+ upInsRows,
lsMod,
props);
@@ -424,6 +416,7 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
wbTable.addSelectionListener(
new SelectionAdapter() {
+ @Override
public void widgetSelected(SelectionEvent e) {
getTableName();
}
@@ -442,23 +435,24 @@ public class PGBulkLoaderDialog extends
BaseTransformDialog implements ITransfor
public void getData() {
logDebug(BaseMessages.getString(PKG,
"PGBulkLoaderDialog.Log.GettingKeyInfo"));
- if (input.getFieldTable() != null) {
- for (int i = 0; i < input.getFieldTable().length; i++) {
+ if (input.getMapping() != null) {
+ for (int i = 0; i < input.getMapping().size(); i++) {
+ PGBulkLoaderMappingMeta mapping = input.getMapping().get(i);
TableItem item = wReturn.table.getItem(i);
- if (input.getFieldTable()[i] != null) {
- item.setText(1, input.getFieldTable()[i]);
+ if (mapping.getFieldTable() != null) {
+ item.setText(1, mapping.getFieldTable());
}
- if (input.getFieldStream()[i] != null) {
- item.setText(2, input.getFieldStream()[i]);
+ if (mapping.getFieldStream() != null) {
+ item.setText(2, mapping.getFieldStream());
}
- String dateMask = input.getDateMask()[i];
+ String dateMask = mapping.getDateMask();
if (dateMask != null) {
if (PGBulkLoaderMeta.DATE_MASK_PASS_THROUGH.equals(dateMask)) {
- item.setText(3, BaseMessages.getString(PKG,
"PGBulkLoaderDialog.PassThrough.Label"));
+ item.setText(3, BaseMessages.getString(PKG, PASSTROUGH_LABEL));
} else if (PGBulkLoaderMeta.DATE_MASK_DATE.equals(dateMask)) {
- item.setText(3, BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateMask.Label"));
+ item.setText(3, BaseMessages.getString(PKG, DATEMASK_LABEL));
} else if (PGBulkLoaderMeta.DATE_MASK_DATETIME.equals(dateMask)) {
- item.setText(3, BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateTimeMask.Label"));
+ item.setText(3, BaseMessages.getString(PKG, DATETIMEMASK_LABEL));
} else {
item.setText(3, "");
}
@@ -679,7 +673,8 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
private void getInfo(PGBulkLoaderMeta inf) {
int nrFields = wReturn.nrNonEmpty();
- inf.allocate(nrFields);
+ // inf.allocate(nrFields);
+ inf.getMapping().clear();
inf.setDbNameOverride(wDbNameOverride.getText());
@@ -687,20 +682,19 @@ public class PGBulkLoaderDialog extends
BaseTransformDialog implements ITransfor
// CHECKSTYLE:Indentation:OFF
for (int i = 0; i < nrFields; i++) {
TableItem item = wReturn.getNonEmpty(i);
- inf.getFieldTable()[i] = item.getText(1);
- inf.getFieldStream()[i] = item.getText(2);
- if (BaseMessages.getString(PKG, "PGBulkLoaderDialog.PassThrough.Label")
- .equals(item.getText(3))) {
- inf.getDateMask()[i] = PGBulkLoaderMeta.DATE_MASK_PASS_THROUGH;
- } else if (BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateMask.Label")
- .equals(item.getText(3))) {
- inf.getDateMask()[i] = PGBulkLoaderMeta.DATE_MASK_DATE;
- } else if (BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateTimeMask.Label")
- .equals(item.getText(3))) {
- inf.getDateMask()[i] = PGBulkLoaderMeta.DATE_MASK_DATETIME;
+ PGBulkLoaderMappingMeta mapping = new PGBulkLoaderMappingMeta();
+ mapping.setFieldTable(item.getText(1));
+ mapping.setFieldStream(item.getText(2));
+ if (BaseMessages.getString(PKG,
PASSTROUGH_LABEL).equals(item.getText(3))) {
+ mapping.setDateMask(PGBulkLoaderMeta.DATE_MASK_PASS_THROUGH);
+ } else if (BaseMessages.getString(PKG,
DATEMASK_LABEL).equals(item.getText(3))) {
+ mapping.setDateMask(PGBulkLoaderMeta.DATE_MASK_DATE);
+ } else if (BaseMessages.getString(PKG,
DATETIMEMASK_LABEL).equals(item.getText(3))) {
+ mapping.setDateMask(PGBulkLoaderMeta.DATE_MASK_DATETIME);
} else {
- inf.getDateMask()[i] = "";
+ mapping.setDateMask("");
}
+ inf.getMapping().add(mapping);
}
inf.setSchemaName(wSchema.getText());
@@ -783,8 +777,7 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
(tableItem, v) -> {
if (v.getType() == IValueMeta.TYPE_DATE) {
// The default is date mask.
- tableItem.setText(
- 3, BaseMessages.getString(PKG,
"PGBulkLoaderDialog.DateMask.Label"));
+ tableItem.setText(3, BaseMessages.getString(PKG,
DATEMASK_LABEL));
} else {
tableItem.setText(3, "");
}
@@ -853,9 +846,9 @@ public class PGBulkLoaderDialog extends BaseTransformDialog
implements ITransfor
Runnable fieldLoader =
() -> {
if (!wTable.isDisposed() && !wConnection.isDisposed() &&
!wSchema.isDisposed()) {
- final String tableName = wTable.getText(),
- connectionName = wConnection.getText(),
- schemaName = wSchema.getText();
+ final String tableName = wTable.getText();
+ final String connectionName = wConnection.getText();
+ final String schemaName = wSchema.getText();
// clear
for (ColumnInfo colInfo : tableFieldColumns) {
diff --git
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMappingMeta.java
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMappingMeta.java
new file mode 100644
index 0000000..ebeb124
--- /dev/null
+++
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMappingMeta.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.pgbulkloader;
+
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+public class PGBulkLoaderMappingMeta {
+ private static final Class<?> PKG = PGBulkLoaderMeta.class; // For Translator
+
+ /** Field value to dateMask after lookup */
+ @HopMetadataProperty(
+ key = "stream_name",
+ injectionKeyDescription = "PGBulkLoader.Injection.StreamName.Label")
+ private String fieldTable;
+
+ /** Field name in the stream */
+ @HopMetadataProperty(
+ key = "field_name",
+ injectionKeyDescription = "PGBulkLoader.Injection.FieldName.Label")
+ private String fieldStream;
+
+ /** boolean indicating if field needs to be updated */
+ @HopMetadataProperty(
+ key = "date_mask",
+ injectionKeyDescription = "PGBulkLoader.Injection.DateMask.Label")
+ private String dateMask;
+
+ public PGBulkLoaderMappingMeta() {
+ fieldTable = "";
+ fieldStream = "";
+ dateMask = "";
+ }
+
+ public PGBulkLoaderMappingMeta(PGBulkLoaderMappingMeta m) {
+ this.fieldTable = m.fieldTable;
+ this.fieldStream = m.fieldStream;
+ this.dateMask = m.dateMask;
+ }
+
+ public String getFieldTable() {
+ return fieldTable;
+ }
+
+ public void setFieldTable(String fieldTable) {
+ this.fieldTable = fieldTable;
+ }
+
+ public String getFieldStream() {
+ return fieldStream;
+ }
+
+ public void setFieldStream(String fieldStream) {
+ this.fieldStream = fieldStream;
+ }
+
+ public String getDateMask() {
+ return dateMask;
+ }
+
+ public void setDateMask(String dateMask) {
+ this.dateMask = dateMask;
+ }
+}
diff --git
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMeta.java
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMeta.java
index 28e1f15..5b188f1 100644
---
a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMeta.java
+++
b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMeta.java
@@ -17,24 +17,19 @@
package org.apache.hop.pipeline.transforms.pgbulkloader;
-import org.apache.hop.core.CheckResult;
-import org.apache.hop.core.Const;
-import org.apache.hop.core.ICheckResult;
-import org.apache.hop.core.IProvidesDatabaseConnectionInformation;
-import org.apache.hop.core.SqlStatement;
+import org.apache.hop.core.*;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.database.Database;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.DatabaseImpact;
import org.apache.hop.pipeline.Pipeline;
@@ -42,8 +37,8 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.ITransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
-import org.w3c.dom.Node;
+import java.util.ArrayList;
import java.util.List;
@Transform(
@@ -52,7 +47,8 @@ import java.util.List;
description = "i18n::PGBulkLoader.Description",
name = "i18n::PGBulkLoader.Name",
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk",
- documentationUrl =
"https://hop.apache.org/manual/latest/pipeline/transforms/postgresbulkloader.html";)
+ documentationUrl =
+
"https://hop.apache.org/manual/latest/pipeline/transforms/postgresbulkloader.html";)
public class PGBulkLoaderMeta extends BaseTransformMeta
implements ITransformMeta<PGBulkLoader, PGBulkLoaderData>,
IProvidesDatabaseConnectionInformation {
@@ -60,36 +56,56 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
private static final Class<?> PKG = PGBulkLoaderMeta.class; // For Translator
/** what's the schema for the target? */
+ @HopMetadataProperty(
+ key = "schema",
+ injectionKeyDescription = "PGBulkLoader.Injection.Schema.Label")
private String schemaName;
/** what's the table for the target? */
+ @HopMetadataProperty(key = "table", injectionKeyDescription =
"PGBulkLoader.Injection.Table.Label")
private String tableName;
/** database connection */
+ @HopMetadataProperty(
+ key = "connection",
+ storeWithName = true,
+ injectionKeyDescription = "PGBulkLoader.Injection.Connection.Label")
private DatabaseMeta databaseMeta;
/** Field value to dateMask after lookup */
- private String[] fieldTable;
-
- /** Field name in the stream */
- private String[] fieldStream;
-
- /** boolean indicating if field needs to be updated */
- private String[] dateMask;
+ @HopMetadataProperty(
+ injectionGroupKey = "mapping",
+ injectionGroupDescription = "PGBulkLoader.Injection.Mapping.Label" )
+ private List<PGBulkLoaderMappingMeta> mapping;
/** Load action */
+ @HopMetadataProperty(
+ key = "load_action",
+ injectionKeyDescription = "PGBulkLoader.Injection.LoadAction.Label")
private String loadAction;
/** Database name override */
+ @HopMetadataProperty(
+ key = "db_override",
+ injectionKeyDescription = "PGBulkLoader.Injection.DBOverride.Label")
private String dbNameOverride;
/** The field delimiter to use for loading */
+ @HopMetadataProperty(
+ key = "delimiter",
+ injectionKeyDescription = "PGBulkLoader.Injection.Delimiter.Label")
private String delimiter;
/** The enclosure to use for loading */
+ @HopMetadataProperty(
+ key = "enclosure",
+ injectionKeyDescription = "PGBulkLoader.Injection.Enclosure.Label")
private String enclosure;
/** Stop On Error */
+ @HopMetadataProperty(
+ key = "stop_on_error",
+ injectionKeyDescription = "PGBulkLoader.Injection.StopOnError.Label")
private boolean stopOnError;
/*
@@ -133,106 +149,88 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
this.tableName = tableName;
}
- /** @return Returns the fieldTable. */
- public String[] getFieldTable() {
- return fieldTable;
- }
-
- /** @param fieldTable The fieldTable to set. */
- public void setFieldTable(String[] fieldTable) {
- this.fieldTable = fieldTable;
- }
-
- /** @return Returns the fieldStream. */
- public String[] getFieldStream() {
- return fieldStream;
- }
-
- /** @param fieldStream The fieldStream to set. */
- public void setFieldStream(String[] fieldStream) {
- this.fieldStream = fieldStream;
- }
-
- public String[] getDateMask() {
- return dateMask;
- }
-
- public void setDateMask(String[] dateMask) {
- this.dateMask = dateMask;
- }
-
- public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- readData(transformNode, metadataProvider);
- }
-
- public void allocate(int nrvalues) {
- fieldTable = new String[nrvalues];
- fieldStream = new String[nrvalues];
- dateMask = new String[nrvalues];
- }
-
- public Object clone() {
- PGBulkLoaderMeta retval = (PGBulkLoaderMeta) super.clone();
- int nrvalues = fieldTable.length;
-
- retval.allocate(nrvalues);
- System.arraycopy(fieldTable, 0, retval.fieldTable, 0, nrvalues);
- System.arraycopy(fieldStream, 0, retval.fieldStream, 0, nrvalues);
- System.arraycopy(dateMask, 0, retval.dateMask, 0, nrvalues);
- return retval;
- }
-
- private void readData(Node transformNode, IHopMetadataProvider
metadataProvider)
- throws HopXmlException {
- try {
- String con = XmlHandler.getTagValue(transformNode, "connection");
- databaseMeta = DatabaseMeta.loadDatabase(metadataProvider, con);
-
- schemaName = XmlHandler.getTagValue(transformNode, "schema");
- tableName = XmlHandler.getTagValue(transformNode, "table");
-
- enclosure = XmlHandler.getTagValue(transformNode, "enclosure");
- delimiter = XmlHandler.getTagValue(transformNode, "delimiter");
-
- loadAction = XmlHandler.getTagValue(transformNode, "load_action");
- dbNameOverride = XmlHandler.getTagValue(transformNode,
"dbname_override");
- stopOnError = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
"stop_on_error"));
-
- int nrvalues = XmlHandler.countNodes(transformNode, "mapping");
- allocate(nrvalues);
-
- for (int i = 0; i < nrvalues; i++) {
- Node vnode = XmlHandler.getSubNodeByNr(transformNode, "mapping", i);
-
- fieldTable[i] = XmlHandler.getTagValue(vnode, "stream_name");
- fieldStream[i] = XmlHandler.getTagValue(vnode, "field_name");
- if (fieldStream[i] == null) {
- fieldStream[i] = fieldTable[i]; // default: the same name!
- }
- String locDateMask = XmlHandler.getTagValue(vnode, "date_mask");
- if (locDateMask == null) {
- dateMask[i] = "";
- } else {
- if (PGBulkLoaderMeta.DATE_MASK_DATE.equals(locDateMask)
- || PGBulkLoaderMeta.DATE_MASK_PASS_THROUGH.equals(locDateMask)
- || PGBulkLoaderMeta.DATE_MASK_DATETIME.equals(locDateMask)) {
- dateMask[i] = locDateMask;
- } else {
- dateMask[i] = "";
- }
- }
- }
- } catch (Exception e) {
- throw new HopXmlException(
- BaseMessages.getString(
- PKG,
"GPBulkLoaderMeta.Exception.UnableToReadTransformMetaFromXML"),
- e);
- }
- }
+ public List<PGBulkLoaderMappingMeta> getMapping() {
+ return mapping;
+ }
+
+ public void setMapping(List<PGBulkLoaderMappingMeta> mapping) {
+ this.mapping = mapping;
+ }
+
+ // public void loadXml(Node transformNode, IHopMetadataProvider
metadataProvider)
+ // throws HopXmlException {
+ // readData(transformNode, metadataProvider);
+ // }
+ //
+ // public void allocate(int nrvalues) {
+ // fieldTable = new String[nrvalues];
+ // fieldStream = new String[nrvalues];
+ // dateMask = new String[nrvalues];
+ // }
+
+ // @Override
+ // public Object clone() {
+ // PGBulkLoaderMeta retval = (PGBulkLoaderMeta) super.clone();
+ // int nrvalues = fieldTable.length;
+ //
+ // //retval.allocate(nrvalues);
+ // System.arraycopy(fieldTable, 0, retval.fieldTable, 0, nrvalues);
+ // System.arraycopy(fieldStream, 0, retval.fieldStream, 0, nrvalues);
+ // System.arraycopy(dateMask, 0, retval.dateMask, 0, nrvalues);
+ // return retval;
+ // }
+
+ // private void readData(Node transformNode, IHopMetadataProvider
metadataProvider)
+ // throws HopXmlException {
+ // try {
+ // String con = XmlHandler.getTagValue(transformNode, "connection");
+ // databaseMeta = DatabaseMeta.loadDatabase(metadataProvider, con);
+ //
+ // schemaName = XmlHandler.getTagValue(transformNode, "schema");
+ // tableName = XmlHandler.getTagValue(transformNode, "table");
+ //
+ // enclosure = XmlHandler.getTagValue(transformNode, "enclosure");
+ // delimiter = XmlHandler.getTagValue(transformNode, "delimiter");
+ //
+ // loadAction = XmlHandler.getTagValue(transformNode, "load_action");
+ // dbNameOverride = XmlHandler.getTagValue(transformNode,
"dbname_override");
+ // stopOnError =
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
+ // "stop_on_error"));
+ //
+ // int nrvalues = XmlHandler.countNodes(transformNode, "mapping");
+ // allocate(nrvalues);
+ //
+ // for (int i = 0; i < nrvalues; i++) {
+ // Node vnode = XmlHandler.getSubNodeByNr(transformNode, "mapping",
i);
+ //
+ // fieldTable[i] = XmlHandler.getTagValue(vnode, "stream_name");
+ // fieldStream[i] = XmlHandler.getTagValue(vnode, "field_name");
+ // if (fieldStream[i] == null) {
+ // fieldStream[i] = fieldTable[i]; // default: the same name!
+ // }
+ // String locDateMask = XmlHandler.getTagValue(vnode, "date_mask");
+ // if (locDateMask == null) {
+ // dateMask[i] = "";
+ // } else {
+ // if (PGBulkLoaderMeta.DATE_MASK_DATE.equals(locDateMask)
+ // ||
PGBulkLoaderMeta.DATE_MASK_PASS_THROUGH.equals(locDateMask)
+ // || PGBulkLoaderMeta.DATE_MASK_DATETIME.equals(locDateMask)) {
+ // dateMask[i] = locDateMask;
+ // } else {
+ // dateMask[i] = "";
+ // }
+ // }
+ // }
+ // } catch (Exception e) {
+ // throw new HopXmlException(
+ // BaseMessages.getString(
+ // PKG,
"GPBulkLoaderMeta.Exception.UnableToReadTransformMetaFromXML"),
+ // e);
+ // }
+ // }
+ @Override
public void setDefault() {
- fieldTable = null;
databaseMeta = null;
schemaName = "";
tableName = BaseMessages.getString(PKG,
"GPBulkLoaderMeta.DefaultTableName");
@@ -241,36 +239,38 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
enclosure = "\"";
stopOnError = false;
int nrvalues = 0;
- allocate(nrvalues);
- }
-
- public String getXml() {
- StringBuilder retval = new StringBuilder(300);
-
- retval
- .append(" ")
- .append(
- XmlHandler.addTagValue(
- "connection", databaseMeta == null ? "" :
databaseMeta.getName()));
- retval.append(" ").append(XmlHandler.addTagValue("schema", schemaName));
- retval.append(" ").append(XmlHandler.addTagValue("table", tableName));
- retval.append(" ").append(XmlHandler.addTagValue("load_action",
loadAction));
- retval.append(" ").append(XmlHandler.addTagValue("dbname_override",
dbNameOverride));
- retval.append(" ").append(XmlHandler.addTagValue("enclosure",
enclosure));
- retval.append(" ").append(XmlHandler.addTagValue("delimiter",
delimiter));
- retval.append(" ").append(XmlHandler.addTagValue("stop_on_error",
stopOnError));
-
- for (int i = 0; i < fieldTable.length; i++) {
- retval.append(" <mapping>").append(Const.CR);
- retval.append(" ").append(XmlHandler.addTagValue("stream_name",
fieldTable[i]));
- retval.append(" ").append(XmlHandler.addTagValue("field_name",
fieldStream[i]));
- retval.append(" ").append(XmlHandler.addTagValue("date_mask",
dateMask[i]));
- retval.append(" </mapping>").append(Const.CR);
- }
-
- return retval.toString();
- }
+ mapping = new ArrayList<>();
+ // allocate(nrvalues);
+ }
+
+ // public String getXml() {
+ // StringBuilder retval = new StringBuilder(300);
+ //
+ // retval
+ // .append(" ")
+ // .append(
+ // XmlHandler.addTagValue(
+ // "connection", databaseMeta == null ? "" :
databaseMeta.getName()));
+ // retval.append(" ").append(XmlHandler.addTagValue("schema",
schemaName));
+ // retval.append(" ").append(XmlHandler.addTagValue("table",
tableName));
+ // retval.append(" ").append(XmlHandler.addTagValue("load_action",
loadAction));
+ // retval.append(" ").append(XmlHandler.addTagValue("dbname_override",
dbNameOverride));
+ // retval.append(" ").append(XmlHandler.addTagValue("enclosure",
enclosure));
+ // retval.append(" ").append(XmlHandler.addTagValue("delimiter",
delimiter));
+ // retval.append(" ").append(XmlHandler.addTagValue("stop_on_error",
stopOnError));
+ //
+ // for (int i = 0; i < fieldTable.length; i++) {
+ // retval.append(" <mapping>").append(Const.CR);
+ // retval.append("
").append(XmlHandler.addTagValue("stream_name", fieldTable[i]));
+ // retval.append("
").append(XmlHandler.addTagValue("field_name", fieldStream[i]));
+ // retval.append(" ").append(XmlHandler.addTagValue("date_mask",
dateMask[i]));
+ // retval.append(" </mapping>").append(Const.CR);
+ // }
+ //
+ // return retval.toString();
+ // }
+ @Override
public void getFields(
IRowMeta rowMeta,
String origin,
@@ -282,6 +282,7 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
// Default: nothing changes to rowMeta
}
+ @Override
public void check(
List<ICheckResult> remarks,
PipelineMeta pipelineMeta,
@@ -296,7 +297,7 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
String errorMessage = "";
if (databaseMeta != null) {
- Database db = new Database(loggingObject, variables, databaseMeta );
+ Database db = new Database(loggingObject, variables, databaseMeta);
try {
db.connect();
@@ -329,8 +330,8 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
errorFound = false;
errorMessage = "";
- for (int i = 0; i < fieldTable.length; i++) {
- String field = fieldTable[i];
+ for (int i = 0; i < mapping.size(); i++) {
+ String field = mapping.get(i).getFieldTable();
IValueMeta v = r.searchValueMeta(field);
if (v == null) {
@@ -380,8 +381,8 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
errorMessage = "";
boolean errorFound = false;
- for (int i = 0; i < fieldStream.length; i++) {
- IValueMeta v = prev.searchValueMeta(fieldStream[i]);
+ for (int i = 0; i < mapping.size(); i++) {
+ IValueMeta v =
prev.searchValueMeta(mapping.get(i).getFieldStream());
if (v == null) {
if (first) {
first = false;
@@ -390,7 +391,7 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
+ Const.CR;
}
errorFound = true;
- errorMessage += "\t\t" + fieldStream[i] + Const.CR;
+ errorMessage += "\t\t" + mapping.get(i).getFieldStream() +
Const.CR;
}
}
if (errorFound) {
@@ -445,6 +446,7 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
}
}
+ @Override
public SqlStatement getSqlStatements(
IVariables variables,
PipelineMeta pipelineMeta,
@@ -461,20 +463,20 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
IRowMeta tableFields = new RowMeta();
// Now change the field names
- for (int i = 0; i < fieldTable.length; i++) {
- IValueMeta v = prev.searchValueMeta(fieldStream[i]);
+ for (int i = 0; i < mapping.size(); i++) {
+ IValueMeta v = prev.searchValueMeta(mapping.get(i).getFieldStream());
if (v != null) {
IValueMeta tableField = v.clone();
- tableField.setName(fieldTable[i]);
+ tableField.setName(mapping.get(i).getFieldTable());
tableFields.addValueMeta(tableField);
} else {
throw new HopTransformException(
- "Unable to find field [" + fieldStream[i] + "] in the input
rows");
+ "Unable to find field [" + mapping.get(i).getFieldStream() +
"] in the input rows");
}
}
if (!Utils.isEmpty(tableName)) {
- Database db = new Database(loggingObject, variables, databaseMeta );
+ Database db = new Database(loggingObject, variables, databaseMeta);
try {
db.connect();
@@ -523,8 +525,8 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
if (prev != null) {
/* DEBUG CHECK THIS */
// Insert dateMask fields : read/write
- for (int i = 0; i < fieldTable.length; i++) {
- IValueMeta v = prev.searchValueMeta(fieldStream[i]);
+ for (int i = 0; i < mapping.size(); i++) {
+ IValueMeta v = prev.searchValueMeta(mapping.get(i).getFieldStream());
DatabaseImpact ii =
new DatabaseImpact(
@@ -533,8 +535,8 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
transformMeta.getName(),
databaseMeta.getDatabaseName(),
variables.resolve(tableName),
- fieldTable[i],
- fieldStream[i],
+ mapping.get(i).getFieldTable(),
+ mapping.get(i).getFieldStream(),
v != null ? v.getOrigin() : "?",
"",
"Type = " + v.toStringMeta());
@@ -556,6 +558,7 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
return new PGBulkLoaderData();
}
+ @Override
public DatabaseMeta[] getUsedDatabaseConnections() {
if (databaseMeta != null) {
return new DatabaseMeta[] {databaseMeta};
@@ -564,12 +567,13 @@ public class PGBulkLoaderMeta extends BaseTransformMeta
}
}
+ @Override
public IRowMeta getRequiredFields(IVariables variables) throws HopException {
String realTableName = variables.resolve(tableName);
String realSchemaName = variables.resolve(schemaName);
if (databaseMeta != null) {
- Database db = new Database(loggingObject, variables, databaseMeta );
+ Database db = new Database(loggingObject, variables, databaseMeta);
try {
db.connect();
diff --git
a/plugins/transforms/pgbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/pgbulkloader/messages/messages_en_US.properties
b/plugins/transforms/pgbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/pgbulkloader/messages/messages_en_US.properties
index 3641621..03f9479 100644
---
a/plugins/transforms/pgbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/pgbulkloader/messages/messages_en_US.properties
+++
b/plugins/transforms/pgbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/pgbulkloader/messages/messages_en_US.properties
@@ -130,3 +130,15 @@
PGBulkLoaderDialog.DoMapping.UnableToFindTargetFields.Message=It was not possibl
PGBulkLoader.Log.NullInputAndOrPSQLProcess=The input stream of this transform
and/or the PSQL process is null. Load was not executed.
PGBulkLoader.Exception.ExitValueNotZero=psql exited with error value [{0}].
PGBulkLoaderDialog.StopOnError.Label=Stop on error
+PGBulkLoader.Injection.Schema.Label=Name of the Database Schema
+PGBulkLoader.Injection.Table.Label=Name of table in Database
+PGBulkLoader.Injection.Connection.Label=Connection Name
+PGBulkLoader.Injection.LoadAction.Label=Insert Table or Truncate and insert
(INSERT/TRUNCATE)
+PGBulkLoader.Injection.DBOverride.Label=Name to override the database name
+PGBulkLoader.Injection.Delimiter.Label=Delimiter used for BULK Copy
+PGBulkLoader.Injection.Enclosure.Label=Enclosure used for BULK Copy
+PGBulkLoader.Injection.StopOnError.Label=Stop the pipeline on Error
+PGBulkLoader.Injection.StreamName.Label=Table Fieldname
+PGBulkLoader.Injection.FieldName.Label=Fieldname in the stream
+PGBulkLoader.Injection.DateMask.Label=Formatting used for date fields
+PGBulkLoader.Injection.Mapping.Label=Field Mapping
\ No newline at end of file
diff --git
a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMetaTest.java
b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMetaTest.java
index fb7e0fc..00b6766 100644
---
a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMetaTest.java
+++
b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderMetaTest.java
@@ -65,7 +65,7 @@ public class PGBulkLoaderMetaTest {
PluginRegistry.init( false );
List<String> attributes =
Arrays.asList( "schemaName", "tableName", "loadAction",
"dbNameOverride", "delimiter",
- "enclosure", "stopOnError", "fieldTable", "fieldStream", "dateMask",
"databaseMeta" );
+ "enclosure", "stopOnError", "databaseMeta" );
Map<String, String> getterMap = new HashMap<String, String>() {
{
@@ -76,9 +76,6 @@ public class PGBulkLoaderMetaTest {
put( "delimiter", "getDelimiter" );
put( "enclosure", "getEnclosure" );
put( "stopOnError", "isStopOnError" );
- put( "fieldTable", "getFieldTable" );
- put( "fieldStream", "getFieldStream" );
- put( "dateMask", "getDateMask" );
put( "databaseMeta", "getDatabaseMeta" );
}
};
@@ -91,9 +88,6 @@ public class PGBulkLoaderMetaTest {
put( "delimiter", "setDelimiter" );
put( "enclosure", "setEnclosure" );
put( "stopOnError", "setStopOnError" );
- put( "fieldTable", "setFieldTable" );
- put( "fieldStream", "setFieldStream" );
- put( "dateMask", "setDateMask" );
put( "databaseMeta", "setDatabaseMeta" );
}
};
@@ -109,7 +103,6 @@ public class PGBulkLoaderMetaTest {
attrValidatorMap.put( "databaseMeta", new DatabaseMetaLoadSaveValidator()
);
Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
- // typeValidatorMap.put( int[].class.getCanonicalName(), new
PrimitiveIntArrayLoadSaveValidator( new IntLoadSaveValidator(), 1 ) );
loadSaveTester = new LoadSaveTester( testMetaClass, attributes, getterMap,
setterMap, attrValidatorMap, typeValidatorMap );
}
diff --git
a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java
b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java
index 23add85..714d7a5 100644
---
a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java
+++
b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java
@@ -31,6 +31,8 @@ import org.apache.hop.junit.rules.RestoreHopEngineEnvironment;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.junit.*;
+import java.util.ArrayList;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -87,7 +89,7 @@ public class PGBulkLoaderTest {
public void testCreateCommandLine() throws Exception {
PGBulkLoaderMeta meta = mock( PGBulkLoaderMeta.class );
doReturn( new DatabaseMeta() ).when( meta ).getDatabaseMeta();
- doReturn( new String[ 0 ] ).when( meta ).getFieldStream();
+ doReturn( new ArrayList<>()).when( meta ).getMapping();
PGBulkLoaderData data = mock( PGBulkLoaderData.class );
PGBulkLoader spy = spy( new PGBulkLoader(
transformMockHelper.transformMeta, meta, data, 0,
transformMockHelper.pipelineMeta, transformMockHelper.pipeline ) );
