This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new 5c8879b [DOC] add list with supported MDI transforms, add hpl to
generate list
new ec09fbb Merge pull request #909 from hansva/master
5c8879b is described below
commit 5c8879b2c674ebdc638d0884811c69ba44701eec
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Jul 1 10:35:36 2021 +0200
[DOC] add list with supported MDI transforms, add hpl to generate list
---
.../ROOT/pages/pipeline/metadata-injection.adoc | 204 +++++++++++++++++++++
.../scripting/0002-udjc-get-MDI-transforms.hpl | 173 +++++++++++++++++
2 files changed, 377 insertions(+)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
index 7d7aa91..7c6dce1 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/metadata-injection.adoc
@@ -32,3 +32,207 @@ We recommend the following basic procedure for using this
transform to inject me
2. Develop pipelines for the repetitive process (the template pipeline), for
metadata injection through the ETL Metadata Injection transform, and for
handling multiple inputs.
The metadata is injected into the template pipeline through any transform that
supports metadata injection.
+
+== Supported Transforms
+
+The goal is to add Metadata Injection support to all transforms,
+The current (1-july 2021) status is:
+
+|===
+|Transform|Supports MDI
+|Abort|Y
+|Add a checksum|Y
+|Add constants|N
+|Add sequence|Y
+|Add value fields changing sequence|N
+|Add XML|Y
+|Analytic query|Y
+|Append streams|Y
+|Avro Decode|Y
+|Avro File Input|Y
+|Azure Event Hubs Listener|N
+|Azure Event Hubs Writer|N
+|Beam BigQuery Input|Y
+|Beam BigQuery Output|Y
+|Beam Bigtable Input|Y
+|Beam Bigtable Output|Y
+|Beam GCP Pub/Sub : Publish|Y
+|Beam GCP Pub/Sub : Subscribe|Y
+|Beam Input|Y
+|Beam Kafka Consume|Y
+|Beam Kafka Produce|Y
+|Beam Output|Y
+|Beam Timestamp|Y
+|Beam Window|Y
+|Block until transforms finish|Y
+|Blocking transform|Y
+|Calculator|Y
+|Call DB procedure|N
+|Cassandra input|Y
+|Cassandra output|Y
+|Change file encoding|N
+|Check if file is locked|N
+|Check if webservice is available|N
+|Clone row|N
+|Closure generator|N
+|Coalesce Fields|Y
+|Column exists|N
+|Combination lookup/update|Y
+|Concat Fields|N
+|Copy rows to result|N
+|Credit card validator|N
+|CSV file input|Y
+|Data grid|Y
+|Database join|N
+|Database lookup|Y
+|De-serialize from file|N
+|Delay row|N
+|Delete|N
+|Detect empty stream|N
+|Dimension lookup/update|Y
+|Dropbox Input |N
+|Dropbox Output |N
+|Dummy (do nothing)|N
+|Dynamic SQL row|N
+|EDI to XML|N
+|Email messages input|N
+|Enhanced JSON Output|N
+|ETL metadata injection|Y
+|Execute a process|N
+|Execute row SQL script|Y
+|Execute SQL script|Y
+|Execute Unit Tests|N
+|Fake data|N
+|File exists|N
+|File Metadata|N
+|Filter rows|Y
+|Fuzzy match|N
+|Generate random value|N
+|Generate rows|Y
+|Get data from XML|N
+|Get file names|N
+|Get files rows count|N
+|Get ID from hop server|N
+|Get Neo4j Logging Info|Y
+|Get records from stream|N
+|Get rows from result|N
+|Get subfolder names|N
+|Get system info|Y
+|Get table names|Y
+|Get variables|Y
+|Google Analytics|Y
+|Google Sheet Output|Y
+|Google Sheets Input|N
+|Group by|Y
+|HTTP client|N
+|HTTP post|N
+|Identify last row in a stream|N
+|If Null|Y
+|Injector|Y
+|Insert / update|Y
+|Java filter|N
+|JavaScript|Y
+|Join rows (cartesian product)|Y
+|JSON input|Y
+|JSON output|N
+|Kafka Consumer|Y
+|Kafka Producer|Y
+|LDAP input|N
+|LDAP output|N
+|Load file content in memory|N
+|Mail|N
+|Mail validator|N
+|Mapping Input|N
+|Mapping Output|N
+|Memory group by|Y
+|Merge join|Y
+|Merge rows (diff)|Y
+|Metadata structure of stream|Y
+|Microsoft Excel input|Y
+|Microsoft Excel writer|N
+|MonetDB bulk loader|Y
+|MongoDB input|Y
+|MongoDB output|Y
+|MQTT Publisher|N
+|MQTT Subscriber|N
+|Multiway merge join|Y
+|Neo4j Cypher|Y
+|Neo4j Generate CSVs|N
+|Neo4j Graph Output|Y
+|Neo4j Import|N
+|Neo4J Output|Y
+|Neo4j Split Graph|N
+|Null if|Y
+|Number range|N
+|Parquet File Input|Y
+|Parquet File Output |Y
+|PGP decrypt stream|N
+|PGP encrypt stream|N
+|Pipeline executor|N
+|Pipeline Logging|N
+|Pipeline Probe|N
+|PostgreSQL Bulk Loader|N
+|Process files|N
+|Properties input|N
+|Properties output|N
+|Regex evaluation|N
+|Replace in string|Y
+|Reservoir sampling|N
+|REST client|N
+|Row denormaliser|Y
+|Row flattener|N
+|Row normaliser|Y
+|Run SSH commands|N
+|Salesforce delete|N
+|Salesforce input|Y
+|Salesforce insert|N
+|Salesforce update|N
+|Salesforce upsert|N
+|Sample rows|N
+|SAS Input|N
+|Select values|Y
+|Serialize to file|N
+|Set field value|Y
+|Set field value to a constant|Y
+|Set variables|N
+|Simple Mapping|N
+|Sort rows|Y
+|Sorted merge|Y
+|Split field to rows|N
+|Split fields|Y
+|Splunk Input|Y
+|SQL file output|N
+|SSTable output|Y
+|Stream lookup|Y
+|Stream Schema Merge|N
+|String operations|Y
+|Strings cut|N
+|Switch / case|Y
+|Synchronize after merge|Y
+|Table compare|N
+|Table exists|N
+|Table input|Y
+|Table output|N
+|Teradata Fastload bulk loader|N
+|Text file input|Y
+|Text file input (deprecated)|N
+|Text file output|Y
+|Token Replacement|Y
+|Unique rows|N
+|Unique rows (HashSet)|N
+|Update|Y
+|User defined Java class|Y
+|User defined Java expression|Y
+|Value mapper|Y
+|Web services lookup|N
+|Workflow executor|N
+|Workflow Logging|N
+|Write to log|N
+|XML input stream (StAX)|N
+|XML join|Y
+|XML output|Y
+|XSD validator|N
+|XSL pipeline|N
+|YAML input |N
+|Zip file|N
+|===
\ No newline at end of file
diff --git a/integration-tests/scripting/0002-udjc-get-MDI-transforms.hpl
b/integration-tests/scripting/0002-udjc-get-MDI-transforms.hpl
new file mode 100644
index 0000000..14e44d9
--- /dev/null
+++ b/integration-tests/scripting/0002-udjc-get-MDI-transforms.hpl
@@ -0,0 +1,173 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0002-udjc-get-MDI-transforms</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2021/07/01 09:35:23.944</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2021/07/01 09:35:23.944</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>dummy row</from>
+ <to>get MDI supported transforms</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>dummy row</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+</fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>get MDI supported transforms</name>
+ <type>UserDefinedJavaClass</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <definitions>
+ <definition>
+ <class_type>TRANSFORM_CLASS</class_type>
+ <class_name>Processor</class_name>
+ <class_source>import org.apache.hop.core.plugins.IPlugin;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.injection.bean.BeanInjectionInfo;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import java.util.List;
+
+
+public boolean processRow() throws HopException
+{
+ // We expect exactly 1 input row
+ //
+ Object[] r = getRow();
+
+ PluginRegistry registry = PluginRegistry.getInstance();
+ List<IPlugin> plugins =
registry.getPlugins(TransformPluginType.class);
+
+ //
+
+
+ for (IPlugin plugin : plugins) {
+ ITransformMeta transformMeta = (ITransformMeta)
registry.loadClass( plugin );
+ String transformName = plugin.getName();
+ String mdiSuport = "N";
+ try {
+ BeanInjectionInfo bii = new BeanInjectionInfo(
transformMeta.getClass() );
+ mdiSuport="Y";
+
+ }catch (Exception e)
+ {
+ }
+ Object[] outputRow = createOutputRow(r, data.outputRowMeta.size());
+ outputRow[0] = transformName;
+ outputRow[1] = mdiSuport;
+
+ putRow(data.outputRowMeta, outputRow);
+
+ }
+
+ // Now render this pipeline...
+ //
+ String svgXml = "test";
+
+
+ // First, get a row from the default input hop
+ //
+
+
+ setOutputDone();
+
+ return false;
+}
+
+
+
+</class_source>
+ </definition>
+ </definitions>
+ <fields>
+ <field>
+ <field_name>transform</field_name>
+ <field_type>String</field_type>
+ <field_length>-1</field_length>
+ <field_precision>-1</field_precision>
+ </field>
+ <field>
+ <field_name>MDI_support</field_name>
+ <field_type>String</field_type>
+ <field_length>-1</field_length>
+ <field_precision>-1</field_precision>
+ </field>
+ </fields>
+ <clear_result_fields>N</clear_result_fields>
+ <info_transforms/>
+ <target_transforms/>
+ <usage_parameters/>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
