This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 8420cc5af5 added metadata injection samples, extended documentation.
fixes #2784
new 31af0d7f50 Merge pull request #2815 from bamaer/2784
8420cc5af5 is described below
commit 8420cc5af58c8d66f0153f3ff9079c15fd7dcbb8
Author: Bart Maertens <[email protected]>
AuthorDate: Sat Apr 8 15:47:12 2023 +0200
added metadata injection samples, extended documentation. fixes #2784
---
.../ROOT/pages/pipeline/transforms/filterrows.adoc | 53 ++++
.../metadata-injection/filter-rows-mdi-child.hpl | 271 +++++++++++++++++++++
.../metadata-injection/filter-rows-mdi-parent.hpl | 242 ++++++++++++++++++
3 files changed, 566 insertions(+)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filterrows.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filterrows.adoc
index d57bc9efae..f682c42aed 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filterrows.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/filterrows.adoc
@@ -116,3 +116,56 @@ This sample transform gives you all the information, such
as the values for the
2. Select the transform, copy it to the clipboard, and then paste it into a
text editor.
Alternatively, you can store the .HPL, and then open the .HPL in a text editor.
3. Find the <condition> element and its nested elements and modify it
accordingly to use it in your MDI scenario.
+
+== MDI Example
+
+The example filter condition below injects the following filter conditions
into a Filter Rows transform.
+
+The full example pipelines are available in the samples project as
`metadata-injection/filter-rows-mdi-parent.hpl` and
`metadata-injection/filter-rows-mdi-child.hpl`
+
+[source, shell]
+----
+stateCode = FL
+AND
+housenr > 100
+----
+
+[source,xml]
+----
+<condition>
+ <negated>N</negated>
+ <conditions>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>stateCode</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>FL</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ <condition>
+ <negated>N</negated>
+ <operator>AND</operator>
+ <leftvalue>housenr</leftvalue>
+ <function>></function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>100</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>####0;-####0</mask>
+ </value>
+ </condition>
+ </conditions>
+ </condition>
+----
\ No newline at end of file
diff --git
a/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-child.hpl
b/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-child.hpl
new file mode 100644
index 0000000000..784d6adb6b
--- /dev/null
+++
b/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-child.hpl
@@ -0,0 +1,271 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0022-filter-rows-child</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/11/29 11:41:29.990</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/11/29 11:41:29.990</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>files/customers-100.txt</from>
+ <to>FL and housenr>100</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>FL and housenr>100</from>
+ <to>True</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>FL and housenr>100</from>
+ <to>False</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>FL and housenr>100</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compare>
+ <condition>
+ <conditions>
+</conditions>
+ <function>=</function>
+ <leftvalue>stateCode</leftvalue>
+ <negated>N</negated>
+ <operator>-</operator>
+ <value>
+ <isnull>N</isnull>
+ <length>-1</length>
+ <name>constant</name>
+ <precision>-1</precision>
+ <text>FL</text>
+ <type>String</type>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>False</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>752</xloc>
+ <yloc>224</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>True</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>752</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>files/customers-100.txt</name>
+ <type>CSVInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <filename>${PROJECT_HOME}/files/customers-100.txt</filename>
+ <filename_field/>
+ <rownum_field/>
+ <include_filename>N</include_filename>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <header>Y</header>
+ <buffer_size>50000</buffer_size>
+ <lazy_conversion>N</lazy_conversion>
+ <add_filename_result>N</add_filename_result>
+ <parallel>N</parallel>
+ <newline_possible>N</newline_possible>
+ <encoding/>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>Integer</type>
+ <format> #</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>10</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>firstname</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>13</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>zip</name>
+ <type>Integer</type>
+ <format> #</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>city</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>8</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>birthdate</name>
+ <type>Date</type>
+ <format>yyyy/MM/dd</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>street</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>11</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>housenr</name>
+ <type>Integer</type>
+ <format> #</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>stateCode</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>9</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ <field>
+ <name>state</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <length>30</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>224</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-parent.hpl
b/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-parent.hpl
new file mode 100644
index 0000000000..9bd735f44a
--- /dev/null
+++
b/plugins/transforms/filterrows/src/main/samples/metadata-injection/filter-rows-mdi-parent.hpl
@@ -0,0 +1,242 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>filter-rows-mdi-parent</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2022/11/29 12:52:24.795</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2022/11/29 12:52:24.795</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>metadata</from>
+ <to>filter-rows-mdi-child.hpl</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>filter-rows-mdi-child.hpl</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+
<filename>${PROJECT_HOME}/metadata-injection/filter-rows-mdi-child.hpl</filename>
+ <run_configuration>local</run_configuration>
+ <source_transform>True</source_transform>
+ <source_output_fields>
+ <source_output_field>
+ <source_output_field_name>id</source_output_field_name>
+ <source_output_field_type>Integer</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>name</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>firstname</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>zip</source_output_field_name>
+ <source_output_field_type>Integer</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>city</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>birthdate</source_output_field_name>
+ <source_output_field_type>Date</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>street</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>housenr</source_output_field_name>
+ <source_output_field_type>Integer</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>stateCode</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>state</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ </source_output_fields>
+ <target_file/>
+ <create_parent_folder>Y</create_parent_folder>
+ <no_execution>N</no_execution>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <mappings>
+ <mapping>
+ <target_transform_name>FL and housenr>100</target_transform_name>
+ <target_attribute_key>SEND_FALSE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>falseTarget</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>FL and housenr>100</target_transform_name>
+ <target_attribute_key>SEND_TRUE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>trueTarget</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>FL and housenr>100</target_transform_name>
+ <target_attribute_key>CONDITION</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>metadata</source_transform>
+ <source_field>conditionXml</source_field>
+ </mapping>
+ </mappings>
+ <attributes/>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>metadata</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item><condition>
+ <negated>N</negated>
+ <conditions>
+ <condition>
+ <negated>N</negated>
+ <leftvalue>stateCode</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>String</type>
+ <text>FL</text>
+ <length>-1</length>
+ <precision>-1</precision>
+ <isnull>N</isnull>
+ <mask/>
+ </value>
+ </condition>
+ <condition>
+ <negated>N</negated>
+ <operator>AND</operator>
+ <leftvalue>housenr</leftvalue>
+ <function>&gt;</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>100</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>####0;-####0</mask>
+ </value>
+ </condition>
+ </conditions>
+ </condition></item>
+ <item>True</item>
+ <item>False</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>conditionXml</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>trueTarget</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>falseTarget</name>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
