This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 8bed329070 finish zip outputstream before closing. fixes #3472
new d06b390608 Merge pull request #3524 from bamaer/master
8bed329070 is described below
commit 8bed3290703d098125317a0e5ab3afe2572652ad
Author: Bart Maertens <[email protected]>
AuthorDate: Thu Dec 28 20:02:05 2023 +0100
finish zip outputstream before closing. fixes #3472
---
...ion-verify-generated-files-in-zip-are-valid.hpl | 442 +++++++++++++++++++++
integration-tests/actions/main-0008-zip-files.hwf | 71 +++-
.../workflow/actions/zipfile/ActionZipFile.java | 1 +
3 files changed, 497 insertions(+), 17 deletions(-)
diff --git
a/integration-tests/actions/0008-zip-action-verify-generated-files-in-zip-are-valid.hpl
b/integration-tests/actions/0008-zip-action-verify-generated-files-in-zip-are-valid.hpl
new file mode 100644
index 0000000000..16c890a27d
--- /dev/null
+++
b/integration-tests/actions/0008-zip-action-verify-generated-files-in-zip-are-valid.hpl
@@ -0,0 +1,442 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0008-zip-action-verify-generated-files-in-zip-are-valid</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2023/12/28 19:44:41.584</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2023/12/28 19:44:41.584</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>10</fontsize>
+ <height>26</height>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ <note>verify https://github.com/apache/hop/issues/3472</note>
+ <width>265</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>generate 1 row</from>
+ <to>get today</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>get today</from>
+ <to>get current year</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>get current year</from>
+ <to>date -> str</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>date -> str</from>
+ <to>build filename</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>build filename</from>
+ <to>read file from zip</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read file from zip</from>
+ <to>Detect empty stream</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Detect empty stream</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>1184</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Detect empty stream</name>
+ <type>DetectEmptyStream</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>1008</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>build filename</name>
+ <type>ScriptValueMod</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <optimizationLevel>9</optimizationLevel>
+ <jsScripts>
+ <jsScript>
+ <jsScript_type>0</jsScript_type>
+ <jsScript_name>Script 1</jsScript_name>
+ <jsScript_script>//Script here
+
+var filename = 'zip:file://' + getVariable('java.io.tmpdir', '') +
'/zip-action-archive/' + current_year + '.zip!' + today_str +
'.txt';</jsScript_script>
+ </jsScript>
+ </jsScripts>
+ <fields>
+ <field>
+ <name>filename</name>
+ <rename>filename</rename>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ <replace>N</replace>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>704</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>date -> str</name>
+ <type>SelectValues</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>today</name>
+ <rename>today_str</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>-2</precision>
+ <conversion_mask>yyyy-MM-dd</conversion_mask>
+ <date_format_lenient>false</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>false</lenient_string_to_number>
+ <encoding/>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <storage_type/>
+ </meta>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>generate 1 row</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+</fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <limit>1</limit>
+ <never_ending>N</never_ending>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>get current year</name>
+ <type>Calculator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <calculation>
+ <calc_type>YEAR_OF_DATE</calc_type>
+ <field_a>today</field_a>
+ <field_name>current_year</field_name>
+ <remove>N</remove>
+ <value_length>-1</value_length>
+ <value_precision>-1</value_precision>
+ <value_type>Integer</value_type>
+ </calculation>
+ <failIfNoFile>Y</failIfNoFile>
+ <attributes/>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>get today</name>
+ <type>SystemInfo</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>today</name>
+ <type>system date (variable)</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>304</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read file from zip</name>
+ <type>TextFileInput2</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <accept_filenames>Y</accept_filenames>
+ <passing_through_fields>N</passing_through_fields>
+ <accept_field>filename</accept_field>
+ <accept_transform_name>build filename</accept_transform_name>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <enclosure_breaks>N</enclosure_breaks>
+ <escapechar/>
+ <header>Y</header>
+ <nr_headerlines>1</nr_headerlines>
+ <footer>N</footer>
+ <nr_footerlines>1</nr_footerlines>
+ <line_wrapped>N</line_wrapped>
+ <nr_wraps>1</nr_wraps>
+ <layout_paged>N</layout_paged>
+ <nr_lines_per_page>80</nr_lines_per_page>
+ <nr_lines_doc_header>0</nr_lines_doc_header>
+ <noempty>Y</noempty>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <rownumByFile>N</rownumByFile>
+ <rownum_field/>
+ <format>mixed</format>
+ <encoding/>
+ <length>Characters</length>
+ <add_to_result_filenames>Y</add_to_result_filenames>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ <type>CSV</type>
+ <compression>None</compression>
+ </file>
+ <filters>
+ </filters>
+ <fields>
+ <field>
+ <name>today</name>
+ <type>Date</type>
+ <format>yyyy/MM/dd HH:mm:ss.SSS</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <nullif>-</nullif>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>counter</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <nullif>-</nullif>
+ <ifnull/>
+ <position>-1</position>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>new_date_str</name>
+ <type>Date</type>
+ <format>yyyy-MM-dd</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <nullif>-</nullif>
+ <ifnull/>
+ <position>-1</position>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>new_date_year</name>
+ <type>Integer</type>
+ <format>#</format>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <nullif>-</nullif>
+ <ifnull/>
+ <position>-1</position>
+ <length>15</length>
+ <precision>0</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ <field>
+ <name>filename</name>
+ <type>String</type>
+ <format/>
+ <currency>$</currency>
+ <decimal>.</decimal>
+ <group>,</group>
+ <nullif>-</nullif>
+ <ifnull/>
+ <position>-1</position>
+ <length>30</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <error_ignored>N</error_ignored>
+ <skip_bad_files>N</skip_bad_files>
+ <file_error_field/>
+ <file_error_message_field/>
+ <error_line_skipped>N</error_line_skipped>
+ <error_count_field/>
+ <error_fields_field/>
+ <error_text_field/>
+ <bad_line_files_destination_directory/>
+ <bad_line_files_extension>warning</bad_line_files_extension>
+ <error_line_files_destination_directory/>
+ <error_line_files_extension>error</error_line_files_extension>
+ <line_number_files_destination_directory/>
+ <line_number_files_extension>line</line_number_files_extension>
+ <date_format_lenient>Y</date_format_lenient>
+ <date_format_locale>en_US</date_format_locale>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>848</xloc>
+ <yloc>128</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/actions/main-0008-zip-files.hwf
b/integration-tests/actions/main-0008-zip-files.hwf
index f75d21b145..eb2f5e1a14 100644
--- a/integration-tests/actions/main-0008-zip-files.hwf
+++ b/integration-tests/actions/main-0008-zip-files.hwf
@@ -53,24 +53,22 @@ limitations under the License.
<description/>
<type>PIPELINE</type>
<attributes/>
-
<filename>${PROJECT_HOME}/0008-zip-action-generate-files-to-zip.hpl</filename>
- <params_from_previous>N</params_from_previous>
- <exec_per_row>N</exec_per_row>
- <clear_rows>N</clear_rows>
- <clear_files>N</clear_files>
- <set_logfile>N</set_logfile>
- <logfile/>
- <logext/>
<add_date>N</add_date>
<add_time>N</add_time>
- <loglevel>Basic</loglevel>
- <set_append_logfile>N</set_append_logfile>
- <wait_until_finished>Y</wait_until_finished>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
- <run_configuration>local</run_configuration>
+ <exec_per_row>N</exec_per_row>
+
<filename>${PROJECT_HOME}/0008-zip-action-generate-files-to-zip.hpl</filename>
+ <loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>288</xloc>
<yloc>48</yloc>
@@ -118,7 +116,7 @@ limitations under the License.
</field>
</fields>
<parallel>N</parallel>
- <xloc>736</xloc>
+ <xloc>1024</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@@ -128,10 +126,35 @@ limitations under the License.
<type>ABORT</type>
<attributes/>
<always_log_rows>N</always_log_rows>
- <message/>
<parallel>N</parallel>
- <xloc>560</xloc>
- <yloc>176</yloc>
+ <xloc>672</xloc>
+ <yloc>160</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>verify files in zip are valid (#3472)</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+
<filename>${PROJECT_HOME}/0008-zip-action-verify-generated-files-in-zip-are-valid.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>784</xloc>
+ <yloc>48</yloc>
<attributes_hac/>
</action>
</actions>
@@ -152,13 +175,27 @@ limitations under the License.
</hop>
<hop>
<from>zip file not empty?</from>
+ <to>abort</to>
+ <enabled>Y</enabled>
+ <evaluation>N</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>zip file not empty?</from>
+ <to>verify files in zip are valid (#3472)</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>verify files in zip are valid (#3472)</from>
<to>delete zip files</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
- <from>zip file not empty?</from>
+ <from>verify files in zip are valid (#3472)</from>
<to>abort</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
diff --git
a/plugins/actions/zipfile/src/main/java/org/apache/hop/workflow/actions/zipfile/ActionZipFile.java
b/plugins/actions/zipfile/src/main/java/org/apache/hop/workflow/actions/zipfile/ActionZipFile.java
index 18fc8b6fa8..7a21aa0c37 100644
---
a/plugins/actions/zipfile/src/main/java/org/apache/hop/workflow/actions/zipfile/ActionZipFile.java
+++
b/plugins/actions/zipfile/src/main/java/org/apache/hop/workflow/actions/zipfile/ActionZipFile.java
@@ -580,6 +580,7 @@ public class ActionZipFile extends ActionBase implements
Cloneable, IAction {
}
out.flush();
out.closeEntry();
+ out.finish();
// Close the current file input stream
in.close();
