This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new d5c6e6a [FIX] : fix integration test numbering and environment
new 568b03c Merge pull request #796 from hansva/master
d5c6e6a is described below
commit d5c6e6a2d1bdae969feea0aea65fc9e3b7e28bb6
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Fri May 7 18:58:26 2021 +0200
[FIX] : fix integration test numbering and environment
---
integration-tests/actions/hop-config.json | 237 +++++++++++++++++++++
.../{0007-fuzzymatch.hpl => 0012-fuzzymatch.hpl} | 78 +++----
...007-fuzzymatch.hwf => main-0012-fuzzymatch.hwf} | 4 +-
.../unit-test/0011-calculator-dates UNIT.json | 2 +-
...zymatch UNIT.json => 0012-fuzzymatch UNIT.json} | 4 +-
5 files changed, 281 insertions(+), 44 deletions(-)
diff --git a/integration-tests/actions/hop-config.json
b/integration-tests/actions/hop-config.json
new file mode 100644
index 0000000..a9c6a68
--- /dev/null
+++ b/integration-tests/actions/hop-config.json
@@ -0,0 +1,237 @@
+{
+ "variables" : [ {
+ "name" : "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION",
+ "value" : "N",
+ "description" : "System wide flag to allow lenient string to number
conversion for backward compatibility. If this setting is set to \"Y\", an
string starting with digits will be converted successfully into a number.
(example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending
on the decimal and grouping symbol). The default (N) will be to throw an error
if non-numeric symbols are found in the string."
+ }, {
+ "name" : "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE",
+ "value" : "N",
+ "description" : "System wide flag to ignore timezone while writing
date/timestamp value to the database."
+ }, {
+ "name" : "HOP_LOG_SIZE_LIMIT",
+ "value" : "0",
+ "description" : "The log size limit for all pipelines and workflows that
don't have the \"log size limit\" property set in their respective properties."
+ }, {
+ "name" : "HOP_EMPTY_STRING_DIFFERS_FROM_NULL",
+ "value" : "N",
+ "description" : "NULL vs Empty String. If this setting is set to Y, an
empty string and null are different. Otherwise they are not."
+ }, {
+ "name" : "HOP_MAX_LOG_SIZE_IN_LINES",
+ "value" : "0",
+ "description" : "The maximum number of log lines that are kept internally
by Hop. Set to 0 to keep all rows (default)"
+ }, {
+ "name" : "HOP_MAX_LOG_TIMEOUT_IN_MINUTES",
+ "value" : "1440",
+ "description" : "The maximum age (in minutes) of a log line while being
kept internally by Hop. Set to 0 to keep all rows indefinitely (default)"
+ }, {
+ "name" : "HOP_MAX_WORKFLOW_TRACKER_SIZE",
+ "value" : "5000",
+ "description" : "The maximum number of workflow trackers kept in memory"
+ }, {
+ "name" : "HOP_MAX_ACTIONS_LOGGED",
+ "value" : "5000",
+ "description" : "The maximum number of action results kept in memory for
logging purposes."
+ }, {
+ "name" : "HOP_MAX_LOGGING_REGISTRY_SIZE",
+ "value" : "10000",
+ "description" : "The maximum number of logging registry entries kept in
memory for logging purposes."
+ }, {
+ "name" : "HOP_LOG_TAB_REFRESH_DELAY",
+ "value" : "1000",
+ "description" : "The hop log tab refresh delay."
+ }, {
+ "name" : "HOP_LOG_TAB_REFRESH_PERIOD",
+ "value" : "1000",
+ "description" : "The hop log tab refresh period."
+ }, {
+ "name" : "HOP_PLUGIN_CLASSES",
+ "value" : null,
+ "description" : "A comma delimited list of classes to scan for plugin
annotations"
+ }, {
+ "name" : "HOP_PLUGIN_PACKAGES",
+ "value" : null,
+ "description" : "A comma delimited list of packages to scan for plugin
annotations (warning: slow!!)"
+ }, {
+ "name" : "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT",
+ "value" : "0",
+ "description" : "The maximum number of transform performance snapshots to
keep in memory. Set to 0 to keep all snapshots indefinitely (default)"
+ }, {
+ "name" : "HOP_ROWSET_GET_TIMEOUT",
+ "value" : "50",
+ "description" : "The name of the variable that optionally contains an
alternative rowset get timeout (in ms). This only makes a difference for
extremely short lived pipelines."
+ }, {
+ "name" : "HOP_ROWSET_PUT_TIMEOUT",
+ "value" : "50",
+ "description" : "The name of the variable that optionally contains an
alternative rowset put timeout (in ms). This only makes a difference for
extremely short lived pipelines."
+ }, {
+ "name" : "HOP_CORE_TRANSFORMS_FILE",
+ "value" : null,
+ "description" : "The name of the project variable that will contain the
alternative location of the hop-transforms.xml file. You can use this to
customize the list of available internal transforms outside of the codebase."
+ }, {
+ "name" : "HOP_CORE_WORKFLOW_ACTIONS_FILE",
+ "value" : null,
+ "description" : "The name of the project variable that will contain the
alternative location of the hop-workflow-actions.xml file."
+ }, {
+ "name" : "HOP_SERVER_OBJECT_TIMEOUT_MINUTES",
+ "value" : "1440",
+ "description" : "This project variable will set a time-out after which
waiting, completed or stopped pipelines and workflows will be automatically
cleaned up. The default value is 1440 (one day)."
+ }, {
+ "name" : "HOP_PIPELINE_PAN_JVM_EXIT_CODE",
+ "value" : null,
+ "description" : "Set this variable to an integer that will be returned as
the Pan JVM exit code."
+ }, {
+ "name" : "HOP_DISABLE_CONSOLE_LOGGING",
+ "value" : "N",
+ "description" : "Set this variable to Y to disable standard Hop logging to
the console. (stdout)"
+ }, {
+ "name" : "HOP_REDIRECT_STDERR",
+ "value" : "N",
+ "description" : "Set this variable to Y to redirect stderr to Hop logging."
+ }, {
+ "name" : "HOP_REDIRECT_STDOUT",
+ "value" : "N",
+ "description" : "Set this variable to Y to redirect stdout to Hop logging."
+ }, {
+ "name" : "HOP_DEFAULT_NUMBER_FORMAT",
+ "value" : null,
+ "description" : "The name of the variable containing an alternative
default number format"
+ }, {
+ "name" : "HOP_DEFAULT_BIGNUMBER_FORMAT",
+ "value" : null,
+ "description" : "The name of the variable containing an alternative
default bignumber format"
+ }, {
+ "name" : "HOP_DEFAULT_INTEGER_FORMAT",
+ "value" : null,
+ "description" : "The name of the variable containing an alternative
default integer format"
+ }, {
+ "name" : "HOP_DEFAULT_DATE_FORMAT",
+ "value" : null,
+ "description" : "The name of the variable containing an alternative
default date format"
+ }, {
+ "name" : "HOP_DEFAULT_TIMESTAMP_FORMAT",
+ "value" : null,
+ "description" : "The name of the variable containing an alternative
default timestamp format"
+ }, {
+ "name" : "HOP_DEFAULT_SERVLET_ENCODING",
+ "value" : null,
+ "description" : "Defines the default encoding for servlets, leave it empty
to use Java default encoding"
+ }, {
+ "name" : "HOP_FAIL_ON_LOGGING_ERROR",
+ "value" : "N",
+ "description" : "Set this variable to Y when you want the
workflow/pipeline fail with an error when the related logging process (e.g. to
a database) fails."
+ }, {
+ "name" : "HOP_AGGREGATION_MIN_NULL_IS_VALUED",
+ "value" : "N",
+ "description" : "Set this variable to Y to set the minimum to NULL if NULL
is within an aggregate. Otherwise by default NULL is ignored by the MIN
aggregate and MIN is set to the minimum value that is not NULL. See also the
variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO."
+ }, {
+ "name" : "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO",
+ "value" : "N",
+ "description" : "Set this variable to Y to return 0 when all values within
an aggregate are NULL. Otherwise by default a NULL is returned when all values
are NULL."
+ }, {
+ "name" : "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER",
+ "value" : "N",
+ "description" : "Set this variable to Y for backward compatibility for the
Text File Output transform. Setting this to Ywill add no header row at all when
the append option is enabled, regardless if the file is existing or not."
+ }, {
+ "name" : "HOP_PASSWORD_ENCODER_PLUGIN",
+ "value" : "Hop",
+ "description" : "Specifies the password encoder plugin to use by ID (Hop
is the default)."
+ }, {
+ "name" : "HOP_SYSTEM_HOSTNAME",
+ "value" : null,
+ "description" : "You can use this variable to speed up hostname lookup.
Hostname lookup is performed by Hop so that it is capable of logging the server
on which a workflow or pipeline is executed."
+ }, {
+ "name" : "HOP_SERVER_JETTY_ACCEPTORS",
+ "value" : null,
+ "description" : "A variable to configure jetty option: acceptors for Carte"
+ }, {
+ "name" : "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE",
+ "value" : null,
+ "description" : "A variable to configure jetty option: acceptQueueSize for
Carte"
+ }, {
+ "name" : "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME",
+ "value" : null,
+ "description" : "A variable to configure jetty option:
lowResourcesMaxIdleTime for Carte"
+ }, {
+ "name" :
"HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL",
+ "value" : "N",
+ "description" : "Set this variable to Y for backward compatibility for the
Merge Rows (diff) transform. Setting this to Y will use the data from the
reference stream (instead of the comparison stream) in case the compared rows
are identical."
+ }, {
+ "name" : "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE",
+ "value" : "false",
+ "description" : "Set this variable to false to preserve enclosure symbol
after splitting the string in the Split fields transform. Changing it to true
will remove first and last enclosure symbol from the resulting string chunks."
+ }, {
+ "name" : "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES",
+ "value" : "false",
+ "description" : "Set this variable to TRUE to allow your pipeline to pass
'null' fields and/or empty types."
+ }, {
+ "name" : "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT",
+ "value" : "false",
+ "description" : "Set this variable to false to preserve global log
variables defined in pipeline / workflow Properties -> Log panel. Changing it
to true will clear it when export pipeline / workflow."
+ }, {
+ "name" : "HOP_FILE_OUTPUT_MAX_STREAM_COUNT",
+ "value" : "1024",
+ "description" : "This project variable is used by the Text File Output
transform. It defines the max number of simultaneously open files within the
transform. The transform will close/reopen files as necessary to insure the max
is not exceeded"
+ }, {
+ "name" : "HOP_FILE_OUTPUT_MAX_STREAM_LIFE",
+ "value" : "0",
+ "description" : "This project variable is used by the Text File Output
transform. It defines the max number of milliseconds between flushes of files
opened by the transform."
+ }, {
+ "name" : "HOP_USE_NATIVE_FILE_DIALOG",
+ "value" : "N",
+ "description" : "Set this value to Y if you want to use the system file
open/save dialog when browsing files"
+ }, {
+ "name" : "HOP_AUTO_CREATE_CONFIG",
+ "value" : "Y",
+ "description" : "Set this value to N if you don't want to automatically
create a hop configuration file (hop-config.json) when it's missing"
+ } ],
+ "LocaleDefault" : "en_BE",
+ "guiProperties" : {
+ "FontFixedSize" : "13",
+ "MaxUndo" : "100",
+ "DarkMode" : "Y",
+ "FontNoteSize" : "13",
+ "ShowOSLook" : "Y",
+ "FontFixedStyle" : "0",
+ "FontNoteName" : ".AppleSystemUIFont",
+ "FontFixedName" : "Monospaced",
+ "FontGraphStyle" : "0",
+ "FontDefaultSize" : "13",
+ "GraphColorR" : "255",
+ "FontGraphSize" : "13",
+ "IconSize" : "32",
+ "BackgroundColorB" : "255",
+ "FontNoteStyle" : "0",
+ "FontGraphName" : ".AppleSystemUIFont",
+ "FontDefaultName" : ".AppleSystemUIFont",
+ "GraphColorG" : "255",
+ "UseGlobalFileBookmarks" : "Y",
+ "FontDefaultStyle" : "0",
+ "GraphColorB" : "255",
+ "BackgroundColorR" : "255",
+ "BackgroundColorG" : "255",
+ "WorkflowDialogStyle" : "RESIZE,MAX,MIN",
+ "LineWidth" : "1",
+ "ContextDialogShowCategories" : "Y"
+ },
+ "projectsConfig" : {
+ "enabled" : true,
+ "projectMandatory" : true,
+ "environmentMandatory" : false,
+ "defaultProject" : "default",
+ "defaultEnvironment" : null,
+ "standardParentProject" : "default",
+ "standardProjectsFolder" : null,
+ "projectConfigurations" : [ {
+ "projectName" : "default",
+ "projectHome" : "${HOP_CONFIG_FOLDER}",
+ "configFilename" : "project-config.json"
+ } ],
+ "lifecycleEnvironments" : [ {
+ "name" : "dev",
+ "purpose" : "Testing",
+ "projectName" : "default",
+ "configurationFiles" : [ "${PROJECT_HOME}/dev-env-config.json" ]
+ } ],
+ "projectLifecycles" : [ ]
+ }
+}
\ No newline at end of file
diff --git a/integration-tests/transforms/0007-fuzzymatch.hpl
b/integration-tests/transforms/0012-fuzzymatch.hpl
similarity index 99%
rename from integration-tests/transforms/0007-fuzzymatch.hpl
rename to integration-tests/transforms/0012-fuzzymatch.hpl
index 5906317..0c111d2 100644
--- a/integration-tests/transforms/0007-fuzzymatch.hpl
+++ b/integration-tests/transforms/0012-fuzzymatch.hpl
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
- <name>0007-fuzzymatch</name>
+ <name>0012-fuzzymatch</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -39,6 +39,44 @@
</hop>
</order>
<transform>
+ <name>Data grid</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>name</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>jwcdf-name</item>
+ </line>
+ <line>
+ <item>rdjin-name</item>
+ </line>
+ </data>
+ <attributes/>
+ <GUI>
+ <xloc>303</xloc>
+ <yloc>162</yloc>
+ </GUI>
+ </transform>
+ <transform>
<name>Fuzzy match</name>
<type>FuzzyMatch</type>
<description/>
@@ -227,44 +265,6 @@
<yloc>64</yloc>
</GUI>
</transform>
- <transform>
- <name>Data grid</name>
- <type>DataGrid</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <fields>
- <field>
- <name>name</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- </fields>
- <data>
- <line>
- <item>jwcdf-name</item>
- </line>
- <line>
- <item>rdjin-name</item>
- </line>
- </data>
- <attributes/>
- <GUI>
- <xloc>303</xloc>
- <yloc>162</yloc>
- </GUI>
- </transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
diff --git a/integration-tests/transforms/main-0007-fuzzymatch.hwf
b/integration-tests/transforms/main-0012-fuzzymatch.hwf
similarity index 94%
rename from integration-tests/transforms/main-0007-fuzzymatch.hwf
rename to integration-tests/transforms/main-0012-fuzzymatch.hwf
index 0bc5aba..a554262 100644
--- a/integration-tests/transforms/main-0007-fuzzymatch.hwf
+++ b/integration-tests/transforms/main-0012-fuzzymatch.hwf
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<workflow>
- <name>main-0007-fuzzymatch</name>
+ <name>main-0012-fuzzymatch</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -37,7 +37,7 @@
<attributes/>
<test_names>
<test_name>
- <name>0007-fuzzymatch UNIT</name>
+ <name>0012-fuzzymatch UNIT</name>
</test_name>
</test_names>
<parallel>N</parallel>
diff --git
a/integration-tests/transforms/metadata/unit-test/0011-calculator-dates
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0011-calculator-dates
UNIT.json
index 2f7283d..8061945 100644
--- a/integration-tests/transforms/metadata/unit-test/0011-calculator-dates
UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0011-calculator-dates
UNIT.json
@@ -93,6 +93,6 @@
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename":
"/home/matt/git/mattcasters/incubator-hop/integration-tests/transforms/0011-calculator-dates.hpl",
+ "pipeline_filename": "./0011-calculator-dates.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
diff --git a/integration-tests/transforms/metadata/unit-test/0007-fuzzymatch
UNIT.json b/integration-tests/transforms/metadata/unit-test/0012-fuzzymatch
UNIT.json
similarity index 95%
rename from integration-tests/transforms/metadata/unit-test/0007-fuzzymatch
UNIT.json
rename to integration-tests/transforms/metadata/unit-test/0012-fuzzymatch
UNIT.json
index eb458f8..0f82c4d 100644
--- a/integration-tests/transforms/metadata/unit-test/0007-fuzzymatch UNIT.json
+++ b/integration-tests/transforms/metadata/unit-test/0012-fuzzymatch UNIT.json
@@ -63,10 +63,10 @@
}
],
"input_data_sets": [],
- "name": "0007-fuzzymatch UNIT",
+ "name": "0012-fuzzymatch UNIT",
"description": "",
"trans_test_tweaks": [],
"persist_filename": "",
- "pipeline_filename": "./0007-fuzzymatch.hpl",
+ "pipeline_filename": "./0012-fuzzymatch.hpl",
"test_type": "UNIT_TEST"
}
\ No newline at end of file
