This is an automated email from the ASF dual-hosted git repository.
hansva pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
from 73a5245070 Merge pull request #4806 from mattcasters/hop-4793
add 2d59f4e2aa A first stab at issue #3454
add f080312825 A first stab at issue #3454 (copyright fix)
add f42a1cb3e2 Implementing issue #4766 (GCP Secret Manager Variable
Resolver plugin)
add bf6eb73ffa Implementing issue #4771 (Hashicorp Vault plugin)
Improvements for issue #3454 (option to disable variable resolving in tooltips)
add b27d33c642 Implementing issue #4771 (Hashicorp Vault plugin)
Improvements for issue #3454 and documentation
add cd0173f1e4 Improvements for issue #3454 and documentation
add bdc11687a1 Improvements for issue #3454 and documentation
add 1726dbce35 Improvements for issue #4771 (IT for Vault)
add 9ff7abe089 Improvements for issues #3454 and #4771 (integration tests)
add 41e7e433e0 Improvements for issues #3454 and #4771 (RAT)
add 6a1c27a703 Improvements for issues #3454 and #4771 (IT fix)
add 8792de1a8f Improvement for issue #4779 (avoid empty folders)
add 1e0ca4e7da Improvement for issue #3454 (extra integration tests)
add e813dd42d5 Improvement for issue #3454 (extra integration tests)
add a394f68d00 Improvement for issue #3454 (doc blurb about nesting)
add 6143caa393 Improvement for issue #3454 (doc update)
new 2fbd10094b Merge pull request #4761 from mattcasters/hop-3454
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assemblies/plugins/pom.xml | 13 +
.../org/apache/hop/core/HopClientEnvironment.java | 2 +
.../java/org/apache/hop/core/util/StringUtil.java | 4 +
.../org/apache/hop/core/variables/Variables.java | 144 ++++++-
.../resolver/IVariableResolver.java} | 32 +-
.../core/variables/resolver/VariableResolver.java | 53 +++
.../resolver/VariableResolverObjectFactory.java} | 15 +-
.../resolver/VariableResolverPlugin.java} | 12 +-
.../resolver/VariableResolverPluginType.java | 82 ++++
.../serializer/json/JsonMetadataProvider.java | 17 -
.../serializer/json/JsonMetadataSerializer.java | 146 +++----
.../serializer/multi/MultiMetadataProvider.java | 53 +--
.../HopMetadataInstance.java} | 27 +-
.../resolver}/messages/messages_en_US.properties | 3 +-
...-neo4j.yaml => integration-tests-resolver.yaml} | 28 +-
.../gcp-secret-manager-server.png | Bin 0 -> 52965 bytes
.../pipeline-variable-resolver-conf.png | Bin 0 -> 102468 bytes
.../pipeline-variable-resolver-pipeline.png | Bin 0 -> 35100 bytes
.../vault-variable-resolver-server.png | Bin 0 -> 69688 bytes
.../images/variables/resolving-variables.png | Bin 0 -> 72507 bytes
docs/hop-user-manual/modules/ROOT/nav.adoc | 4 +
.../modules/ROOT/pages/metadata-types/index.adoc | 1 +
.../google-secret-manager-variable-resolver.adoc | 53 +++
.../metadata-types/variable-resolver/index.adoc | 61 +++
.../pipeline-variable-resolver.adoc | 85 ++++
.../variable-resolver/vault-variable-resolver.adoc | 57 +++
.../modules/ROOT/pages/variables.adoc | 20 +
.../src/main/java/org/apache/hop/run/HopRun.java | 2 +
.../main/java/org/apache/hop/www/HopServer.java | 2 +
.../0001-vault-add-secrets.hpl} | 120 +++---
.../resolver/0001-vault-resolve-secrets.hpl | 112 +++---
.../0002-pipeline-get-configurations.hpl} | 230 +++++------
.../0002-pipeline-resolver-test.hpl} | 437 ++++++++++-----------
.../0003-nested-resolver.hpl} | 59 ++-
.../datasets/0001-golden-resolved-variables.csv | 2 +
.../datasets/0002-golden-pipeline-resolver.csv | 4 +
.../datasets/0003-golden-nested-resolver.csv | 2 +
integration-tests/resolver/dev-env-config.json | 19 +
.../resolver/files/db1.properties | 11 +-
.../resolver/files/db2.properties | 11 +-
.../resolver/files/db3.properties | 10 +-
.../{duckdb => resolver}/hop-config.json | 0
.../main-0001-resolve-secrets.hwf} | 44 +--
.../main-0002-pipeline-resolver.hwf} | 20 +-
.../main-0003-nested-resolver.hwf} | 58 +--
.../dataset/0001-golden-resolved-variables.json} | 16 +-
.../dataset/0002-golden-pipeline-resolver.json} | 16 +-
.../dataset/0003-golden-nested-resolver.json} | 10 +-
.../metadata/pipeline-run-configuration/local.json | 3 +-
.../unit-test/0001-vault-resolve-secrets UNIT.json | 53 +++
.../0002-pipeline-resolver-test UNIT.json | 53 +++
.../unit-test/0003-nested-resolver UNIT.json | 38 ++
.../resolver/metadata/variable-resolver/conf.json | 13 +
.../metadata/variable-resolver/pipeline.json | 13 +
.../resolver/metadata/variable-resolver/vault.json | 16 +
.../metadata/workflow-run-configuration/local.json | 0
.../project-config.json | 0
lib/pom.xml | 4 +-
plugins/engines/beam/pom.xml | 11 +
.../org/apache/hop/projects/util/ProjectsUtil.java | 2 +
plugins/pom.xml | 1 +
.../{actions/abort => resolvers/pipeline}/pom.xml | 6 +-
.../pipeline}/src/assembly/assembly.xml | 9 +-
.../pipeline/VariableResolverPipeline.java | 207 ++++++++++
.../pipeline}/messages/messages_en_US.properties | 11 +-
.../pipeline}/src/main/resources/version.xml | 0
plugins/{engines => resolvers}/pom.xml | 6 +-
plugins/tech/google/pom.xml | 15 +-
plugins/tech/google/src/assembly/assembly.xml | 10 +
.../GooleSecretManagerVariableResolver.java | 144 +++++++
.../resolver}/messages/messages_en_US.properties | 6 +-
plugins/tech/pom.xml | 1 +
plugins/tech/{neo4j => vault}/pom.xml | 13 +-
.../{neo4j => vault}/src/assembly/assembly.xml | 19 +-
.../resolver/vault/VaultVariableResolver.java | 233 +++++++++++
.../vault/messages/messages_en_US.properties} | 12 +-
.../vault}/src/main/resources/version.xml | 0
rest/src/main/java/org/apache/hop/rest/Hop.java | 2 +
.../main/java/org/apache/hop/ui/core/PropsUi.java | 10 +
.../hop/ui/core/gui/GuiCompositeWidgets.java | 9 +-
.../variables/resolver/VariableResolverEditor.java | 324 +++++++++++++++
.../org/apache/hop/ui/core/widget/TextVar.java | 6 +-
.../main/java/org/apache/hop/ui/hopgui/HopGui.java | 2 +
.../tabs/ConfigGeneralOptionsTab.java | 24 ++
.../core/dialog/messages/messages_en_US.properties | 1 +
.../resolver}/messages/messages_en_US.properties | 4 +-
86 files changed, 2524 insertions(+), 854 deletions(-)
copy core/src/main/java/org/apache/hop/core/{xml/IXml.java =>
variables/resolver/IVariableResolver.java} (54%)
create mode 100644
core/src/main/java/org/apache/hop/core/variables/resolver/VariableResolver.java
copy
core/src/main/java/org/apache/hop/core/{database/DatabaseMetaObjectFactory.java
=> variables/resolver/VariableResolverObjectFactory.java} (72%)
copy
core/src/main/java/org/apache/hop/core/{encryption/TwoWayPasswordEncoderPlugin.java
=> variables/resolver/VariableResolverPlugin.java} (83%)
create mode 100644
core/src/main/java/org/apache/hop/core/variables/resolver/VariableResolverPluginType.java
copy
core/src/main/java/org/apache/hop/metadata/{api/IHasHopMetadataProvider.java =>
util/HopMetadataInstance.java} (57%)
copy core/src/{test/resources/org/apache/hop/foo =>
main/resources/org/apache/hop/core/variables/resolver}/messages/messages_en_US.properties
(84%)
copy docker/integration-tests/{integration-tests-neo4j.yaml =>
integration-tests-resolver.yaml} (72%)
create mode 100644
docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/variable-resolver/gcp-secret-manager-server.png
create mode 100644
docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/variable-resolver/pipeline-variable-resolver-conf.png
create mode 100644
docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/variable-resolver/pipeline-variable-resolver-pipeline.png
create mode 100644
docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/variable-resolver/vault-variable-resolver-server.png
create mode 100644
docs/hop-user-manual/modules/ROOT/assets/images/variables/resolving-variables.png
create mode 100644
docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/google-secret-manager-variable-resolver.adoc
create mode 100644
docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/index.adoc
create mode 100644
docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/pipeline-variable-resolver.adoc
create mode 100644
docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/vault-variable-resolver.adoc
copy integration-tests/{mdi/0023-flattener-child.hpl =>
resolver/0001-vault-add-secrets.hpl} (65%)
copy
plugins/transforms/schemamapping/src/main/samples/transforms/csvinput-with-schemadefinition.hpl
=> integration-tests/resolver/0001-vault-resolve-secrets.hpl (65%)
copy integration-tests/{transforms/0021-row-denormaliser.hpl =>
resolver/0002-pipeline-get-configurations.hpl} (54%)
copy integration-tests/{transforms/0003-field-splitter-escape-string.hpl =>
resolver/0002-pipeline-resolver-test.hpl} (51%)
copy integration-tests/{mdi/0026-inject-using-constants-only-child.hpl =>
resolver/0003-nested-resolver.hpl} (71%)
create mode 100644
integration-tests/resolver/datasets/0001-golden-resolved-variables.csv
create mode 100644
integration-tests/resolver/datasets/0002-golden-pipeline-resolver.csv
create mode 100644
integration-tests/resolver/datasets/0003-golden-nested-resolver.csv
create mode 100644 integration-tests/resolver/dev-env-config.json
copy rest/src/main/resources/hop-rest.properties =>
integration-tests/resolver/files/db1.properties (90%)
copy rest/src/main/resources/hop-rest.properties =>
integration-tests/resolver/files/db2.properties (90%)
copy rest/src/main/resources/hop-rest.properties =>
integration-tests/resolver/files/db3.properties (90%)
copy integration-tests/{duckdb => resolver}/hop-config.json (100%)
copy integration-tests/{transforms/main-0056-result-files.hwf =>
resolver/main-0001-resolve-secrets.hwf} (86%)
copy integration-tests/{transforms/main-0049-closure-generator.hwf =>
resolver/main-0002-pipeline-resolver.hwf} (90%)
copy integration-tests/{transforms/main-0037-apache-tika.hwf =>
resolver/main-0003-nested-resolver.hwf} (67%)
copy integration-tests/{transforms/metadata/dataset/golden-groupby-concat.json
=> resolver/metadata/dataset/0001-golden-resolved-variables.json} (76%)
copy integration-tests/{transforms/metadata/dataset/golden-groupby-concat.json
=> resolver/metadata/dataset/0002-golden-pipeline-resolver.json} (76%)
copy integration-tests/{gcp/metadata/dataset/golden-simple-mapping.json =>
resolver/metadata/dataset/0003-golden-nested-resolver.json} (73%)
copy integration-tests/{json =>
resolver}/metadata/pipeline-run-configuration/local.json (94%)
create mode 100644
integration-tests/resolver/metadata/unit-test/0001-vault-resolve-secrets
UNIT.json
create mode 100644
integration-tests/resolver/metadata/unit-test/0002-pipeline-resolver-test
UNIT.json
create mode 100644
integration-tests/resolver/metadata/unit-test/0003-nested-resolver UNIT.json
create mode 100644
integration-tests/resolver/metadata/variable-resolver/conf.json
create mode 100644
integration-tests/resolver/metadata/variable-resolver/pipeline.json
create mode 100644
integration-tests/resolver/metadata/variable-resolver/vault.json
copy integration-tests/{actions =>
resolver}/metadata/workflow-run-configuration/local.json (100%)
copy integration-tests/{beam_directrunner => resolver}/project-config.json
(100%)
copy plugins/{actions/abort => resolvers/pipeline}/pom.xml (88%)
copy plugins/{databases/universe =>
resolvers/pipeline}/src/assembly/assembly.xml (86%)
create mode 100644
plugins/resolvers/pipeline/src/main/java/org/apache/hop/resolvers/pipeline/VariableResolverPipeline.java
copy {core/src/main/resources/org/apache/hop/core/util =>
plugins/resolvers/pipeline/src/main/resources/org/apache/hop/resolvers/pipeline}/messages/messages_en_US.properties
(69%)
copy plugins/{actions/abort =>
resolvers/pipeline}/src/main/resources/version.xml (100%)
copy plugins/{engines => resolvers}/pom.xml (90%)
create mode 100644
plugins/tech/google/src/main/java/org/apache/hop/core/variables/resolver/GooleSecretManagerVariableResolver.java
copy {ui/src/main/resources/org/apache/hop/ui/hopgui/welcome =>
plugins/tech/google/src/main/resources/org/apache/hop/core/variables/resolver}/messages/messages_en_US.properties
(84%)
copy plugins/tech/{neo4j => vault}/pom.xml (81%)
copy plugins/tech/{neo4j => vault}/src/assembly/assembly.xml (80%)
create mode 100644
plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolver.java
copy
plugins/{actions/ftp/src/main/resources/org/apache/hop/workflow/actions/ftpput/messages/messages_de_DE.properties
=>
tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties}
(57%)
copy plugins/{databases/cockroachdb =>
tech/vault}/src/main/resources/version.xml (100%)
create mode 100644
ui/src/main/java/org/apache/hop/ui/core/variables/resolver/VariableResolverEditor.java
copy {core/src/test/resources/org/apache/hop/foo =>
ui/src/main/resources/org/apache/hop/ui/core/variables/resolver}/messages/messages_en_US.properties
(81%)
