This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 17eecc61af issue #6716 (small path prefix feature for the Hashicorp
Vault variable resolver) (#6722)
17eecc61af is described below
commit 17eecc61afbf035572df8a9319184dbaa36da54e
Author: Matt Casters <[email protected]>
AuthorDate: Fri Mar 6 13:39:08 2026 +0100
issue #6716 (small path prefix feature for the Hashicorp Vault variable
resolver) (#6722)
---
.../hashicorp-vault-variable-resolver.adoc | 15 ++
.../resolver/0001-vault-add-secrets.hpl | 24 +++-
... => 0001-vault-resolve-secrets-with-prefix.hpl} | 152 ++++++++++-----------
.../resolver/main-0001-resolve-secrets.hwf | 25 +++-
...001-vault-resolve-secrets-with-prefix UNIT.json | 53 +++++++
.../metadata/variable-resolver/vault-prefix.json | 17 +++
.../resolver/vault/VaultVariableResolver.java | 33 +++--
.../vault/messages/messages_en_US.properties | 1 +
8 files changed, 230 insertions(+), 90 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
index a89a345ba4..85db1acc79 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
@@ -27,6 +27,21 @@ Here are the options to use:
* Vault address: The base address and port of the Vault server (for example:
https://vault-server:8200)
* Vault token: The token to use to authenticate
+* Path prefix: an optional path prefix which gets added before the key paths
in the resolver expressions. For example, if you put `kv-other/data` in here,
expression
+
+[source]
+----
+#{vault:db:password}
+----
+
+will resolve internally to:
+
+[source]
+----
+#{vault:kv-other/data/db:password}
+----
+
+
* Validate HTTPS connections?: It's recommended to enable connection
validation in production. This secures the connection with the X.509
certificate specified in one of either next option.
* PEM file path: The name of the file (VFS) containing the X.509 certificate
string
* PEM string: The X.509 string itself in case you're not using a file
diff --git a/integration-tests/resolver/0001-vault-add-secrets.hpl
b/integration-tests/resolver/0001-vault-add-secrets.hpl
index 9ad503a4f2..aa5deade06 100644
--- a/integration-tests/resolver/0001-vault-add-secrets.hpl
+++ b/integration-tests/resolver/0001-vault-add-secrets.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0001-add-secrets</name>
+ <name>0001-vault-add-secrets</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -80,6 +80,10 @@ limitations under the License.
<result>
<name>result</name>
</result>
+ <retryMethods>
+</retryMethods>
+ <retryStatusCodes>
+</retryStatusCodes>
<trustStorePassword>Encrypted </trustStorePassword>
<urlField>url</urlField>
<urlInField>Y</urlInField>
@@ -103,7 +107,7 @@ limitations under the License.
<data>
<line>
<item>myroot</item>
- <item>http://vault:8200/v1/secret/data/hop</item>
+ <item>http://localhost:8200/v1/secret/data/hop</item>
<item>application/json</item>
<item>{
"data": {
@@ -121,29 +125,45 @@ limitations under the License.
<field>
<length>-1</length>
<precision>-1</precision>
+ <currency/>
<set_empty_string>N</set_empty_string>
<name>root</name>
+ <format/>
+ <group/>
+ <decimal/>
<type>String</type>
</field>
<field>
<length>-1</length>
<precision>-1</precision>
+ <currency/>
<set_empty_string>N</set_empty_string>
<name>url</name>
+ <format/>
+ <group/>
+ <decimal/>
<type>String</type>
</field>
<field>
<length>-1</length>
<precision>-1</precision>
+ <currency/>
<set_empty_string>N</set_empty_string>
<name>contentType</name>
+ <format/>
+ <group/>
+ <decimal/>
<type>String</type>
</field>
<field>
<length>-1</length>
<precision>-1</precision>
+ <currency/>
<set_empty_string>N</set_empty_string>
<name>data</name>
+ <format/>
+ <group/>
+ <decimal/>
<type>String</type>
</field>
</fields>
diff --git a/integration-tests/resolver/0001-vault-add-secrets.hpl
b/integration-tests/resolver/0001-vault-resolve-secrets-with-prefix.hpl
similarity index 54%
copy from integration-tests/resolver/0001-vault-add-secrets.hpl
copy to integration-tests/resolver/0001-vault-resolve-secrets-with-prefix.hpl
index 9ad503a4f2..e19f293819 100644
--- a/integration-tests/resolver/0001-vault-add-secrets.hpl
+++ b/integration-tests/resolver/0001-vault-resolve-secrets-with-prefix.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0001-add-secrets</name>
+ <name>0001-vault-resolve-secrets-with-prefix</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -31,22 +31,43 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2025/01/14 15:47:02.970</created_date>
+ <created_date>2025/01/14 15:54:11.134</created_date>
<modified_user>-</modified_user>
- <modified_date>2025/01/14 15:47:02.970</modified_date>
+ <modified_date>2025/01/14 15:54:11.134</modified_date>
</info>
<notepads>
+ <notepad>
+ <backgroundcolorblue>210</backgroundcolorblue>
+ <backgroundcolorgreen>136</backgroundcolorgreen>
+ <backgroundcolorred>15</backgroundcolorred>
+ <bordercolorblue>250</bordercolorblue>
+ <bordercolorgreen>231</bordercolorgreen>
+ <bordercolorred>200</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>250</fontcolorblue>
+ <fontcolorgreen>231</fontcolorgreen>
+ <fontcolorred>200</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Cantarell</fontname>
+ <fontsize>11</fontsize>
+ <height>32</height>
+ <xloc>48</xloc>
+ <yloc>112</yloc>
+ <note>The vault-prefix resolver contains the path prefix making the
expressions shorter.
+This tests that functionality.</note>
+ <width>32</width>
+ </notepad>
</notepads>
<order>
<hop>
- <from>Secrets</from>
- <to>REST client</to>
+ <from>Get variables</from>
+ <to>Output</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>REST client</name>
- <type>Rest</type>
+ <name>Get variables</name>
+ <type>GetVariable</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@@ -55,102 +76,77 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <applicationType>TEXT PLAIN</applicationType>
- <bodyField>data</bodyField>
- <connectionTimeout>10000</connectionTimeout>
- <dynamicMethod>N</dynamicMethod>
- <headers>
- <header>
- <field>contentType</field>
- <name>Content-Type</name>
- </header>
- <header>
- <field>root</field>
- <name>X-Vault-Token</name>
- </header>
- </headers>
- <ignoreSsl>N</ignoreSsl>
- <matrixParameters>
-</matrixParameters>
- <method>POST</method>
- <parameters>
-</parameters>
- <preemptive>N</preemptive>
- <readTimeout>10000</readTimeout>
- <result>
- <name>result</name>
- </result>
- <trustStorePassword>Encrypted </trustStorePassword>
- <urlField>url</urlField>
- <urlInField>Y</urlInField>
- <attributes/>
- <GUI>
- <xloc>208</xloc>
- <yloc>64</yloc>
- </GUI>
- </transform>
- <transform>
- <name>Secrets</name>
- <type>DataGrid</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <data>
- <line>
- <item>myroot</item>
- <item>http://vault:8200/v1/secret/data/hop</item>
- <item>application/json</item>
- <item>{
- "data": {
- "db": "test",
- "hostname": "localhost",
- "password": "some-password",
- "port": "3306",
- "username": "john"
- },
- "options": {}
-}</item>
- </line>
- </data>
<fields>
<field>
<length>-1</length>
+ <name>json-data</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{vault-prefix:hop}</variable>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>hostname</name>
<precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- <name>root</name>
+ <trim_type>none</trim_type>
<type>String</type>
+ <variable>#{vault-prefix:hop:hostname}</variable>
</field>
<field>
<length>-1</length>
+ <name>port</name>
<precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- <name>url</name>
+ <trim_type>none</trim_type>
<type>String</type>
+ <variable>#{vault-prefix:hop:port}</variable>
</field>
<field>
<length>-1</length>
+ <name>db</name>
<precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- <name>contentType</name>
+ <trim_type>none</trim_type>
<type>String</type>
+ <variable>#{vault-prefix:hop:db}</variable>
</field>
<field>
<length>-1</length>
+ <name>username</name>
<precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- <name>data</name>
+ <trim_type>none</trim_type>
<type>String</type>
+ <variable>#{vault-prefix:hop:username}</variable>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>password</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{vault-prefix:hop:password}</variable>
</field>
</fields>
<attributes/>
<GUI>
<xloc>64</xloc>
- <yloc>64</yloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>48</yloc>
</GUI>
</transform>
<transform_error_handling>
diff --git a/integration-tests/resolver/main-0001-resolve-secrets.hwf
b/integration-tests/resolver/main-0001-resolve-secrets.hwf
index ed5f53cb6a..de9c30c47b 100644
--- a/integration-tests/resolver/main-0001-resolve-secrets.hwf
+++ b/integration-tests/resolver/main-0001-resolve-secrets.hwf
@@ -36,6 +36,7 @@ limitations under the License.
<type>SPECIAL</type>
<attributes/>
<DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
<hour>12</hour>
<intervalMinutes>60</intervalMinutes>
<intervalSeconds>0</intervalSeconds>
@@ -60,8 +61,6 @@ limitations under the License.
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/0001-vault-add-secrets.hpl</filename>
- <logext/>
- <logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@@ -85,6 +84,9 @@ limitations under the License.
<test_name>
<name>0001-vault-resolve-secrets UNIT</name>
</test_name>
+ <test_name>
+ <name>0001-vault-resolve-secrets-with-prefix UNIT</name>
+ </test_name>
</test_names>
<parallel>N</parallel>
<xloc>416</xloc>
@@ -109,6 +111,25 @@ limitations under the License.
</hop>
</hops>
<notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontsize>-1</fontsize>
+ <height>32</height>
+ <xloc>119</xloc>
+ <yloc>196</yloc>
+ <note>VAULT_ADDR='http://0.0.0.0:8200'</note>
+ <width>32</width>
+ </notepad>
</notepads>
<attributes/>
</workflow>
diff --git
a/integration-tests/resolver/metadata/unit-test/0001-vault-resolve-secrets-with-prefix
UNIT.json
b/integration-tests/resolver/metadata/unit-test/0001-vault-resolve-secrets-with-prefix
UNIT.json
new file mode 100644
index 0000000000..8a5f7271cc
--- /dev/null
+++
b/integration-tests/resolver/metadata/unit-test/0001-vault-resolve-secrets-with-prefix
UNIT.json
@@ -0,0 +1,53 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "json-data",
+ "data_set_field": "json-data"
+ },
+ {
+ "transform_field": "hostname",
+ "data_set_field": "hostname"
+ },
+ {
+ "transform_field": "port",
+ "data_set_field": "port"
+ },
+ {
+ "transform_field": "db",
+ "data_set_field": "db"
+ },
+ {
+ "transform_field": "username",
+ "data_set_field": "username"
+ },
+ {
+ "transform_field": "password",
+ "data_set_field": "password"
+ }
+ ],
+ "field_order": [
+ "json-data",
+ "hostname",
+ "port",
+ "db",
+ "username",
+ "password"
+ ],
+ "data_set_name": "0001-golden-resolved-variables",
+ "transform_name": "Output"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0001-vault-resolve-secrets-with-prefix UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0001-vault-resolve-secrets-with-prefix.hpl"
+}
\ No newline at end of file
diff --git
a/integration-tests/resolver/metadata/variable-resolver/vault-prefix.json
b/integration-tests/resolver/metadata/variable-resolver/vault-prefix.json
new file mode 100644
index 0000000000..4f4165e96f
--- /dev/null
+++ b/integration-tests/resolver/metadata/variable-resolver/vault-prefix.json
@@ -0,0 +1,17 @@
+{
+ "virtualPath": "",
+ "name": "vault-prefix",
+ "description": "Hashicorp Vault",
+ "variable-resolver": {
+ "Vault-Variable-Resolver": {
+ "vaultAddress": "http://vault:8200";,
+ "pemString": "",
+ "openTimeout": "",
+ "verifyingSsl": false,
+ "pemFilePath": "",
+ "readTimeout": "",
+ "pathPrefix": "secret/data/",
+ "vaultToken": "myroot"
+ }
+ }
+}
\ No newline at end of file
diff --git
a/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolver.java
b/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolver.java
index 45982924ff..aa232253b8 100644
---
a/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolver.java
+++
b/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolver.java
@@ -54,7 +54,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "vaultAddress",
- order = "01",
+ order = "10",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.vaultAddress",
type = GuiElementType.TEXT,
@@ -64,7 +64,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "vaultToken",
- order = "02",
+ order = "20",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.vaultToken",
type = GuiElementType.TEXT,
@@ -73,9 +73,19 @@ public class VaultVariableResolver implements
IVariableResolver {
@HopMetadataProperty
private String vaultToken;
+ @GuiWidgetElement(
+ id = "pathPrefix",
+ order = "30",
+ label =
+
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.pathPrefix",
+ type = GuiElementType.TEXT,
+ parentId = VariableResolver.GUI_PLUGIN_ELEMENT_PARENT_ID)
+ @HopMetadataProperty
+ private String pathPrefix;
+
@GuiWidgetElement(
id = "verifyingSsl",
- order = "03",
+ order = "40",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.verifyingSsl",
toolTip =
@@ -87,7 +97,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "pemFilePath",
- order = "04",
+ order = "50",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.pemFilePath",
type = GuiElementType.FILENAME,
@@ -97,7 +107,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "pemString",
- order = "05",
+ order = "60",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.pemString",
type = GuiElementType.TEXT,
@@ -108,7 +118,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "openTimeout",
- order = "06",
+ order = "70",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.openTimeout",
type = GuiElementType.TEXT,
@@ -118,7 +128,7 @@ public class VaultVariableResolver implements
IVariableResolver {
@GuiWidgetElement(
id = "readTimeout",
- order = "07",
+ order = "80",
label =
"i18n:org.apache.hop.core.variables.resolver.vault:VaultVariableResolver.label.readTimeout",
type = GuiElementType.TEXT,
@@ -173,7 +183,14 @@ public class VaultVariableResolver implements
IVariableResolver {
final Vault vault = new Vault(vaultConfig);
- LogicalResponse logicalResponse = vault.logical().read(secretPath);
+ String path;
+ if (StringUtils.isNotEmpty(pathPrefix)) {
+ path = variables.resolve(pathPrefix) + secretPath;
+ } else {
+ path = secretPath;
+ }
+
+ LogicalResponse logicalResponse = vault.logical().read(path);
if (logicalResponse == null) {
LogChannel.GENERAL.logDetailed(
"The secret with path '" + secretPath + "' was not found in the
vault");
diff --git
a/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
b/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
index 7b07c4180d..361b8e92d1 100644
---
a/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
+++
b/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
@@ -18,6 +18,7 @@
VaultVariableResolver.label.vaultAddress = Vault address
VaultVariableResolver.label.vaultToken = Vault token
+VaultVariableResolver.label.pathPrefix = Path prefix (optional)
VaultVariableResolver.label.verifyingSsl = Validate HTTPS connections?
VaultVariableResolver.tooltip.verifyingSsl = Enable this in production! It
validates connections with an X.509 certificate specified with one of the PEM
options below.
VaultVariableResolver.label.pemFilePath = PEM (X.509 certificate) file path
