This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e547aa33c8 Verify TLS certificates by default in the Vault and OpenBao
resolvers, fixes #8392 (#8396)
e547aa33c8 is described below
commit e547aa33c80a329af8d56317177707df995096d4
Author: Bart Maertens <[email protected]>
AuthorDate: Wed Sep 16 10:40:37 2026 +0200
Verify TLS certificates by default in the Vault and OpenBao resolvers,
fixes #8392 (#8396)
verifyingSsl had no initialiser, so a resolver verified the secrets server's
certificate only when someone ticked the option. Both resolvers extend the
same
base class, so neither did.
The initialiser alone is not enough: both serializers overwrite a boolean
with
HopMetadataProperty.defaultBoolean() before reading the stored value, so the
annotation carries the default and the initialiser covers resolvers built in
code.
The property is serialised per resolver and booleans are always written, so
this
only reaches resolvers created from now on. Definitions saved with the
option off
keep reaching the secrets server unverified, which the docs now tell
operators to
go and check. A resolver that does so on an https address logs a warning,
once
per address; plain http has no certificate to verify and stays quiet.
The vault integration-test project gained a second Vault on HTTPS with a
self-signed certificate and three resolvers against it: one given the CA,
one
without it, and one that stores verifyingSsl=false. Only the middle one
fails to
resolve, and the third proves existing definitions keep working.
---
THREAT_MODEL.md | 5 +-
.../integration-tests/integration-tests-vault.yaml | 37 +++++
.../hashicorp-vault-variable-resolver.adoc | 13 +-
.../openbao-variable-resolver.adoc | 13 +-
.../vault/0005-vault-tls-add-secrets.hpl | 178 +++++++++++++++++++++
.../vault/0005-vault-tls-verification.hpl | 138 ++++++++++++++++
.../datasets/0005-golden-tls-verification.csv | 2 +
.../vault/main-0005-tls-verification.hwf | 132 +++++++++++++++
.../dataset/0005-golden-tls-verification.json | 32 ++++
.../0005-vault-tls-verification UNIT.json | 38 +++++
.../variable-resolver/vault-tls-untrusted.json | 15 ++
.../variable-resolver/vault-tls-unverified.json | 16 ++
.../metadata/variable-resolver/vault-tls.json | 15 ++
.../resolver/vault/BaseVaultVariableResolver.java | 38 ++++-
.../vault/messages/messages_en_US.properties | 2 +-
.../resolver/vault/VaultVariableResolverTest.java | 35 ++++
16 files changed, 703 insertions(+), 6 deletions(-)
diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md
index 5ae04a397a..0a97593869 100644
--- a/THREAT_MODEL.md
+++ b/THREAT_MODEL.md
@@ -234,7 +234,10 @@ where strict outbound TLS verification is required (see
§9).
also *constructs* a self-signed-accepting, no-hostname-verify socket factory
in
`sslMode`, but it is not actually wired into the returned client — latent.)
Several transforms also offer opt-in `ignoreSsl`/`trustAllCertificates`
- switches (all default to verification on).
+ switches (all default to verification on). The HashiCorp Vault and OpenBAO
+ variable resolvers verify the secrets server's certificate by default too,
but
+ the setting is serialised per resolver, so a definition saved with it off
keeps
+ reaching the secrets server unverified until an operator changes it.
- **No automatic sanitization of variables.** `${VAR}`/`%%VAR%%`/`$[hex]`
tokens
are resolved by raw, unescaped string substitution
([`StringUtil.environmentSubstitute`](core/src/main/java/org/apache/hop/core/util/StringUtil.java));
the result is concatenated directly into
diff --git a/docker/integration-tests/integration-tests-vault.yaml
b/docker/integration-tests/integration-tests-vault.yaml
index acd8f5821f..11ce5e7853 100644
--- a/docker/integration-tests/integration-tests-vault.yaml
+++ b/docker/integration-tests/integration-tests-vault.yaml
@@ -17,6 +17,7 @@
volumes:
k8s-jwt:
+ vault-tls-certs:
services:
integration_test_vault:
@@ -26,12 +27,16 @@ services:
depends_on:
vault:
condition: service_healthy
+ vault-tls:
+ condition: service_healthy
k8s-mock:
condition: service_healthy
links:
- vault
+ - vault-tls
volumes:
- k8s-jwt:/tmp/k8s-sa:ro
+ - vault-tls-certs:/tmp/vault-tls:ro
vault:
image: hashicorp/vault
ports:
@@ -47,6 +52,38 @@ services:
timeout: 3s
retries: 10
start_period: 5s
+ # A second Vault, this one on HTTPS with a certificate it generates for
itself, so the
+ # integration tests can tell a resolver that verifies the certificate from
one that does not.
+ # -dev-tls writes vault-ca.pem / vault-cert.pem / vault-key.pem into the
cert dir, and
+ # -dev-tls-san puts this service name in the certificate so hostname
verification passes.
+ #
+ # The cert dir is /vault/file (Vault's file-storage path, unused in dev
mode) because that
+ # directory exists in the image owned by the unprivileged vault user: a
named volume mounted
+ # over it inherits that ownership, which a volume mounted on a fresh path
would not, and Vault
+ # cannot write its certificates into a root-owned directory.
+ vault-tls:
+ image: hashicorp/vault
+ ports:
+ - "8200"
+ environment:
+ - VAULT_DEV_ROOT_TOKEN_ID=myroot
+ - VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200
+ - SKIP_SETCAP=true
+ - VAULT_DISABLE_MLOCK=true
+ command:
+ - server
+ - -dev
+ - -dev-tls
+ - -dev-tls-cert-dir=/vault/file
+ - -dev-tls-san=vault-tls
+ volumes:
+ - vault-tls-certs:/vault/file
+ healthcheck:
+ test: [ "CMD", "wget", "--spider", "--proxy", "off",
"--no-check-certificate", "https://vault-tls:8200/v1/sys/health?standbyok=true";
]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 5s
k8s-mock:
image: python:3.12-alpine
depends_on:
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
index 42fd0b1ab3..ae1259c3ef 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/hashicorp-vault-variable-resolver.adoc
@@ -29,12 +29,23 @@
image::metadata-types/variable-resolver/vault-variable-resolver-connection-tab.p
* Vault address: The base address and port of the Vault server (for example:
https://vault-server:8200)
* Namespace: The Vault namespace to use (optional, primarily used in
enterprise/multi-tenant setups)
-* Validate HTTPS connections?: It's recommended to enable connection
validation in production. This secures the connection with the X.509
certificate specified in one of either next option.
+* Validate HTTPS connections?: Verifies the certificate the Vault server
presents. On for new resolvers. Leave it on in production, and identify the
certificate with one of the two PEM options below when the server does not use
a certificate from a public CA.
* PEM file path: The name of the file (VFS) containing the X.509 certificate
string
* PEM string: The X.509 string itself in case you're not using a file
* Open connection timeout: The connection timeout when getting a http(s)
connection in milliseconds.
* Read timeout: The timeout in milliseconds when reading.
+[IMPORTANT]
+====
+**Validate HTTPS connections?** became the default in Hop 2.20.
+The option is stored in the resolver, so resolvers created before that keep
the setting they have: verification stays off in every resolver definition that
was saved with the box unticked, whatever Hop version you upgrade to.
+Open your Vault and OpenBAO resolvers and check the option rather than
assuming the upgrade covered them.
+Hop logs a warning when a resolver reaches an `https` address without
verifying the certificate.
+
+A resolver that verifies the certificate needs to be able to build the chain
to a trusted root.
+Against a Vault with a self-signed certificate or one signed by a private CA,
give the resolver that certificate through **PEM file path** or **PEM string**;
without it the lookup fails and the expression is left unresolved.
+====
+
=== Authentication
How Hop proves its identity to Vault. `TOKEN` is the default and keeps
existing resolvers working. `KUBERNETES` is intended for Hop Server (or Hop
Gui) running in a Kubernetes pod.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/openbao-variable-resolver.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/openbao-variable-resolver.adoc
index 1da08c847c..a018874a09 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/openbao-variable-resolver.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/openbao-variable-resolver.adoc
@@ -46,12 +46,23 @@ will resolve internally to:
----
-* Validate HTTPS connections?: It's recommended to enable connection
validation in production. This secures the connection with the X.509
certificate specified in one of either next option.
+* Validate HTTPS connections?: Verifies the certificate the OpenBAO server
presents. On for new resolvers. Leave it on in production, and identify the
certificate with one of the two PEM options below when the server does not use
a certificate from a public CA.
* PEM file path: The name of the file (VFS) containing the X.509 certificate
string
* PEM string: The X.509 string itself in case you're not using a file
* Open connection timeout: The connection timeout when getting a http(s)
connection in milliseconds.
* Read connection timeout: The timeout in milliseconds when reading.
+[IMPORTANT]
+====
+**Validate HTTPS connections?** became the default in Hop 2.20.
+The option is stored in the resolver, so resolvers created before that keep
the setting they have: verification stays off in every resolver definition that
was saved with the box unticked, whatever Hop version you upgrade to.
+Open your OpenBAO and Vault resolvers and check the option rather than
assuming the upgrade covered them.
+Hop logs a warning when a resolver reaches an `https` address without
verifying the certificate.
+
+A resolver that verifies the certificate needs to be able to build the chain
to a trusted root.
+Against an OpenBAO with a self-signed certificate or one signed by a private
CA, give the resolver that certificate through **PEM file path** or **PEM
string**; without it the lookup fails and the expression is left unresolved.
+====
+
The variable expression you can resolve with this plugin type is (as always)
in the following format:
`+#{name:path-key:value-key}+`
diff --git a/integration-tests/vault/0005-vault-tls-add-secrets.hpl
b/integration-tests/vault/0005-vault-tls-add-secrets.hpl
new file mode 100644
index 0000000000..a8fc2e9cc9
--- /dev/null
+++ b/integration-tests/vault/0005-vault-tls-add-secrets.hpl
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0005-vault-tls-add-secrets</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/01/14 15:47:02.970</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/01/14 15:47:02.970</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Secrets</from>
+ <to>REST client</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>REST client</name>
+ <type>Rest</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <applicationType>TEXT PLAIN</applicationType>
+ <bodyField>data</bodyField>
+ <connectionTimeout>10000</connectionTimeout>
+ <dynamicMethod>N</dynamicMethod>
+ <headers>
+ <header>
+ <field>contentType</field>
+ <name>Content-Type</name>
+ </header>
+ <header>
+ <field>root</field>
+ <name>X-Vault-Token</name>
+ </header>
+ </headers>
+ <ignoreSsl>Y</ignoreSsl>
+ <matrixParameters>
+</matrixParameters>
+ <method>POST</method>
+ <parameters>
+</parameters>
+ <readTimeout>10000</readTimeout>
+ <result>
+ <name>result</name>
+ </result>
+ <retryMethods>
+</retryMethods>
+ <retryStatusCodes>
+</retryStatusCodes>
+ <trustStorePassword>Encrypted </trustStorePassword>
+ <urlField>url</urlField>
+ <urlInField>Y</urlInField>
+ <attributes/>
+ <GUI>
+ <xloc>208</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Secrets</name>
+ <type>DataGrid</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <data>
+ <line>
+ <item>myroot</item>
+ <item>https://vault-tls:8200/v1/secret/data/hop</item>
+ <item>application/json</item>
+ <item>{
+ "data": {
+ "db": "test",
+ "hostname": "localhost",
+ "password": "some-password",
+ "port": "3306",
+ "username": "john"
+ },
+ "options": {}
+}</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>root</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>url</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>contentType</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <set_empty_string>N</set_empty_string>
+ <name>data</name>
+ <format/>
+ <group/>
+ <decimal/>
+ <type>String</type>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>64</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/vault/0005-vault-tls-verification.hpl
b/integration-tests/vault/0005-vault-tls-verification.hpl
new file mode 100644
index 0000000000..8bb2ecb51d
--- /dev/null
+++ b/integration-tests/vault/0005-vault-tls-verification.hpl
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0005-vault-tls-verification</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2025/01/14 15:54:11.134</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/01/14 15:54:11.134</modified_date>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontsize>-1</fontsize>
+ <height>160</height>
+ <xloc>32</xloc>
+ <yloc>144</yloc>
+ <note>The three resolvers point at the same HTTPS Vault.
+
+vault-tls and vault-tls-untrusted store no verifyingSsl flag, so they
+fall back to the default. Verification is on by default, so only the
+one that is given the CA of the server certificate resolves the
+secret; the other fails and its expression is left unresolved.
+
+vault-tls-unverified stores verifyingSsl=false, the way a resolver
+created before verification became the default does, and keeps
+resolving.</note>
+ <width>560</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>Get variables</from>
+ <to>Output</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Get variables</name>
+ <type>GetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <length>-1</length>
+ <name>trusted-ca</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{vault-tls:secret/data/hop:password}</variable>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>untrusted-ca</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{vault-tls-untrusted:secret/data/hop:password}</variable>
+ </field>
+ <field>
+ <length>-1</length>
+ <name>verification-off</name>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <variable>#{vault-tls-unverified:secret/data/hop:password}</variable>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>64</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Output</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>48</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/vault/datasets/0005-golden-tls-verification.csv
b/integration-tests/vault/datasets/0005-golden-tls-verification.csv
new file mode 100644
index 0000000000..3175009123
--- /dev/null
+++ b/integration-tests/vault/datasets/0005-golden-tls-verification.csv
@@ -0,0 +1,2 @@
+trusted-ca,untrusted-ca,verification-off
+some-password,#{vault-tls-untrusted:secret/data/hop:password},some-password
diff --git a/integration-tests/vault/main-0005-tls-verification.hwf
b/integration-tests/vault/main-0005-tls-verification.hwf
new file mode 100644
index 0000000000..49c4acd83a
--- /dev/null
+++ b/integration-tests/vault/main-0005-tls-verification.hwf
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0005-tls-verification</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2025/01/14 15:46:57.125</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2025/01/14 15:46:57.125</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0005-vault-tls-add-secrets.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0005-vault-tls-add-secrets.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0005-vault-tls-verification UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>416</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0005-vault-tls-add-secrets.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0005-vault-tls-add-secrets.hpl</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontsize>-1</fontsize>
+ <height>32</height>
+ <xloc>119</xloc>
+ <yloc>196</yloc>
+ <note>VAULT_ADDR='https://vault-tls:8200'</note>
+ <width>32</width>
+ </notepad>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/vault/metadata/dataset/0005-golden-tls-verification.json
b/integration-tests/vault/metadata/dataset/0005-golden-tls-verification.json
new file mode 100644
index 0000000000..463ba3bc27
--- /dev/null
+++ b/integration-tests/vault/metadata/dataset/0005-golden-tls-verification.json
@@ -0,0 +1,32 @@
+{
+ "base_filename": "0005-golden-tls-verification.csv",
+ "name": "0005-golden-tls-verification",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "trusted-ca",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "untrusted-ca",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "verification-off",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git
a/integration-tests/vault/metadata/unit-test/0005-vault-tls-verification
UNIT.json
b/integration-tests/vault/metadata/unit-test/0005-vault-tls-verification
UNIT.json
new file mode 100644
index 0000000000..b4349bda9b
--- /dev/null
+++ b/integration-tests/vault/metadata/unit-test/0005-vault-tls-verification
UNIT.json
@@ -0,0 +1,38 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "trusted-ca",
+ "data_set_field": "trusted-ca"
+ },
+ {
+ "transform_field": "untrusted-ca",
+ "data_set_field": "untrusted-ca"
+ },
+ {
+ "transform_field": "verification-off",
+ "data_set_field": "verification-off"
+ }
+ ],
+ "field_order": [
+ "trusted-ca",
+ "untrusted-ca",
+ "verification-off"
+ ],
+ "data_set_name": "0005-golden-tls-verification",
+ "transform_name": "Output"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0005-vault-tls-verification UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0005-vault-tls-verification.hpl"
+}
diff --git
a/integration-tests/vault/metadata/variable-resolver/vault-tls-untrusted.json
b/integration-tests/vault/metadata/variable-resolver/vault-tls-untrusted.json
new file mode 100644
index 0000000000..53b74d9b9d
--- /dev/null
+++
b/integration-tests/vault/metadata/variable-resolver/vault-tls-untrusted.json
@@ -0,0 +1,15 @@
+{
+ "virtualPath": "",
+ "name": "vault-tls-untrusted",
+ "description": "Hashicorp Vault over HTTPS without the CA of the server
certificate",
+ "variable-resolver": {
+ "Vault-Variable-Resolver": {
+ "vaultAddress": "https://vault-tls:8200";,
+ "pemString": "",
+ "openTimeout": "",
+ "pemFilePath": "",
+ "readTimeout": "",
+ "vaultToken": "myroot"
+ }
+ }
+}
diff --git
a/integration-tests/vault/metadata/variable-resolver/vault-tls-unverified.json
b/integration-tests/vault/metadata/variable-resolver/vault-tls-unverified.json
new file mode 100644
index 0000000000..50e61c17d9
--- /dev/null
+++
b/integration-tests/vault/metadata/variable-resolver/vault-tls-unverified.json
@@ -0,0 +1,16 @@
+{
+ "virtualPath": "",
+ "name": "vault-tls-unverified",
+ "description": "Hashicorp Vault over HTTPS with certificate verification
explicitly turned off",
+ "variable-resolver": {
+ "Vault-Variable-Resolver": {
+ "vaultAddress": "https://vault-tls:8200";,
+ "pemString": "",
+ "openTimeout": "",
+ "verifyingSsl": false,
+ "pemFilePath": "",
+ "readTimeout": "",
+ "vaultToken": "myroot"
+ }
+ }
+}
diff --git a/integration-tests/vault/metadata/variable-resolver/vault-tls.json
b/integration-tests/vault/metadata/variable-resolver/vault-tls.json
new file mode 100644
index 0000000000..d70c8c3558
--- /dev/null
+++ b/integration-tests/vault/metadata/variable-resolver/vault-tls.json
@@ -0,0 +1,15 @@
+{
+ "virtualPath": "",
+ "name": "vault-tls",
+ "description": "Hashicorp Vault over HTTPS, trusting the server certificate
through its CA",
+ "variable-resolver": {
+ "Vault-Variable-Resolver": {
+ "vaultAddress": "https://vault-tls:8200";,
+ "pemString": "",
+ "openTimeout": "",
+ "pemFilePath": "/tmp/vault-tls/vault-ca.pem",
+ "readTimeout": "",
+ "vaultToken": "myroot"
+ }
+ }
+}
diff --git
a/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/BaseVaultVariableResolver.java
b/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/BaseVaultVariableResolver.java
index fe9ac7e52a..3425e5e0ea 100644
---
a/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/BaseVaultVariableResolver.java
+++
b/plugins/tech/vault/src/main/java/org/apache/hop/core/variables/resolver/vault/BaseVaultVariableResolver.java
@@ -34,6 +34,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@@ -91,6 +92,13 @@ public abstract class BaseVaultVariableResolver
private final Object clientLock = new Object();
+ /**
+ * Addresses already warned about, so a pipeline that resolves a dozen
expressions does not log a
+ * dozen identical lines. A resolver is loaded from the metadata again for
every expression, so
+ * this cannot be kept per instance.
+ */
+ private static final Set<String> UNVERIFIED_ADDRESSES_WARNED =
ConcurrentHashMap.newKeySet();
+
private transient Vault vaultClient;
private transient String clientSignature;
private transient long tokenExpiryMillis;
@@ -130,8 +138,11 @@ public abstract class BaseVaultVariableResolver
groupType = GuiWidgetGroupType.BOXES,
group = GROUP_CONNECTION,
groupOrder = "010")
- @HopMetadataProperty
- protected boolean verifyingSsl;
+ // A resolver that does not carry the property at all - a newly created one
- verifies the
+ // secrets server's certificate. defaultBoolean is what the serializers
apply for an absent
+ // value; the field initialiser covers the resolvers built in code.
+ @HopMetadataProperty(defaultBoolean = true)
+ protected boolean verifyingSsl = true;
@GuiWidgetElement(
id = ID_PEM_FILE_PATH,
@@ -496,6 +507,7 @@ public abstract class BaseVaultVariableResolver
final SslConfig sslConfig = new SslConfig();
sslConfig.verify(isVerifyingSsl());
+ warnWhenNotVerifying(actualVaultAddress);
String pemUtf8 = null;
if (StringUtils.isNotEmpty(pemFilePath)) {
try (InputStream is =
HopVfs.getInputStream(variables.resolve(pemFilePath), variables)) {
@@ -530,6 +542,28 @@ public abstract class BaseVaultVariableResolver
}
}
+ /**
+ * Turning verification off is silent otherwise, which is easy to miss in a
resolver definition
+ * that was created before verification became the default. A plain http
address has no
+ * certificate to verify, so the option changes nothing there and saying so
would only be noise.
+ */
+ private void warnWhenNotVerifying(String actualVaultAddress) {
+ String address = Const.NVL(actualVaultAddress, "").trim();
+ if (isVerifyingSsl()
+ || address.toLowerCase(Locale.ROOT).startsWith("http://";)
+ || !UNVERIFIED_ADDRESSES_WARNED.add(getPluginId() + "\t" + address)) {
+ return;
+ }
+ LogChannel.GENERAL.logBasic(
+ "WARNING: the "
+ + getPluginName()
+ + " reaches '"
+ + address
+ + "' without verifying the certificate of the secrets server.
Switch 'Validate HTTPS "
+ + "connections?' back on and point the resolver at the server's
certificate with the "
+ + "PEM file path or PEM string option.");
+ }
+
private String clientSignature(IVariables variables) {
return String.join(
"\t",
diff --git
a/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
b/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
index 3b255675e2..573390e27b 100644
---
a/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
+++
b/plugins/tech/vault/src/main/resources/org/apache/hop/core/variables/resolver/vault/messages/messages_en_US.properties
@@ -21,7 +21,7 @@ VaultVariableResolver.label.vaultToken = Vault token
VaultVariableResolver.label.pathPrefix = Path prefix (optional)
VaultVariableResolver.label.namespace = Namespace
VaultVariableResolver.label.verifyingSsl = Validate HTTPS connections?
-VaultVariableResolver.tooltip.verifyingSsl = Enable this in production! It
validates connections with an X.509 certificate specified with one of the PEM
options below.
+VaultVariableResolver.tooltip.verifyingSsl = On by default. It verifies the
certificate of the secrets server, which needs one of the PEM options below
when that certificate is self-signed or signed by a private CA.
VaultVariableResolver.label.pemFilePath = PEM (X.509 certificate) file path
VaultVariableResolver.label.pemString = PEM (X.509 certificate) String
VaultVariableResolver.label.openTimeout = Open connection timeout
diff --git
a/plugins/tech/vault/src/test/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolverTest.java
b/plugins/tech/vault/src/test/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolverTest.java
index 8348e83d83..a6ea614138 100644
---
a/plugins/tech/vault/src/test/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolverTest.java
+++
b/plugins/tech/vault/src/test/java/org/apache/hop/core/variables/resolver/vault/VaultVariableResolverTest.java
@@ -18,6 +18,7 @@
package org.apache.hop.core.variables.resolver.vault;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -41,6 +42,9 @@ import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -90,6 +94,37 @@ class VaultVariableResolverTest {
assertEquals("Hashicorp Vault Variable Resolver",
resolver.getPluginName());
}
+ @Test
+ void testNewResolversVerifyTheServerCertificate() {
+ assertTrue(new VaultVariableResolver().isVerifyingSsl());
+ assertTrue(new OpenBaoVariableResolver().isVerifyingSsl());
+ }
+
+ @Test
+ void testMetadataWithoutTheFlagVerifiesTheServerCertificate() throws
Exception {
+ assertTrue(
+
loadFromXml("<resolver><vaultAddress>https://vault:8200</vaultAddress></resolver>")
+ .isVerifyingSsl());
+ }
+
+ @Test
+ void testMetadataKeepsVerificationTurnedOff() throws Exception {
+ // Resolvers written before verification became the default carry the flag
explicitly, and
+ // keep working the way they did.
+ assertFalse(
+ loadFromXml(
+ "<resolver><vaultAddress>https://vault:8200</vaultAddress>"
+ + "<verifyingSsl>N</verifyingSsl></resolver>")
+ .isVerifyingSsl());
+ }
+
+ private static VaultVariableResolver loadFromXml(String xml) throws
Exception {
+ return XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.getSubNode(XmlHandler.loadXmlString(xml), "resolver"),
+ VaultVariableResolver.class,
+ new MemoryMetadataProvider());
+ }
+
@Test
void testDefaultAuthenticationTypeIsToken() {
assertEquals(VaultAuthType.TOKEN.name(), new
VaultVariableResolver().getAuthenticationType());
