deniskuzZ commented on code in PR #6474:
URL: https://github.com/apache/hive/pull/6474#discussion_r3578697912
##########
iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q:
##########
@@ -24,9 +28,13 @@ set hive.stats.autogather=false;
set metastore.client.impl=org.apache.iceberg.hive.client.HiveRESTCatalogClient;
set metastore.catalog.default=ice01;
set iceberg.catalog.ice01.type=rest;
+set
iceberg.catalog.ice01.header.X-Iceberg-Access-Delegation=vended-credentials;
+--! Vended credentials travel via DAG credentials (HIVE-20651); an in-process
fetch task cannot
+--! restore them, so fetch conversion must be off for credential-vending
catalogs.
+set hive.fetch.task.conversion=none;
Review Comment:
````
Subject: [PATCH] new patch
---
Index:
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
(revision 4c3214ed63152c640e1febbd46873016775a3bdc)
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
(date 1784029753143)
@@ -26,6 +26,7 @@
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.mapred.JobConf;
import org.apache.iceberg.BaseMetadataTable;
@@ -46,7 +47,6 @@
import org.apache.iceberg.io.SupportsStorageCredentials;
import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.util.SerializationUtil;
@@ -59,10 +59,6 @@
*/
public final class IcebergVendedCredentialUtil {
- /** Vended config safe to display; any key not listed here routes to the
secrets channel. */
- private static final ImmutableSet<String> NON_SECRET_ICEBERG_KEYS =
ImmutableSet.of(
- S3FileIOProperties.ENDPOINT, S3FileIOProperties.PATH_STYLE_ACCESS,
AwsClientProperties.CLIENT_REGION);
-
private IcebergVendedCredentialUtil() {
}
@@ -135,11 +131,11 @@
}
String resolvedValue = resolveCredentialValue(catalogName, icebergKey,
value, conf);
- if (jobProperties != null && !isSecretKey(icebergKey)) {
+ if (jobProperties != null && !isSecretKey(icebergKey, conf)) {
addNonSecretCredentialEntry(catalogName, bucket, icebergKey,
resolvedValue, jobProperties);
}
- if (jobSecrets != null && isSecretKey(icebergKey)) {
+ if (jobSecrets != null && isSecretKey(icebergKey, conf)) {
addSecretCredentialEntry(bucket, icebergKey, resolvedValue,
jobSecrets);
}
}
@@ -320,13 +316,11 @@
FileIO io = table.io();
Map<String, String> ioProps = new LinkedHashMap<>();
- if (io.properties() != null) {
- io.properties().forEach((key, value) -> {
- if (NON_SECRET_ICEBERG_KEYS.contains(key)) {
- ioProps.put(key, value);
- }
- });
- }
+ io.properties().forEach((k, v) -> {
+ if (!isSecretKey(k, conf)) {
+ ioProps.put(k, v);
+ }
+ });
FileIO cleanIo = CatalogUtil.loadFileIO(io.getClass().getName(),
ioProps, conf);
return new BaseTable(
@@ -370,8 +364,14 @@
}
}
- private static boolean isSecretKey(String icebergKey) {
- return !NON_SECRET_ICEBERG_KEYS.contains(icebergKey);
+ /**
+ * A vended config key is secret — routed to the Credentials channel,
never job properties, and
+ * stripped from the serialized table's FileIO — exactly when {@code
hive.conf.hidden.list} covers
+ * it. That is Hive's own registry of sensitive configuration (the same
one behind
+ * {@code HiveConf#isHiddenConfig}) that masks values in EXPLAIN, the Tez
UI, and ATS.
+ */
+ private static boolean isSecretKey(String key, Configuration conf) {
+ return
HiveConfUtil.getHiddenSet(conf).stream().anyMatch(key::startsWith);
}
static List<StorageCredential> extractCredentials(Table table) {
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
(revision 4c3214ed63152c640e1febbd46873016775a3bdc)
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (date
1784029871481)
@@ -375,6 +375,13 @@
Class<? extends InputFormat> formatter =
currDesc.getInputFileFormatClass();
Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job);
+ // A fetch task runs in-process without a DAG, so storage-handler
secrets (HIVE-20651) are
+ // never restored from the job Credentials as on a task. Surface them
into this process-local
+ // conf (never serialized or shown in EXPLAIN) so in-process readers
can access storage.
+ Map<String, String> jobSecrets =
currDesc.getTableDesc().getJobSecrets();
+ if (jobSecrets != null) {
+ jobSecrets.forEach(job::set);
+ }
InputFormat inputFormat = getInputFormatFromCache(formatter, job);
if(inputFormat instanceof HiveSequenceFileInputFormat) {
// input format could be cached, in which case we need to reset the
list of files to fetch
Index:
iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q
(revision 4c3214ed63152c640e1febbd46873016775a3bdc)
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q
(date 1784016928801)
@@ -29,9 +29,6 @@
set metastore.catalog.default=ice01;
set iceberg.catalog.ice01.type=rest;
set
iceberg.catalog.ice01.header.X-Iceberg-Access-Delegation=vended-credentials;
---! Vended credentials travel via DAG credentials (HIVE-20651); an
in-process fetch task cannot
---! restore them, so fetch conversion must be off for credential-vending
catalogs.
-set hive.fetch.task.conversion=none;
--! REST URI, OAuth, MinIO + Gravitino S3 warehouse / credential vending,
and host S3A are set in
--! TestIcebergRESTCatalogGravitinoLlapLocalCliDriver.
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(revision 4c3214ed63152c640e1febbd46873016775a3bdc)
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (date
1784021299978)
@@ -5601,6 +5601,10 @@
+ ",fs.s3a.secret.key"
+ ",fs.s3a.proxy.password"
+ ",iceberg.vended.storage.credentials"
+ // Iceberg FileIO vended credential keys (S3FileIOProperties)
+ + ",s3.access-key-id"
+ + ",s3.secret-access-key"
+ + ",s3.session-token"
+ ",dfs.adls.oauth2.credential"
+ ",fs.adl.oauth2.credential"
+ ",fs.azure.account.oauth2.client.secret"
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
