atinvento100 opened a new issue, #14021:
URL: https://github.com/apache/iceberg/issues/14021
### Query engine
### Query engine
I am using spark query engine (spark 4.0.0)
### Question
### Question
I am using Pyspark 4 and iceberg-spark-runtime-4.0_2.13:1.10.0-SNAPSHOT
but when creating a spark session as below
```
spark = (SparkSession.builder.appName("app_name")
.config("spark.jars.packages",
"org.apache.hadoop:hadoop-aws:3.3.4,org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0-SNAPSHOT")
.config("spark.jars.repositories",
"https://repository.apache.org/content/repositories/snapshots";)
.config("spark.hadoop.fs.s3a.path.style.access", "True")
.config("spark.sql.catalogImplementation", "hive")
.config(
"spark.sql.extensions",
"org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
)
.config("spark.sql.iceberg.vectorization.enabled", "False")
.config(
"spark.sql.catalog.{catalog_name}".format(catalog_name=configs['CATALOG_NAME']),
"org.apache.iceberg.spark.SparkCatalog",
)
.config(
"spark.sql.catalog.{catalog_name}.type".format(
catalog_name=configs['CATALOG_NAME']
),
"hive",
)
.config(
"spark.sql.catalog.{catalog_name}.uri".format(
catalog_name=configs['CATALOG_NAME']
),
configs['CATALOG_METASTORE_URI'],
)
.config(
"spark.sql.catalog.{catalog_name}.warehouse".format(
catalog_name=configs['CATALOG_NAME']
),
"spark-warehouse/iceberg",
)
.config("spark.hadoop.hive.metastore.uris",
configs['CATALOG_METASTORE_URI'])
.config("spark.hive.metastore.client.auth.mode", "PLAIN")
.config("spark.hive.metastore.client.plain.username",
"<API_KEY>")
.config("spark.hive.metastore.client.plain.password",
configs['WXD_API_KEY'])
.config("spark.hive.metastore.use.SSL", "true")
.config("spark.hive.metastore.truststore.type", "JKS")
.config(
"spark.hive.metastore.truststore.path",
configs['CATALOG_METASTORE_TRUSTSTORE_PATH'],
)
.config(
"spark.hive.metastore.truststore.password",
configs['CATALOG_METASTORE_TRUSTSTORE_PASSWORD'],
)
.config(
"spark.hadoop.fs.s3a.bucket.{}.endpoint".format(
configs['CATALOG_COS_BUCKET_NAME']
),
configs['CATALOG_COS_BUCKET_ENDPOINT'],
)
.config(
"spark.hadoop.fs.s3a.bucket.{}.access.key".format(
configs['CATALOG_COS_BUCKET_NAME']
),
configs['CATALOG_COS_BUCKET_ACCESS_KEY'],
)
.config(
"spark.hadoop.fs.s3a.bucket.{}.secret.key".format(
configs['CATALOG_COS_BUCKET_NAME']
),
configs['CATALOG_COS_BUCKET_SECRET_KEY'],
)
.config("spark.sql.legacy.parquet.nanosAsLong", "true")
.config("spark.hadoop.fs.s3a.path.style.access", "True")
.config("spark.driver.extraClassPath", configs['DRIVER_PATH'])
.enableHiveSupport()
.getOrCreate())
```
i'm getting error as follows
```
25/09/08 12:08:48 WARN HiveConf: HiveConf of name
hive.metastore.client.plain.username does not exist
25/09/08 12:08:48 WARN HiveConf: HiveConf of name
hive.metastore.client.plain.password does not exist
25/09/08 12:08:48 WARN HiveConf: HiveConf of name
hive.metastore.client.auth.mode does not exist
25/09/08 12:08:48 WARN HiveConf: HiveConf of name
hive.metastore.truststore.type does not exist
25/09/08 12:08:50 WARN metastore: set_ugi() not successful, Likely cause:
new client talking to old server. Continuing without it.
at
org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:184)
at
org.apache.thrift.transport.TTransport.readAll(TTransport.java:109)
at
org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:464)
at
org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:362)
at
org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:245)
at
org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_set_ugi(ThriftHiveMetastore.java:4229)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.set_ugi(ThriftHiveMetastore.java:4215)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:510)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:246)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:481)
at
org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1740)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:83)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:133)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:97)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at
org.apache.iceberg.common.DynMethods$UnboundMethod.invokeChecked(DynMethods.java:60)
at
org.apache.iceberg.common.DynMethods$UnboundMethod.invoke(DynMethods.java:72)
at
org.apache.iceberg.common.DynMethods$StaticMethod.invoke(DynMethods.java:185)
at
org.apache.iceberg.hive.HiveClientPool.newClient(HiveClientPool.java:63)
at
org.apache.iceberg.hive.HiveClientPool.newClient(HiveClientPool.java:34)
at org.apache.iceberg.ClientPoolImpl.get(ClientPoolImpl.java:143)
at org.apache.iceberg.ClientPoolImpl.run(ClientPoolImpl.java:70)
at org.apache.iceberg.ClientPoolImpl.run(ClientPoolImpl.java:65)
at
org.apache.iceberg.hive.CachedClientPool.run(CachedClientPool.java:122)
at
org.apache.iceberg.hive.HiveTableOperations.doRefresh(HiveTableOperations.java:107)
at
org.apache.iceberg.BaseMetastoreTableOperations.refresh(BaseMetastoreTableOperations.java:88)
at
org.apache.iceberg.BaseMetastoreTableOperations.current(BaseMetastoreTableOperations.java:71)
at
org.apache.iceberg.BaseMetastoreCatalog.loadTable(BaseMetastoreCatalog.java:49)
at
org.apache.iceberg.shaded.com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406)
```
This is blocking our planned upgrade to pypsark v4.0.
Had already faced issues before and had created an issue for the same -
https://github.com/apache/iceberg/issues/13358. have been following up on that,
but again trying brings this error.
Please help with the same.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
