aokolnychyi commented on a change in pull request #1784:
URL: https://github.com/apache/iceberg/pull/1784#discussion_r527205882
##########
File path: spark/src/main/java/org/apache/iceberg/actions/BaseSparkAction.java
##########
@@ -128,33 +127,56 @@
return manifestDF.union(otherMetadataFileDF).union(manifestListDF);
}
+ private static boolean isExpectedCatalogLookupException(Exception exception)
{
+ return exception.getMessage().contains("AnalysisException") ||
+ exception.getMessage().contains("SparkException") ||
+ exception.getMessage().contains("NoSuchTableException") ||
+ exception.getMessage().contains("CatalogNotFoundException");
+ }
+
+ // Attempt to use Spark3 Catalog resolution if available on the path
+ private static DynMethods.StaticMethod loadCatalogImpl = null;
+
+ private static Dataset<Row> loadCatalogMetadataTable(SparkSession spark,
String tableName, MetadataTableType type)
+ throws NoSuchMethodException {
+ if (loadCatalogImpl == null) {
+ loadCatalogImpl = DynMethods.builder("loadCatalogMetadataTable")
+ .hiddenImpl("org.apache.iceberg.spark.Spark3Util",
+ SparkSession.class, String.class, MetadataTableType.class)
+ .buildStaticChecked();
+ }
+ return loadCatalogImpl.invoke(spark, tableName, type);
+ }
+
protected static Dataset<Row> loadMetadataTable(SparkSession spark, String
tableName, String tableLocation,
MetadataTableType type) {
- DataFrameReader noCatalogReader = spark.read().format("iceberg");
+ DataFrameReader dataFrameReader = spark.read().format("iceberg");
if (tableName.contains("/")) {
// Hadoop Table or Metadata location passed, load without a catalog
- return noCatalogReader.load(tableName + "#" + type);
+ return dataFrameReader.load(tableName + "#" + type);
}
- // Try catalog based name based resolution
try {
- return spark.table(tableName + "." + type);
- } catch (Exception e) {
- if (!(e instanceof ParseException || e instanceof AnalysisException)) {
- // Rethrow unexpected exceptions
- throw e;
+ // Try DSV2 catalog based name based resolution
+ if (!spark.version().startsWith("2")) {
Review comment:
Can we invert the condition and check if it starts with 3?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
