pvary commented on a change in pull request #2129:
URL: https://github.com/apache/iceberg/pull/2129#discussion_r603069650
##########
File path: mr/src/main/java/org/apache/iceberg/mr/Catalogs.java
##########
@@ -182,48 +188,100 @@ public static boolean dropTable(Configuration conf,
Properties props) {
/**
* Returns true if HiveCatalog is used
* @param conf a Hadoop conf
+ * @param props the controlling properties
* @return true if the Catalog is HiveCatalog
*/
- public static boolean hiveCatalog(Configuration conf) {
- return HIVE.equalsIgnoreCase(conf.get(InputFormatConfig.CATALOG));
+ public static boolean hiveCatalog(Configuration conf, Properties props) {
+ String catalogName = props.getProperty(InputFormatConfig.CATALOG_NAME);
+ return
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE.equalsIgnoreCase(getCatalogType(conf,
catalogName));
}
@VisibleForTesting
- static Optional<Catalog> loadCatalog(Configuration conf) {
- String catalogLoaderClass =
conf.get(InputFormatConfig.CATALOG_LOADER_CLASS);
-
- if (catalogLoaderClass != null) {
- CatalogLoader loader = (CatalogLoader)
DynConstructors.builder(CatalogLoader.class)
- .impl(catalogLoaderClass)
- .build()
- .newInstance();
- Catalog catalog = loader.load(conf);
- LOG.info("Loaded catalog {} using {}", catalog, catalogLoaderClass);
- return Optional.of(catalog);
+ static Optional<Catalog> loadCatalog(Configuration conf, String catalogName)
{
+ String catalogType = getCatalogType(conf, catalogName);
+ if (catalogType == null) {
+ throw new NoSuchNamespaceException("Catalog definition for %s is not
found.", catalogName);
}
+ String name = catalogName == null ? ICEBERG_DEFAULT_CATALOG_NAME :
catalogName;
- String catalogName = conf.get(InputFormatConfig.CATALOG);
-
- if (catalogName != null) {
- Catalog catalog;
- switch (catalogName.toLowerCase()) {
- case HADOOP:
- String warehouseLocation =
conf.get(InputFormatConfig.HADOOP_CATALOG_WAREHOUSE_LOCATION);
-
- catalog = (warehouseLocation != null) ? new HadoopCatalog(conf,
warehouseLocation) : new HadoopCatalog(conf);
- LOG.info("Loaded Hadoop catalog {}", catalog);
- return Optional.of(catalog);
- case HIVE:
- catalog = CatalogUtil.loadCatalog(HiveCatalog.class.getName(),
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
- ImmutableMap.of(), conf);
- LOG.info("Loaded Hive Metastore catalog {}", catalog);
- return Optional.of(catalog);
- default:
- throw new NoSuchNamespaceException("Catalog %s is not supported.",
catalogName);
- }
+ switch (catalogType.toLowerCase()) {
+ case NO_CATALOG_TYPE:
+ return Optional.empty();
+ default:
+ return Optional.of(CatalogUtil.buildIcebergCatalog(name,
+ getCatalogProperties(conf, name, catalogType), conf));
}
+ }
- LOG.info("Catalog is not configured");
- return Optional.empty();
+ /**
+ * Collect all the catalog specific configuration from the global hive
configuration.
+ * @param conf global hive configuration
+ * @param catalogName name of the catalog
+ * @param catalogType type of the catalog
+ * @return complete map of catalog properties
+ */
+ private static Map<String, String> getCatalogProperties(Configuration conf,
String catalogName, String catalogType) {
+ String keyPrefix = InputFormatConfig.CATALOG_CONFIG_PREFIX + catalogName;
+ Map<String, String> catalogProperties = Streams.stream(conf.iterator())
+ .filter(e -> e.getKey().startsWith(keyPrefix))
+ .collect(Collectors.toMap(e ->
e.getKey().substring(keyPrefix.length() + 1), Map.Entry::getValue));
+ return addCatalogPropertiesIfMissing(conf, catalogType, catalogProperties);
+ }
+
+ /**
+ * This method is used for backward-compatible catalog configuration.
+ * Collect all the catalog specific configuration from the global hive
configuration.
Review comment:
nit: maybe a comment that this should be removed when the old
configuration is deprecated?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
