sunxiaojian commented on code in PR #5868:
URL: https://github.com/apache/gravitino/pull/5868#discussion_r1889715280
##########
flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java:
##########
@@ -96,43 +105,56 @@ public boolean contains(String catalogName) throws
CatalogException {
return gravitinoCatalogManager.contains(catalogName);
}
- private String getGravitinoCatalogProvider(Configuration configuration) {
+ private BaseCatalogFactory catalogFactory(Map<String, String> configuration)
{
String catalogType =
Preconditions.checkNotNull(
- configuration.get(CommonCatalogOptions.CATALOG_TYPE),
+ configuration.get(CommonCatalogOptions.CATALOG_TYPE.key()),
"%s should not be null.",
CommonCatalogOptions.CATALOG_TYPE);
- switch (catalogType) {
- case GravitinoHiveCatalogFactoryOptions.IDENTIFIER:
- return "hive";
- default:
- throw new IllegalArgumentException(
- String.format("The catalog type is not supported:%s",
catalogType));
- }
+ return discoverFactories(
+ catalogFactory ->
(catalogFactory.factoryIdentifier().equalsIgnoreCase(catalogType)),
+ String.format(
+ "Flink catalog type [%s] matched multiple flink catalog factories,
it should only match one.",
+ catalogType));
}
- private Catalog.Type getGravitinoCatalogType(Configuration configuration) {
- String catalogType =
- Preconditions.checkNotNull(
- configuration.get(CommonCatalogOptions.CATALOG_TYPE),
- "%s should not be null.",
- CommonCatalogOptions.CATALOG_TYPE);
-
- switch (catalogType) {
- case GravitinoHiveCatalogFactoryOptions.IDENTIFIER:
- return Catalog.Type.RELATIONAL;
- default:
- throw new IllegalArgumentException(
- String.format("The catalog type is not supported:%s",
catalogType));
- }
+ private BaseCatalogFactory catalogFactory(String provider) {
+ return discoverFactories(
+ catalogFactory ->
+ ((BaseCatalogFactory) catalogFactory)
+ .gravitinoCatalogProvider()
+ .equalsIgnoreCase(provider),
+ String.format(
+ "Gravitino catalog provider [%s] matched multiple flink catalog
factories, it should only match one.",
+ provider));
}
- private PropertiesConverter getPropertiesConverter(String provider) {
- switch (provider) {
- case "hive":
- return HivePropertiesConverter.INSTANCE;
+ private BaseCatalogFactory discoverFactories(Predicate<Factory> predicate,
String errorMessage) {
+ Iterator<Factory> serviceLoaderIterator =
ServiceLoader.load(Factory.class).iterator();
+ final List<Factory> factories = new ArrayList<>();
+ while (true) {
+ try {
+ if (!serviceLoaderIterator.hasNext()) {
+ break;
+ }
+ Factory catalogFactory = serviceLoaderIterator.next();
+ if (catalogFactory instanceof BaseCatalogFactory &&
predicate.test(catalogFactory)) {
+ factories.add(catalogFactory);
+ }
+ } catch (Throwable t) {
+ if (t instanceof NoClassDefFoundError) {
+ LOG.debug(
Review Comment:
> There is a real case:
>
> FlinkHiveCatalogIT > testHivePartitionTable() FAILED
java.lang.NoClassDefFoundError:
org/apache/flink/table/gateway/api/endpoint/SqlGatewayEndpointFactory at
java.base/java.lang.ClassLoader.defineClass1(Native Method) at
java.base/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) at
java.base/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
at
java.base/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
at
java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:760)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) at
java.base/java.lang.Class.forName0(Native Method) at java.base/java
.lang.Class.forName(Class.java:467) at
java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.nextProviderClass(ServiceLoader.java:1217)
at
java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNextService(ServiceLoader.java:1228)
at
java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNext(ServiceLoader.java:1273)
at java.base/java.util.ServiceLoader$2.hasNext(ServiceLoader.java:1309) at
java.base/java.util.ServiceLoader$3.hasNext(ServiceLoader.java:1393) at
java.base/java.util.Iterator.forEachRemaining(Iterator.java:132) at
java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1845)
at
java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)
at
java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
at
java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at
java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:23
4) at
java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:682)
at
org.apache.gravitino.flink.connector.store.GravitinoCatalogStore.discoverFactories(GravitinoCatalogStore.java:144)
at
org.apache.gravitino.flink.connector.store.GravitinoCatalogStore.catalogFactory(GravitinoCatalogStore.java:126)
at
org.apache.gravitino.flink.connector.store.GravitinoCatalogStore.getCatalog(GravitinoCatalogStore.java:82)
at
org.apache.flink.table.catalog.CatalogManager.getCatalog(CatalogManager.java:401)
at
org.apache.flink.table.catalog.CatalogManager.setCurrentCatalog(CatalogManager.java:458)
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.useCatalog(TableEnvironmentImpl.java:1222)
at
org.apache.gravitino.flink.connector.integration.test.FlinkEnvIT.doWithSchema(FlinkEnvIT.java:167)
at
org.apache.gravitino.flink.connector.integration.test.hive.FlinkHiveCatalogIT.testHivePartitionTable(FlinkHiveCatalogIT.java:310)
Caused by: java.lang.ClassNotFoundException: org.apa
che.flink.table.gateway.api.endpoint.SqlGatewayEndpointFactory at
java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at
java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525) ... 31 more
This is likely the cause; this class implements SqlGatewayEndpointFactory,
but we do not actually need it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
