leesf commented on a change in pull request #3823:
URL: https://github.com/apache/hudi/pull/3823#discussion_r736658212
##########
File path:
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/factory/HoodieSparkKeyGeneratorFactory.java
##########
@@ -51,44 +49,70 @@
private static final Logger LOG =
LoggerFactory.getLogger(HoodieSparkKeyGeneratorFactory.class);
public static KeyGenerator createKeyGenerator(TypedProperties props) throws
IOException {
- // keyGenerator class name has higher priority
- KeyGenerator keyGenerator =
KeyGenUtils.createKeyGeneratorByClassName(props);
-
- return Objects.isNull(keyGenerator) ? createKeyGeneratorByType(props) :
keyGenerator;
+ String keyGeneratorClass = getKeyGeneratorClassName(props);
+ try {
+ return (KeyGenerator) ReflectionUtils.loadClass(keyGeneratorClass,
props);
+ } catch (Throwable e) {
+ throw new IOException("Could not load key generator class " +
keyGeneratorClass, e);
+ }
}
- private static BuiltinKeyGenerator createKeyGeneratorByType(TypedProperties
props) throws IOException {
- // Use KeyGeneratorType.SIMPLE as default keyGeneratorType
- String keyGeneratorType =
- props.getString(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), null);
+ public static String getKeyGeneratorClassName(TypedProperties props) {
+ String keyGeneratorClass =
props.getString(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), null);
- if (StringUtils.isNullOrEmpty(keyGeneratorType)) {
- LOG.info("The value of {} is empty, use SIMPLE",
HoodieWriteConfig.KEYGENERATOR_TYPE.key());
- keyGeneratorType = KeyGeneratorType.SIMPLE.name();
+ if (StringUtils.isNullOrEmpty(keyGeneratorClass)) {
+ String keyGeneratorType =
props.getString(HoodieWriteConfig.KEYGENERATOR_TYPE.key(),
KeyGeneratorType.SIMPLE.name());
+ KeyGeneratorType keyGeneratorTypeEnum;
+ try {
+ keyGeneratorTypeEnum =
KeyGeneratorType.valueOf(keyGeneratorType.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException e) {
+ throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type "
+ keyGeneratorType);
+ }
+ switch (keyGeneratorTypeEnum) {
+ case SIMPLE:
+ keyGeneratorClass = SimpleKeyGenerator.class.getName();
+ break;
+ case COMPLEX:
+ keyGeneratorClass = ComplexKeyGenerator.class.getName();
+ break;
+ case TIMESTAMP:
+ keyGeneratorClass = TimestampBasedKeyGenerator.class.getName();
+ break;
+ case CUSTOM:
+ keyGeneratorClass = CustomKeyGenerator.class.getName();
+ break;
+ case NON_PARTITION:
+ keyGeneratorClass = NonpartitionedKeyGenerator.class.getName();
+ break;
+ case GLOBAL_DELETE:
+ keyGeneratorClass = GlobalDeleteKeyGenerator.class.getName();
+ break;
+ default:
+ throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type
" + keyGeneratorType);
+ }
}
+ return keyGeneratorClass;
+ }
- KeyGeneratorType keyGeneratorTypeEnum;
- try {
- keyGeneratorTypeEnum =
KeyGeneratorType.valueOf(keyGeneratorType.toUpperCase(Locale.ROOT));
- } catch (IllegalArgumentException e) {
- throw new HoodieKeyGeneratorException("Unsupported keyGenerator Type " +
keyGeneratorType);
- }
- switch (keyGeneratorTypeEnum) {
- case SIMPLE:
- return new SimpleKeyGenerator(props);
- case COMPLEX:
- return new ComplexKeyGenerator(props);
- case TIMESTAMP:
- return new TimestampBasedKeyGenerator(props);
- case CUSTOM:
- return new CustomKeyGenerator(props);
- case NON_PARTITION:
- return new NonpartitionedKeyGenerator(props);
- case GLOBAL_DELETE:
- return new GlobalDeleteKeyGenerator(props);
+ /**
+ * Convert hoodie-common KeyGenerator to SparkKeyGeneratorInterface
implement.
+ */
+ public static String convertToSparkKeyGenerator(String
keyGeneratorClassName) {
+ switch (keyGeneratorClassName) {
+ case "org.apache.hudi.keygen.ComplexAvroKeyGenerator":
+ return "org.apache.hudi.keygen.ComplexKeyGenerator";
Review comment:
what's the difference between `case
"org.apache.hudi.keygen.ComplexAvroKeyGenerator":
return "org.apache.hudi.keygen.ComplexKeyGenerator";`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
