danny0405 commented on code in PR #5528:
URL: https://github.com/apache/hudi/pull/5528#discussion_r867610262
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -746,16 +762,18 @@ public static Configuration flatOptions(Configuration
conf) {
: key;
propsMap.put(subKey, value);
});
- return fromMap(propsMap);
+ return Configuration.fromMap(propsMap);
}
- private static boolean hasPropertyOptions(Map<String, String> options) {
- return options.keySet().stream().anyMatch(k ->
k.startsWith(PROPERTIES_PREFIX));
+ private static boolean hasPropertyOptions(Map<String, String> options,
String prefix) {
+ return options.keySet().stream().anyMatch(k -> k.startsWith(prefix));
}
/**
* Creates a new configuration that is initialized with the options of the
given map.
+ * @deprecated Use {@link Configuration#fromMap(Map)} instead.
*/
+ @Deprecated
public static Configuration fromMap(Map<String, String> map) {
Review Comment:
We better keep this method for compatibility.
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java:
##########
@@ -708,32 +709,47 @@ private FlinkOptions() {
// Prefix for Hoodie specific properties.
private static final String PROPERTIES_PREFIX = "properties.";
-
- /**
- * Collects the config options that start with 'properties.' into a
'key'='value' list.
- */
- public static Map<String, String> getHoodieProperties(Map<String, String>
options) {
- return getHoodiePropertiesWithPrefix(options, PROPERTIES_PREFIX);
- }
+ private static final String HADOOP_PREFIX = "hadoop.";
+ private static final String PARQUET_PREFIX = "parquet.";
/**
* Collects the config options that start with specified prefix {@code
prefix} into a 'key'='value' list.
*/
- public static Map<String, String> getHoodiePropertiesWithPrefix(Map<String,
String> options, String prefix) {
+ public static Map<String, String> getPropertiesWithPrefix(Map<String,
String> options, String prefix) {
final Map<String, String> hoodieProperties = new HashMap<>();
-
- if (hasPropertyOptions(options)) {
+ if (hasPropertyOptions(options, prefix)) {
options.keySet().stream()
- .filter(key -> key.startsWith(PROPERTIES_PREFIX))
+ .filter(key -> key.startsWith(prefix))
.forEach(key -> {
final String value = options.get(key);
- final String subKey = key.substring((prefix).length());
+ final String subKey = key.substring(prefix.length());
hoodieProperties.put(subKey, value);
});
}
return hoodieProperties;
}
+ public static org.apache.hadoop.conf.Configuration getParquetConf(
+ org.apache.flink.configuration.Configuration options,
+ org.apache.hadoop.conf.Configuration hadoopConf) {
+ org.apache.hadoop.conf.Configuration copy = new
org.apache.hadoop.conf.Configuration(hadoopConf);
+ Map<String, String> parquetOptions =
getPropertiesWithPrefix(options.toMap(), PARQUET_PREFIX);
+ parquetOptions.forEach((k, v) -> copy.set(PARQUET_PREFIX + k, v));
+ return copy;
+ }
+
+ // Keep the redundant to avoid too many modifications.
+ public static org.apache.hadoop.conf.Configuration
getHadoopConf(Configuration conf) {
+ if (conf == null) {
+ return FlinkClientUtil.getHadoopConf();
Review Comment:
In which case the conf can be null ?
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java:
##########
@@ -351,6 +355,7 @@ public static HoodieTableMetaClient createMetaClient(String
basePath, org.apache
/**
* Creates the meta client.
*/
+ @Deprecated
public static HoodieTableMetaClient createMetaClient(String basePath) {
Review Comment:
Can we remove the method directly ?
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java:
##########
@@ -140,9 +140,13 @@ public static DFSPropertiesConfiguration
readConfig(org.apache.hadoop.conf.Confi
return conf;
}
- // Keep the redundant to avoid too many modifications.
+ /**
+ * Keep the redundant to avoid too many modifications.
+ * @deprecated Use {@link FlinkOptions#getHadoopConf(Configuration)} instead.
+ */
+ @Deprecated
Review Comment:
Can we remove the method directly ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
