yihua commented on code in PR #19304:
URL: https://github.com/apache/hudi/pull/19304#discussion_r4102929779
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java:
##########
@@ -385,4 +408,91 @@ public static boolean
mayUseNewEncodingForComplexKeyGen(HoodieTableConfig tableC
return tableConfig.getTableVersion().lesserThan(HoodieTableVersion.NINE)
&& isComplexKeyGeneratorWithSingleRecordKeyField(tableConfig);
}
+
+ public static StoragePath getComplexKeyEncodingFilePath(StoragePath
basePath) {
+ return new StoragePath(basePath, AUXILIARYFOLDER_NAME + "/" +
COMPLEX_KEY_ENCODING_FILE_NAME);
+ }
+
+ public static Option<Boolean>
readComplexKeyEncodingFromAuxFile(HoodieStorage storage, StoragePath basePath) {
+ StoragePath encodingFilePath = getComplexKeyEncodingFilePath(basePath);
+ try {
+ if (storage.exists(encodingFilePath)) {
+ Properties props = new Properties();
+ try (InputStream inputStream = storage.open(encodingFilePath)) {
+ props.load(inputStream);
+ }
+ String value = props.getProperty(COMPLEX_KEYGEN_NEW_ENCODING.key());
+ if (value != null) {
+ return Option.of(Boolean.parseBoolean(value));
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to read complex key encoding from aux file: {}",
encodingFilePath, e);
+ }
+ return Option.empty();
+ }
+
+ public static void writeComplexKeyEncodingToAuxFile(HoodieStorage storage,
StoragePath basePath, boolean useNewEncoding) {
+ StoragePath encodingFilePath = getComplexKeyEncodingFilePath(basePath);
+ try {
+ Properties props = new Properties();
+ props.setProperty(COMPLEX_KEYGEN_NEW_ENCODING.key(),
String.valueOf(useNewEncoding));
+ try (OutputStream outputStream = storage.create(encodingFilePath, true))
{
+ props.store(outputStream, "Complex key generator encoding format");
+ }
+ LOG.info("Wrote complex key encoding to aux file: {}", useNewEncoding);
+ } catch (IOException e) {
+ throw new HoodieKeyException("Failed to write complex key encoding file
to " + encodingFilePath, e);
+ }
+ }
+
+ public static final boolean DEFAULT_NEW_ENCODING_FOR_NEW_TABLE = true;
+
+ public static boolean deduceComplexKeyEncodingFromData(HoodieTableMetaClient
metaClient, String recordKeyFieldName) {
+ HoodieTimeline completedTimeline =
metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
+ if (completedTimeline.empty()) {
+ LOG.info("No completed commits found in table {}; defaulting complex key
encoding to useNewEncoding={} (new/empty table).",
+ metaClient.getBasePath(), DEFAULT_NEW_ENCODING_FOR_NEW_TABLE);
+ return DEFAULT_NEW_ENCODING_FOR_NEW_TABLE;
+ }
+
+ try {
+ HoodieStorage storage = metaClient.getStorage();
+ FileFormatUtils fileFormatUtils = HoodieIOFactory.getIOFactory(storage)
+ .getFileFormatUtils(HoodieFileFormat.PARQUET);
+
+ List<HoodieInstant> instants =
completedTimeline.getReverseOrderedInstants().collect(Collectors.toList());
+ for (HoodieInstant instant : instants) {
+ HoodieCommitMetadata commitMetadata =
TimelineUtils.getCommitMetadata(instant, completedTimeline);
+ for (HoodieWriteStat writeStat : commitMetadata.getWriteStats()) {
+ String filePath = writeStat.getPath();
+ if (filePath == null || filePath.isEmpty() ||
!filePath.endsWith(".parquet")) {
+ continue;
+ }
+ StoragePath baseFilePath = new StoragePath(metaClient.getBasePath(),
filePath);
+ if (!storage.exists(baseFilePath)) {
+ continue;
+ }
+ try (ClosableIterator<HoodieKey> keyIterator =
fileFormatUtils.getHoodieKeyIterator(storage, baseFilePath)) {
+ if (keyIterator.hasNext()) {
+ HoodieKey hoodieKey = keyIterator.next();
+ String hoodieRecordKey = hoodieKey.getRecordKey();
+ String expectedPrefix = recordKeyFieldName +
DEFAULT_COLUMN_VALUE_SEPARATOR;
+ boolean usesNewEncoding =
!hoodieRecordKey.startsWith(expectedPrefix);
+ LOG.info("Deduced complex key encoding from base file {} (commit
{}): useNewEncoding={}",
+ baseFilePath, instant.requestedTime(), usesNewEncoding);
+ return usesNewEncoding;
+ }
+ }
+ }
+ }
+
+ LOG.info("No base files with records found in table {}; defaulting
complex key encoding to useNewEncoding={}.",
Review Comment:
Fixed. Log files are also considered.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
