vinishjail97 commented on code in PR #835:
URL: https://github.com/apache/incubator-xtable/pull/835#discussion_r4163040789
##########
xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java:
##########
@@ -244,13 +243,32 @@ ReplaceMetadata convertDiff(
return ReplaceMetadata.of(partitionToReplacedFileIds, writeStatuses);
}
- private String getFileId(Path filePath) {
- String fileName = filePath.getName();
+ private String getFileId(InternalDataFile file) {
+ String fileName = new CachingPath(file.getPhysicalPath()).getName();
// if file was created by Hudi use original fileId, otherwise use the file
name as IDs
if (isFileCreatedByHudiWriter(fileName)) {
return FSUtils.getFileId(fileName);
}
- return fileName;
+ // Files under a partition subdirectory keep it as part of the fileId so
the subdirectory can be
+ // recovered when Hudi resolves the physical path of the externally
created file.
+ return file.getPartitionSubdirectory().map(prefix -> prefix + "/" +
fileName).orElse(fileName);
+ }
+
+ /**
+ * Resolves the Hudi partition path for a file. A partition subdirectory
(e.g. Paimon's {@code
+ * bucket-N}) is a file group within the partition (see Hudi PR #17788), so
it is stripped from
+ * the partition path.
+ */
+ private String getHudiPartitionPath(Path tableBasePath, InternalDataFile
file) {
+ String partitionPath =
+ HudiPathUtils.getPartitionPath(tableBasePath, new
CachingPath(file.getPhysicalPath()));
+ return file.getPartitionSubdirectory()
+ .map(
+ prefix ->
+ prefix.equals(partitionPath)
+ ? ""
+ : partitionPath.substring(0, partitionPath.length() -
prefix.length() - 1))
Review Comment:
Added the check in d342cea.
##########
xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java:
##########
@@ -273,17 +291,33 @@ private WriteStatus toWriteStatus(
WriteStatus writeStatus = new WriteStatus();
Path path = new CachingPath(file.getPhysicalPath());
String partitionPath =
- partitionPathOptional.orElseGet(() ->
HudiPathUtils.getPartitionPath(tableBasePath, path));
- String fileId = getFileId(path);
+ partitionPathOptional.orElseGet(() ->
getHudiPartitionPath(tableBasePath, file));
+ String fileId = getFileId(file);
String filePath =
path.toUri().getPath().substring(tableBasePath.toUri().getPath().length() + 1);
String fileName = path.getName();
+ Optional<String> partitionSubdirectory = file.getPartitionSubdirectory();
+ // For files under a partition subdirectory encode it as the file-group
prefix in the marker and
+ // keep the
Review Comment:
Reflowed both comments in d342cea.
##########
xtable-core/src/main/java/org/apache/xtable/hudi/HudiTargetConfig.java:
##########
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.hudi;
+
+import java.util.Properties;
+
+import lombok.Value;
+
+import org.apache.hudi.common.table.HoodieTableVersion;
+
+/** Configuration of the Hudi conversion target. */
+@Value
+public class HudiTargetConfig {
+ /**
+ * Table format version to write for the Hudi target. Supported values are
{@code 6} (the legacy
+ * 0.x timeline layout, column-stats index V1) and {@code 9} (the Hudi 1.x
timeline layout,
+ * column-stats index V2). Defaults to {@code 6}.
+ */
+ public static final String HUDI_TABLE_VERSION =
"xtable.hudi.target.table_version";
+
+ static final HoodieTableVersion DEFAULT_TABLE_VERSION =
HoodieTableVersion.SIX;
+
+ HoodieTableVersion tableVersion;
+
+ public static HudiTargetConfig fromProperties(Properties properties) {
+ HoodieTableVersion tableVersion = DEFAULT_TABLE_VERSION;
+ if (properties != null) {
+ String configured = properties.getProperty(HUDI_TABLE_VERSION);
+ if (configured != null && !configured.trim().isEmpty()) {
+ tableVersion = parseTableVersion(configured.trim());
+ }
+ }
+ if (tableVersion != HoodieTableVersion.SIX && tableVersion !=
HoodieTableVersion.NINE) {
+ throw
unsupportedTableVersion(String.valueOf(tableVersion.versionCode()), null);
+ }
+ return new HudiTargetConfig(tableVersion);
+ }
+
+ private static HoodieTableVersion parseTableVersion(String configured) {
+ try {
+ return HoodieTableVersion.fromVersionCode(Integer.parseInt(configured));
+ } catch (RuntimeException e) {
Review Comment:
Done in d342cea.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
