linliu-code commented on code in PR #13300:
URL: https://github.com/apache/hudi/pull/13300#discussion_r2093404642
##########
hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java:
##########
@@ -110,22 +115,61 @@ public Set<Pair<String, Long>>
filterRowKeys(HoodieStorage storage, StoragePath
@Override
public ClosableIterator<Pair<HoodieKey, Long>>
fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
- throw new UnsupportedOperationException("HFileUtils does not support
fetchRecordKeysWithPositions");
+ return fetchRecordKeysWithPositions(storage, filePath, Option.empty(),
Option.empty());
}
@Override
public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage
storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt,
Option<String> partitionPath) {
- throw new UnsupportedOperationException("HFileUtils does not support
getHoodieKeyIterator");
+ try {
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(HadoopFSUtils.getFs(filePath.toString(),
conf).getConf());
+ HoodieNativeAvroHFileReader reader = new
HoodieNativeAvroHFileReader(storage, filePath, Option.empty());
+ ClosableIterator<String> keyIterator = reader.getRecordKeyIterator();
+ return new ClosableIterator<HoodieKey>() {
+ @Override
+ public void close() {
+ keyIterator.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return keyIterator.hasNext();
+ }
+
+ @Override
+ public HoodieKey next() {
+ String key = keyIterator.next();
+ if (partitionPath.isPresent()) {
+ return new HoodieKey(key, partitionPath.get());
+ } else {
+ return new HoodieKey(key, null);
+ }
+ }
+ };
+ } catch (IOException e) {
+ throw new HoodieIOException("Unable to read the HFile: ", e);
+ }
}
@Override
public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage
storage, StoragePath filePath) {
- throw new UnsupportedOperationException("HFileUtils does not support
getHoodieKeyIterator");
+ return getHoodieKeyIterator(storage, filePath, Option.empty(),
Option.empty());
}
@Override
public ClosableIterator<Pair<HoodieKey, Long>>
fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt, Option<String> partitionPath) {
- throw new UnsupportedOperationException("HFileUtils does not support
fetchRecordKeysWithPositions");
+ try {
+ if (filePath == null || !storage.exists(filePath)) {
+ return ClosableIterator.wrap(Collections.emptyIterator());
+ } else {
+ AtomicLong position = new AtomicLong(0);
Review Comment:
I don't know if we have that use case or not.
So I implemented it first, and later to decide if we should use it or not.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
