smallzhongfeng commented on code in PR #680:
URL: https://github.com/apache/incubator-uniffle/pull/680#discussion_r1125483941
##########
coordinator/src/main/java/org/apache/uniffle/coordinator/strategy/storage/AbstractSelectStorageStrategy.java:
##########
@@ -76,6 +92,50 @@ public void readAndWriteHdfsStorage(FileSystem fs, Path
testPath,
}
}
+ @Override
+ public void detectStorage() {
+ uris =
Lists.newCopyOnWriteArrayList(remoteStoragePathRankValue.entrySet());
+ if (remoteStoragePathRankValue.size() > 1) {
+ CountDownLatch countDownLatch = new CountDownLatch(uris.size());
+ uris.parallelStream().forEach(uri -> {
+ if
(uri.getKey().startsWith(ApplicationManager.getPathSchema().get(0))) {
+ Path remotePath = new Path(uri.getKey());
+ String rssTest = uri.getKey() + "/rssTest-" + getCoordinatorId()
+ + Thread.currentThread().getName();
+ Path testPath = new Path(rssTest);
+ RankValue rankValue = remoteStoragePathRankValue.get(uri.getKey());
+ rankValue.setHealthy(new AtomicBoolean(true));
+ long startWriteTime = System.currentTimeMillis();
+ try {
+ FileSystem fs = HadoopFilesystemProvider.getFilesystem(remotePath,
hdfsConf);
+ for (int j = 0; j < readAndWriteTimes; j++) {
+ readAndWriteHdfsStorage(fs, testPath, uri.getKey(), rankValue);
+ }
+ } catch (Exception e) {
+ LOG.error("Storage read and write error, we will not use this
remote path {}.", uri, e);
+ rankValue.setHealthy(new AtomicBoolean(false));
+ } finally {
+ if (this instanceof LowestIOSampleCostSelectStorageStrategy) {
+ ((LowestIOSampleCostSelectStorageStrategy) this)
+ .sortPathByRankValue(uri.getKey(), rssTest, startWriteTime,
hdfsConf);
+ } else if (this instanceof AppBalanceSelectStorageStrategy) {
+ ((AppBalanceSelectStorageStrategy) this)
+ .sortPathByRankValue(uri.getKey(), rssTest, hdfsConf);
Review Comment:
Do you mean like this
https://github.com/apache/incubator-uniffle/pull/680/commits/fd752501e6165d6f59e9b88c5d52d449a9a154bf
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
