nsivabalan commented on code in PR #6284:
URL: https://github.com/apache/hudi/pull/6284#discussion_r1014904091
##########
hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java:
##########
@@ -204,4 +204,40 @@ public static Option<byte[]> readDataFromPath(FileSystem
fileSystem, org.apache.
public static Option<byte[]> readDataFromPath(FileSystem fileSystem,
org.apache.hadoop.fs.Path detailPath) {
return readDataFromPath(fileSystem, detailPath, false);
}
+
+ /**
+ * Return the configured local directories where hudi can write files. This
+ * method does not create any directories on its own, it only encapsulates
the
+ * logic of locating the local directories according to deployment mode.
+ */
+ public static String[] getConfiguredLocalDirs() {
+ if (isRunningInYarnContainer()) {
+ // If we are in yarn mode, systems can have different disk layouts so we
must set it
+ // to what Yarn on this system said was available. Note this assumes
that Yarn has
+ // created the directories already, and that they are secured so that
only the
+ // user has access to them.
+ return getYarnLocalDirs().split(",");
+ } else if (System.getProperty("java.io.tmpdir") != null) {
+ return System.getProperty("java.io.tmpdir").split(",");
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isRunningInYarnContainer() {
+ // These environment variables are set by YARN.
+ return System.getenv("CONTAINER_ID") != null;
+ }
+
+ /**
+ * Get the Yarn approved local directories.
+ */
+ public static String getYarnLocalDirs() {
Review Comment:
same comment as above.
##########
hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java:
##########
@@ -204,4 +204,40 @@ public static Option<byte[]> readDataFromPath(FileSystem
fileSystem, org.apache.
public static Option<byte[]> readDataFromPath(FileSystem fileSystem,
org.apache.hadoop.fs.Path detailPath) {
return readDataFromPath(fileSystem, detailPath, false);
}
+
+ /**
+ * Return the configured local directories where hudi can write files. This
+ * method does not create any directories on its own, it only encapsulates
the
+ * logic of locating the local directories according to deployment mode.
+ */
+ public static String[] getConfiguredLocalDirs() {
+ if (isRunningInYarnContainer()) {
+ // If we are in yarn mode, systems can have different disk layouts so we
must set it
+ // to what Yarn on this system said was available. Note this assumes
that Yarn has
+ // created the directories already, and that they are secured so that
only the
+ // user has access to them.
+ return getYarnLocalDirs().split(",");
+ } else if (System.getProperty("java.io.tmpdir") != null) {
+ return System.getProperty("java.io.tmpdir").split(",");
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isRunningInYarnContainer() {
Review Comment:
why these are public? can we make it private.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
