YikSanChan commented on a change in pull request #15813:
URL: https://github.com/apache/flink/pull/15813#discussion_r629227421
##########
File path: flink-python/src/main/java/org/apache/flink/python/util/TarUtils.java
##########
@@ -0,0 +1,98 @@
+package org.apache.flink.python.util;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.IOUtils;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/** Utils used to extract tar files. */
+@Internal
+public class TarUtils {
+ public static void unTar(String inFilePath, String targetDirPath)
+ throws IOException, InterruptedException {
+ File targetDir = new File(targetDirPath);
+ if (!targetDir.mkdirs()) {
+ if (!targetDir.isDirectory()) {
+ throw new IOException("Mkdirs failed to create " + targetDir);
+ }
+ }
+ boolean gzipped = inFilePath.endsWith("gz");
+ if (DecompressUtils.isUnix()) {
+ unTarUsingTar(inFilePath, targetDirPath, gzipped);
+ } else {
+ unTarUsingJava(inFilePath, targetDirPath, gzipped);
+ }
+ }
+
+ // Copy and simplify from hadoop-common package that is used in YARN
+ // See
https://github.com/apache/hadoop/blob/7f93349ee74da5f35276b7535781714501ab2457/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+ private static void unTarUsingTar(String inFilePath, String targetDirPath,
boolean gzipped)
+ throws IOException, InterruptedException {
+ inFilePath = makeSecureShellPath(inFilePath);
+ targetDirPath = makeSecureShellPath(targetDirPath);
+ String untarCommand =
+ gzipped
+ ? String.format(
+ "gzip -dc '%s' | (cd '%s' && tar -xf -)",
inFilePath, targetDirPath)
Review comment:
I wonder if I can use `tar -xzf` directly? It is simpler, but I am not
exactly sure why hadoop-common chooses to do the more verbose way, i.e., `gzip
... | tar -xf ...`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
