This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
commit 33d2995bc2612e0d3e9ad0cfc2a88a9a7f5141bc Author: Ivan Suller <[email protected]> AuthorDate: Tue Jul 9 17:43:09 2019 +0200 HIVE-21933: Remove unused methods from Utilities (Ivan Suller via Laszlo Bodor) Signed-off-by: Zoltan Haindrich <[email protected]> --- .../org/apache/hadoop/hive/ql/exec/Utilities.java | 155 ++------------------- .../apache/hadoop/hive/ql/exec/TestUtilities.java | 9 +- 2 files changed, 12 insertions(+), 152 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 68986dd..1346bed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -404,7 +404,7 @@ public final class Utilities { /** * Pushes work into the global work map */ - public static void setBaseWork(Configuration conf, String name, BaseWork work) { + private static void setBaseWork(Configuration conf, String name, BaseWork work) { Path path = getPlanPath(conf, name); setHasWork(conf, name); gWorkMap.get(conf).put(path, work); @@ -753,14 +753,6 @@ public final class Utilities { } } - public static HashMap makeMap(Object... olist) { - HashMap ret = new HashMap(); - for (int i = 0; i < olist.length; i += 2) { - ret.put(olist[i], olist[i + 1]); - } - return (ret); - } - public static Properties makeProperties(String... olist) { Properties ret = new Properties(); for (int i = 0; i < olist.length; i += 2) { @@ -799,37 +791,11 @@ public final class Utilities { return new PartitionDesc(part, tableDesc); } - public static PartitionDesc getPartitionDesc(Partition part) throws HiveException { - return new PartitionDesc(part); - } - public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part, boolean usePartSchemaProperties) throws HiveException { return new PartitionDesc(part, tblDesc, usePartSchemaProperties); } - private static String getOpTreeSkel_helper(Operator<?> op, String indent) { - if (op == null) { - return StringUtils.EMPTY; - } - - StringBuilder sb = new StringBuilder(); - sb.append(indent); - sb.append(op.toString()); - sb.append("\n"); - if (op.getChildOperators() != null) { - for (Object child : op.getChildOperators()) { - sb.append(getOpTreeSkel_helper((Operator<?>) child, indent + " ")); - } - } - - return sb.toString(); - } - - public static String getOpTreeSkel(Operator<?> op) { - return getOpTreeSkel_helper(op, StringUtils.EMPTY); - } - private static boolean isWhitespace(int c) { if (c == -1) { return false; @@ -926,22 +892,6 @@ public final class Utilities { } /** - * Convert an output stream to a compressed output stream based on codecs and compression options - * specified in the Job Configuration. - * - * @param jc - * Job Configuration - * @param out - * Output Stream to be converted into compressed output stream - * @return compressed output stream - */ - public static OutputStream createCompressedStream(JobConf jc, OutputStream out) - throws IOException { - boolean isCompressed = FileOutputFormat.getCompressOutput(jc); - return createCompressedStream(jc, out, isCompressed); - } - - /** * Convert an output stream to a compressed output stream based on codecs codecs in the Job * Configuration. Caller specifies directly whether file is compressed or not * @@ -966,22 +916,6 @@ public final class Utilities { } /** - * Based on compression option and configured output codec - get extension for output file. This - * is only required for text files - not sequencefiles - * - * @param jc - * Job Configuration - * @param isCompressed - * Whether the output file is compressed or not - * @return the required file extension (example: .gz) - * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)} - */ - @Deprecated - public static String getFileExtension(JobConf jc, boolean isCompressed) { - return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat()); - } - - /** * Based on compression option, output format, and configured output codec - * get extension for output file. Text files require an extension, whereas * others, like sequence files, do not. @@ -1014,27 +948,6 @@ public final class Utilities { } /** - * Create a sequencefile output stream based on job configuration. - * - * @param jc - * Job configuration - * @param fs - * File System to create file in - * @param file - * Path to be created - * @param keyClass - * Java Class for key - * @param valClass - * Java Class for value - * @return output stream over the created sequencefile - */ - public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file, - Class<?> keyClass, Class<?> valClass, Progressable progressable) throws IOException { - boolean isCompressed = FileOutputFormat.getCompressOutput(jc); - return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed, progressable); - } - - /** * Create a sequencefile output stream based on job configuration Uses user supplied compression * flag (rather than obtaining it from the Job Configuration). * @@ -1055,11 +968,11 @@ public final class Utilities { throws IOException { CompressionCodec codec = null; CompressionType compressionType = CompressionType.NONE; - Class codecClass = null; + Class<? extends CompressionCodec> codecClass = null; if (isCompressed) { compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc); codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); - codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc); + codec = ReflectionUtil.newInstance(codecClass, jc); } return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec, progressable); @@ -1156,7 +1069,7 @@ public final class Utilities { /** * Detect if the supplied file is a temporary path. */ - public static boolean isTempPath(FileStatus file) { + private static boolean isTempPath(FileStatus file) { String name = file.getPath().getName(); // in addition to detecting hive temporary files, we also check hadoop // temporary folders that used to show up in older releases @@ -1181,46 +1094,6 @@ public final class Utilities { } } - /** - * Moves files from src to dst if it is within the specified set of paths - * @param fs - * @param src - * @param dst - * @param filesToMove - * @throws IOException - * @throws HiveException - */ - private static void moveSpecifiedFiles(FileSystem fs, Path src, Path dst, Set<Path> filesToMove) - throws IOException, HiveException { - if (!fs.exists(dst)) { - fs.mkdirs(dst); - } - - FileStatus[] files = fs.listStatus(src); - for (FileStatus file : files) { - if (filesToMove.contains(file.getPath())) { - Utilities.moveFile(fs, file, dst); - } else if (file.isDir()) { - // Traverse directory contents. - // Directory nesting for dst needs to match src. - Path nestedDstPath = new Path(dst, file.getPath().getName()); - Utilities.moveSpecifiedFiles(fs, file.getPath(), nestedDstPath, filesToMove); - } - } - } - - public static void moveSpecifiedFiles(FileSystem fs, Path dst, Set<Path> filesToMove) - throws IOException, HiveException { - if (!fs.exists(dst)) { - fs.mkdirs(dst); - } - - for (Path path: filesToMove) { - FileStatus fsStatus = fs.getFileStatus(path); - Utilities.moveFile(fs, fsStatus, dst); - } - } - private static void moveFile(FileSystem fs, FileStatus file, Path dst) throws IOException, HiveException { Path srcFilePath = file.getPath(); @@ -1333,7 +1206,7 @@ public final class Utilities { * @param filename * filename to extract taskid from */ - public static String getPrefixedTaskIdFromFilename(String filename) { + private static String getPrefixedTaskIdFromFilename(String filename) { return getTaskIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX); } @@ -1341,7 +1214,7 @@ public final class Utilities { return getIdFromFilename(filename, pattern, 1); } - public static int getAttemptIdFromFilename(String filename) { + private static int getAttemptIdFromFilename(String filename) { String attemptStr = getIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX, 3); return Integer.parseInt(attemptStr.substring(1)); } @@ -1364,14 +1237,6 @@ public final class Utilities { return taskId; } - public static String getFileNameFromDirName(String dirName) { - int dirEnd = dirName.lastIndexOf(Path.SEPARATOR); - if (dirEnd != -1) { - return dirName.substring(dirEnd + 1); - } - return dirName; - } - /** * Replace the task id from the filename. It is assumed that the filename is derived from the * output of getTaskId @@ -1481,7 +1346,7 @@ public final class Utilities { } - public static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) { + private static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) { // we are avoiding rename/move only if following conditions are met // * execution engine is tez // * query cache is disabled @@ -1647,7 +1512,7 @@ public final class Utilities { return removeTempOrDuplicateFiles(fs, stats, dpCtx, conf, hconf, isBaseDir); } - public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, + private static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, boolean isBaseDir) throws IOException { return removeTempOrDuplicateFiles(fs, fileStats, dpCtx, conf, hconf, null, isBaseDir); } @@ -1657,7 +1522,7 @@ public final class Utilities { * * @return a list of path names corresponding to should-be-created empty buckets. */ - public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, + private static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<FileStatus> filesKept, boolean isBaseDir) throws IOException { int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(), @@ -2711,7 +2576,6 @@ public final class Utilities { } } - @SuppressWarnings("unchecked") private static <T> List<T> getTasks(List<Task<? extends Serializable>> tasks, TaskFilterFunction<T> function) { DAGTraversal.traverse(tasks, function); @@ -3665,7 +3529,6 @@ public final class Utilities { * @param rWork Used to find FileSinkOperators * @throws IOException */ - @SuppressWarnings("unchecked") public static void createTmpDirs(Configuration conf, ReduceWork rWork) throws IOException { if (rWork == null) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index dc851c0..fbf948c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -101,10 +101,6 @@ public class TestUtilities { getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat())); assertEquals("Deflate for uncompressed text format", ".deflate", getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat())); - assertEquals("No extension for uncompressed default format", "", - getFileExtension(jc, false)); - assertEquals("Deflate for uncompressed default format", ".deflate", - getFileExtension(jc, true)); String extension = ".myext"; jc.set("hive.output.file.extension", extension); @@ -349,7 +345,7 @@ public class TestUtilities { String testPartitionName = "p=" + i; testPartitionsPaths[i] = new Path(testTablePath, "p=" + i); mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); - mapWork.getAliasToWork().put(testPartitionName, (Operator<?>) mock(Operator.class)); + mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class)); mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc); } @@ -501,7 +497,7 @@ public class TestUtilities { pathToAliasTable.put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName)); - mapWork.getAliasToWork().put(testPartitionName, (Operator<?>) mock(Operator.class)); + mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class)); } mapWork.setPathToAliases(pathToAliasTable); @@ -638,6 +634,7 @@ public class TestUtilities { super.addDependentTask(wrappedDep); } + @Override public boolean addDependentTask(Task<? extends Serializable> dependent) { return wrappedDep.addDependentTask(dependent); }
