This is an automated email from the ASF dual-hosted git repository. krisden pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 4e2b961508bb705c9602fe65244b606678b6c5de Author: Mike Drob <[email protected]> AuthorDate: Mon Aug 1 12:21:17 2022 -0500 Update to Hadoop 3.3.3 (#907) * Update to Hadoop 3.3.3 * Updates to FileUtils from Hadoop upgrade From apache/hadoop@7f33a4e99205ea8c14e1c9e50797a5dd3818fe8b From apache/hadoop@fd96d5c2d5278aa6e7d527efa80761384c87bc26 --- solr/licenses/hadoop-annotations-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-annotations-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-auth-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-auth-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-client-api-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-client-api-3.3.3.jar.sha1 | 1 + .../hadoop-client-minicluster-3.3.2.jar.sha1 | 1 - .../hadoop-client-minicluster-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-common-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-common-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 | 1 - solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 | 1 + solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 | 1 + solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 | 1 - solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 | 1 + solr/modules/hadoop-auth/build.gradle | 1 + .../src/test/org/apache/hadoop/fs/FileUtil.java | 51 +++++++++++++++++----- versions.lock | 18 ++++---- versions.props | 2 +- 22 files changed, 61 insertions(+), 29 deletions(-) diff --git a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.2.jar.sha1 deleted file mode 100644 index d5a4186fb51..00000000000 --- a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2b47cd39c02c873b6bb29193962735d1d56f6572 diff --git a/solr/licenses/hadoop-annotations-3.3.3.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1 new file mode 100644 index 00000000000..b5e4d34342b --- /dev/null +++ b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1 @@ -0,0 +1 @@ +f093f02eb980be6afcda2c86aff71bcf8d9c5d9c diff --git a/solr/licenses/hadoop-auth-3.3.2.jar.sha1 b/solr/licenses/hadoop-auth-3.3.2.jar.sha1 deleted file mode 100644 index 889352f9ca9..00000000000 --- a/solr/licenses/hadoop-auth-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -32b81a77ea6ffcbf524d1cda8ab20ea3522e6fd5 diff --git a/solr/licenses/hadoop-auth-3.3.3.jar.sha1 b/solr/licenses/hadoop-auth-3.3.3.jar.sha1 new file mode 100644 index 00000000000..4c2d0542654 --- /dev/null +++ b/solr/licenses/hadoop-auth-3.3.3.jar.sha1 @@ -0,0 +1 @@ +6788fe94013801dca0712e3fd19b3c140f03b6e0 diff --git a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.2.jar.sha1 deleted file mode 100644 index 9c2cc214789..00000000000 --- a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -48f1af0a3a0270095dc59dc9f7d698969de4b4bf diff --git a/solr/licenses/hadoop-client-api-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1 new file mode 100644 index 00000000000..cd914d8cd2d --- /dev/null +++ b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1 @@ -0,0 +1 @@ +d0593aed2d4df9bcee507550913d29d589ebd84a diff --git a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1 deleted file mode 100644 index 379ff4a3bb3..00000000000 --- a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c08ddd065de27d21c2c2b398084092377f16a06b diff --git a/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1 new file mode 100644 index 00000000000..d0b9ae91b1b --- /dev/null +++ b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1 @@ -0,0 +1 @@ +5820fe54634c163e08c8040747fb089f31acb90d diff --git a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 deleted file mode 100644 index bd6ffdb79bc..00000000000 --- a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0112f2b7420fa77c62148799175c073594197e6c diff --git a/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 new file mode 100644 index 00000000000..1821815ab2a --- /dev/null +++ b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 @@ -0,0 +1 @@ +52619ecfb0225d7ae67b15264521064824ac57ca diff --git a/solr/licenses/hadoop-common-3.3.2.jar.sha1 b/solr/licenses/hadoop-common-3.3.2.jar.sha1 deleted file mode 100644 index a97b8aa2432..00000000000 --- a/solr/licenses/hadoop-common-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -edec4cdc7f1b1208f7d135f9f228ba44b83cf58f diff --git a/solr/licenses/hadoop-common-3.3.3.jar.sha1 b/solr/licenses/hadoop-common-3.3.3.jar.sha1 new file mode 100644 index 00000000000..46804c863ce --- /dev/null +++ b/solr/licenses/hadoop-common-3.3.3.jar.sha1 @@ -0,0 +1 @@ +2e1293fddedc9d5c42ce07703d22c33eb3b39aa3 diff --git a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 deleted file mode 100644 index a3abe8edce5..00000000000 --- a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20b47a6fe7780ae0c250081d2b9f30cd07691b6c diff --git a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 deleted file mode 100644 index 7f48dfd8182..00000000000 --- a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aed57238fd4e669043bcc29d6bea3f0e6420950d diff --git a/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 new file mode 100644 index 00000000000..f8fe3f45396 --- /dev/null +++ b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 @@ -0,0 +1 @@ +1bbe7acb60ffd432cc4bce646673279d0febdbc2 diff --git a/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 new file mode 100644 index 00000000000..5bc3a315c32 --- /dev/null +++ b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 @@ -0,0 +1 @@ +d4d199760c11d47f90e12fe3882e2b24c77e4eb5 diff --git a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 deleted file mode 100644 index 7e62097c3a6..00000000000 --- a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -749d44e9fe84566b4daa8898c62d8e88fd8e016f diff --git a/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 new file mode 100644 index 00000000000..ff534b2c382 --- /dev/null +++ b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 @@ -0,0 +1 @@ +1871c97108af3689f82c619cfd92f01b38e4f47c diff --git a/solr/modules/hadoop-auth/build.gradle b/solr/modules/hadoop-auth/build.gradle index 3f614e8bc8b..47382984ef9 100644 --- a/solr/modules/hadoop-auth/build.gradle +++ b/solr/modules/hadoop-auth/build.gradle @@ -95,6 +95,7 @@ dependencies { testImplementation ('org.apache.hadoop:hadoop-minikdc', { exclude group:'org.apache.kerby', module:'kerby-xdr' exclude group:'org.apache.kerby', module:'token-provider' + exclude group:'org.slf4j', module:'slf4j-reload4j' }) // Zookeeper dependency - some tests like HdfsCloudBackupRestore need this diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java index dbbb2b9f29e..a960b8ed3d1 100644 --- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java +++ b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java @@ -54,6 +54,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.AccessDeniedException; import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -888,10 +889,13 @@ public class FileUtil { private static void unTarUsingTar(File inFile, File untarDir, boolean gzipped) throws IOException { StringBuffer untarCommand = new StringBuffer(); + // not using canonical path here; this postpones relative path + // resolution until bash is executed. + final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'"; if (gzipped) { - untarCommand.append(" gzip -dc '") - .append(FileUtil.makeSecureShellPath(inFile)) - .append("' | ("); + untarCommand.append(" gzip -dc ") + .append(source) + .append(" | ("); } untarCommand.append("cd '") .append(FileUtil.makeSecureShellPath(untarDir)) @@ -901,15 +905,17 @@ public class FileUtil { if (gzipped) { untarCommand.append(" -)"); } else { - untarCommand.append(FileUtil.makeSecureShellPath(inFile)); + untarCommand.append(source); } + LOG.debug("executing [{}]", untarCommand); String[] shellCmd = { "bash", "-c", untarCommand.toString() }; ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); shexec.execute(); int exitcode = shexec.getExitCode(); if (exitcode != 0) { throw new IOException("Error untarring file " + inFile + - ". Tar process exited with exit code " + exitcode); + ". Tar process exited with exit code " + exitcode + + " from command " + untarCommand); } } @@ -966,6 +972,14 @@ public class FileUtil { + " would create entry outside of " + outputDir); } + if (entry.isSymbolicLink() || entry.isLink()) { + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + if (!canonicalTargetPath.startsWith(targetDirPath)) { + throw new IOException( + "expanding " + entry.getName() + " would create entry outside of " + outputDir); + } + } + if (entry.isDirectory()) { File subDir = new File(outputDir, entry.getName()); if (!subDir.mkdirs() && !subDir.isDirectory()) { @@ -981,10 +995,12 @@ public class FileUtil { } if (entry.isSymbolicLink()) { - // Create symbolic link relative to tar parent dir - Files.createSymbolicLink(FileSystems.getDefault() - .getPath(outputDir.getPath(), entry.getName()), - FileSystems.getDefault().getPath(entry.getLinkName())); + // Create symlink with canonical target path to ensure that we don't extract + // outside targetDirPath + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + Files.createSymbolicLink( + FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()), + FileSystems.getDefault().getPath(canonicalTargetPath)); return; } @@ -996,7 +1012,8 @@ public class FileUtil { } if (entry.isLink()) { - File src = new File(outputDir, entry.getLinkName()); + String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir); + File src = new File(canonicalTargetPath); HardLink.createHardLink(src, outputFile); return; } @@ -1004,6 +1021,20 @@ public class FileUtil { org.apache.commons.io.FileUtils.copyToFile(tis, outputFile); } + /** + * Gets the canonical path for the given path. + * + * @param path The path for which the canonical path needs to be computed. + * @param parentDir The parent directory to use if the path is a relative path. + * @return The canonical path of the given path. + */ + private static String getCanonicalPath(String path, File parentDir) throws IOException { + java.nio.file.Path targetPath = Paths.get(path); + return (targetPath.isAbsolute() ? + new File(path) : + new File(parentDir, path)).getCanonicalPath(); + } + /** * Class for creating hardlinks. * Supports Unix, WindXP. diff --git a/versions.lock b/versions.lock index 50daa2a73ff..28b35d1a9a0 100644 --- a/versions.lock +++ b/versions.lock @@ -132,11 +132,11 @@ org.apache.commons:commons-text:1.9 (2 constraints: 00165ad0) org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2) org.apache.curator:curator-framework:4.3.0 (2 constraints: ff13b474) org.apache.curator:curator-recipes:4.3.0 (1 constraints: 09050836) -org.apache.hadoop:hadoop-annotations:3.3.2 (1 constraints: 0a050536) -org.apache.hadoop:hadoop-auth:3.3.2 (1 constraints: 0a050536) -org.apache.hadoop:hadoop-client-api:3.3.2 (3 constraints: 1928ac5e) -org.apache.hadoop:hadoop-client-runtime:3.3.2 (2 constraints: 67170443) -org.apache.hadoop:hadoop-common:3.3.2 (1 constraints: 0a050536) +org.apache.hadoop:hadoop-annotations:3.3.3 (1 constraints: 0b050636) +org.apache.hadoop:hadoop-auth:3.3.3 (1 constraints: 0b050636) +org.apache.hadoop:hadoop-client-api:3.3.3 (3 constraints: 1c28435f) +org.apache.hadoop:hadoop-client-runtime:3.3.3 (2 constraints: 69173a43) +org.apache.hadoop:hadoop-common:3.3.3 (1 constraints: 0b050636) org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 (1 constraints: 0505f435) org.apache.httpcomponents:httpclient:4.5.13 (9 constraints: 5d801b3e) org.apache.httpcomponents:httpcore:4.4.15 (8 constraints: 1c6d2913) @@ -275,7 +275,7 @@ org.quicktheories:quicktheories:0.26 (1 constraints: dc04f530) org.reactivestreams:reactive-streams:1.0.3 (3 constraints: 3c2b02fd) org.slf4j:jcl-over-slf4j:1.7.36 (3 constraints: 05188eb8) org.slf4j:jul-to-slf4j:1.7.36 (3 constraints: 5928c263) -org.slf4j:slf4j-api:1.7.36 (40 constraints: 02f92fa3) +org.slf4j:slf4j-api:1.7.36 (40 constraints: 08f98db1) org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528) org.tallison:jmatio:1.5 (1 constraints: ff0b57e9) org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28) @@ -341,9 +341,9 @@ net.bytebuddy:byte-buddy:1.10.20 (2 constraints: 7c10a9d0) net.minidev:accessors-smart:2.4.7 (1 constraints: 4e0a90b8) net.minidev:json-smart:2.4.7 (1 constraints: 160e936e) no.nav.security:mock-oauth2-server:0.4.3 (1 constraints: 0905fa35) -org.apache.hadoop:hadoop-client-minicluster:3.3.2 (1 constraints: 0a050536) -org.apache.hadoop:hadoop-hdfs:3.3.2 (1 constraints: 0a050536) -org.apache.hadoop:hadoop-minikdc:3.3.2 (1 constraints: 0a050536) +org.apache.hadoop:hadoop-client-minicluster:3.3.3 (1 constraints: 0b050636) +org.apache.hadoop:hadoop-hdfs:3.3.3 (1 constraints: 0b050636) +org.apache.hadoop:hadoop-minikdc:3.3.3 (1 constraints: 0b050636) org.apache.kerby:kerb-admin:1.0.1 (1 constraints: 840d892f) org.apache.kerby:kerb-client:1.0.1 (1 constraints: 840d892f) org.apache.kerby:kerb-common:1.0.1 (2 constraints: a51841ca) diff --git a/versions.props b/versions.props index 23498efabca..aeb53bf0c0a 100644 --- a/versions.props +++ b/versions.props @@ -42,7 +42,7 @@ org.apache.commons:commons-math3=3.6.1 org.apache.commons:commons-text=1.9 org.apache.curator:*=4.3.0 org.apache.hadoop.thirdparty:*=1.1.1 -org.apache.hadoop:*=3.3.2 +org.apache.hadoop:*=3.3.3 org.apache.httpcomponents:httpclient=4.5.13 org.apache.httpcomponents:httpcore=4.4.15 org.apache.httpcomponents:httpmime=4.5.13
