Repository: ant-ivy Updated Branches: refs/heads/master 87c4e0656 -> 40708bdf0
IVY-1498 Fix potential Ivy cache corruption during retrieve task when symlink options come into play Project: http://git-wip-us.apache.org/repos/asf/ant-ivy/repo Commit: http://git-wip-us.apache.org/repos/asf/ant-ivy/commit/40708bdf Tree: http://git-wip-us.apache.org/repos/asf/ant-ivy/tree/40708bdf Diff: http://git-wip-us.apache.org/repos/asf/ant-ivy/diff/40708bdf Branch: refs/heads/master Commit: 40708bdf04dae6aa49bf8f3617f2efc16c83c5f8 Parents: 87c4e06 Author: Jaikiran Pai <[email protected]> Authored: Wed Jul 26 18:26:51 2017 +0530 Committer: Jaikiran Pai <[email protected]> Committed: Wed Jul 26 18:26:51 2017 +0530 ---------------------------------------------------------------------- src/java/org/apache/ivy/util/FileUtil.java | 25 ++- .../apache/ivy/core/retrieve/RetrieveTest.java | 162 +++++++++++++++---- .../1/org/foo-bar/ivys/ivy-2.3.4.xml | 22 +++ .../1/org/foo-bar/jars/foo-bar-1.2.3.jar | Bin 0 -> 470 bytes .../1/org/foo-bar/jars/foo-bar-2.3.4.jar | Bin 0 -> 475 bytes test/repositories/1/org/mod1/ivys/ivy-5.0.xml | 25 +++ test/repositories/1/org/mod1/jars/mod1-5.0.jar | 0 7 files changed, 197 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/src/java/org/apache/ivy/util/FileUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/ivy/util/FileUtil.java b/src/java/org/apache/ivy/util/FileUtil.java index 5446e9d..0ff62fa 100644 --- a/src/java/org/apache/ivy/util/FileUtil.java +++ b/src/java/org/apache/ivy/util/FileUtil.java @@ -17,6 +17,9 @@ */ package org.apache.ivy.util; +import org.apache.ivy.core.settings.TimeoutConstraint; +import org.apache.ivy.util.url.URLHandlerRegistry; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -29,6 +32,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.URL; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -45,9 +49,6 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.ZipInputStream; -import org.apache.ivy.core.settings.TimeoutConstraint; -import org.apache.ivy.util.url.URLHandlerRegistry; - /** * Utility class used to deal with file related operations, like copy, full reading, symlink, ... */ @@ -174,7 +175,7 @@ public final class FileUtil { return copy(src, dest, l, false); } - public static boolean prepareCopy(File src, File dest, boolean overwrite) throws IOException { + public static boolean prepareCopy(final File src, final File dest, final boolean overwrite) throws IOException { if (src.isDirectory()) { if (dest.exists()) { if (!dest.isDirectory()) { @@ -188,11 +189,25 @@ public final class FileUtil { } // else it is a file copy if (dest.exists()) { + // If overwrite is specified as "true" and the dest file happens to be a + // symlink, we delete the "link" (a.k.a unlink it). This is for cases + // like https://issues.apache.org/jira/browse/IVY-1498 where not unlinking + // the existing symlink can lead to potentially overwriting the wrong "target" file + // TODO: This behaviour is intentionally hardcoded here for now, since I don't + // see a reason (yet) to expose it as a param of this method. If any use case arises + // we can have this behaviour decided by the callers of this method, by passing a value for this + // param + final boolean unlinkSymlinkIfOverwrite = true; if (!dest.isFile()) { throw new IOException("impossible to copy: destination is not a file: " + dest); } if (overwrite) { - if (!dest.canWrite()) { + if (Files.isSymbolicLink(dest.toPath()) && unlinkSymlinkIfOverwrite) { + // unlink (a.k.a delete the symlink path) + dest.delete(); + } else if (!dest.canWrite()) { + // if the file *isn't* "writable" (see javadoc of File.canWrite() on what that means) + // we delete it. dest.delete(); } // if dest is writable, the copy will overwrite it without requiring a delete } else { http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java ---------------------------------------------------------------------- diff --git a/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java b/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java index 89a11fb..aae45d2 100644 --- a/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java +++ b/test/java/org/apache/ivy/core/retrieve/RetrieveTest.java @@ -17,21 +17,6 @@ */ package org.apache.ivy.core.retrieve; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - import org.apache.ivy.Ivy; import org.apache.ivy.TestHelper; import org.apache.ivy.core.IvyPatternHelper; @@ -55,6 +40,24 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class RetrieveTest { private Ivy ivy; @@ -250,23 +253,8 @@ public class RetrieveTest { "jar", "default")); } - private void assertLink(String filename) throws IOException { - // if the OS is known to support symlink, check that the file is a symlink, - // otherwise just check the file exist. - - File file = new File(filename); - assertTrue("The file " + filename + " doesn't exist", file.exists()); - - String os = System.getProperty("os.name"); - if (os.equals("Linux") || os.equals("Solaris") || os.equals("FreeBSD") - || os.equals("Mac OS X")) { - // these OS should support symlink, so check that the file is actually a symlink. - // this is done be checking that the canonical path is different from the absolute - // path. - File absFile = file.getAbsoluteFile(); - File canFile = file.getCanonicalFile(); - assertFalse("The file " + filename + " isn't a symlink", absFile.equals(canFile)); - } + private void assertLink(final String filename) throws IOException { + assertTrue(filename + " was expected to be a symlink", Files.isSymbolicLink(Paths.get(filename))); } @Test @@ -409,6 +397,116 @@ public class RetrieveTest { assertTrue("Retrieved artifact at " + dest.getAbsolutePath() + " is not a file", dest.isFile()); } + /** + * Tests that more than one retrieve, with one having symlink enabled and the other not, doesn't lead to repository + * cache corruption as reported in https://issues.apache.org/jira/browse/IVY-1498 + * <p> + * The test does the following: + * - Resolves a module, let's call it "irrelevant-A" which has a dependency on "org:foo-bar:1.2.3" + * <p> + * - Invokes a retrieve RT1, with {@link RetrieveOptions#setMakeSymlinks(boolean) symlinks true} + * on that resolved module with a pattern that translates to a path "a/b/c/foo-bar.jar". When the RT1 retrieve is + * done, the path "a/b/c/foo-bar.jar" will be a symlink to a path "repo/cache/org/foo-bar/foo-bar-1.2.3.jar" in the + * Ivy cache. + * All fine so far. + * <p> + * - We then resolve another module, let's call it "irrelevant-B" which has a dependency on "org:foo-bar:2.3.4" + * <p> + * - Next, do a new retrieve RT2, on this newly resolved module with + * {@link RetrieveOptions#setMakeSymlinks(boolean) symlinks false} and {@link RetrieveOptions#setOverwriteMode(String) overwrite true} + * and with the same pattern as before), that translates to a path "a/b/c/foo-bar.jar". + * <p> + * When RT2 retrieve is done, we expect the path "a/b/c/foo-bar.jar" will *not* be a symlink and instead be an actual file that represents + * the org:foo-bar:2.3.4 artifact jar. + * <p> + * We also expect that this RT2 retrieve will not update/overwrite the file at path "repo/cache/org/foo-bar/foo-bar-1.2.3.jar" + * in the Ivy cache - the file that was the end target of the symlink generated by the previous RT1. + * + * @throws Exception + * @see <a href="https://issues.apache.org/jira/browse/IVY-1498";>IVY-1498</a> + */ + @Test + public void testRetrieveWithSymlinkToggling() throws Exception { + // org:mod1:2.0 depends on org:foo-bar:1.2.3 + final ResolveReport resolve1Report = ivy.resolve(new File("test/repositories/1/org/mod1/ivys/ivy-2.0.xml").toURI().toURL(), + getResolveOptions(new String[]{"*"})); + assertNotNull("Resolve report isn't available", resolve1Report); + assertNotNull("Module descriptor missing in resolve report", resolve1Report.getModuleDescriptor()); + + // verify the downloaded dependency artifact in cache + final ModuleRevisionId fooBar123Mrid = ModuleRevisionId.newInstance("org", "foo-bar", "1.2.3"); + final ArtifactDownloadReport[] downloadReports = resolve1Report.getArtifactsReports(fooBar123Mrid); + assertNotNull("No artifact download report found for the " + fooBar123Mrid + " dependency", downloadReports); + assertEquals("Unexpected number for artifact download reports for the " + fooBar123Mrid + " dependency", 1, downloadReports.length); + final File fooBar123ArtifactInCache = downloadReports[0].getLocalFile(); + assertNotNull("Local file for " + fooBar123Mrid + " missing in download report", fooBar123ArtifactInCache); + assertTrue("Artifact file for " + fooBar123Mrid + " isn't a file, in cache at " + fooBar123ArtifactInCache, + fooBar123ArtifactInCache.isFile()); + final byte[] fooBar123ArtifactContentsInCache = Files.readAllBytes(fooBar123ArtifactInCache.toPath()); + assertTrue("Artifact content was empty at " + fooBar123ArtifactContentsInCache, fooBar123ArtifactContentsInCache.length > 0); + + + // now do a retrieve of the resolved module + final String retrievePattern = "build/test/retrieve/symlink-test/[module]/[artifact].[ext]"; + ivy.retrieve(resolve1Report.getModuleDescriptor().getModuleRevisionId(), + getRetrieveOptions().setMakeSymlinks(true).setDestArtifactPattern(retrievePattern)); + // we expect org:foo-bar:1.2.3 to have been retrieved + final Path retrievedArtifactSymlinkPath = Paths.get(IvyPatternHelper.substitute(retrievePattern, "org", "foo-bar", + "1.2.3", "foo-bar", "jar", "jar", "default")); + assertTrue("Artifact wasn't retrieved to " + retrievedArtifactSymlinkPath, Files.exists(retrievedArtifactSymlinkPath)); + assertTrue("Artifact retrieved at " + retrievedArtifactSymlinkPath + " was expected to be a " + + "symlink", Files.isSymbolicLink(retrievedArtifactSymlinkPath)); + + // get hold of the contents of the retrieved artifact + final byte[] retrievedArtifactContent = Files.readAllBytes(retrievedArtifactSymlinkPath); + // compare it to the contents of org:foo-bar:1.2.3 artifact in repo cache. Should be the same + assertTrue("Unexpected content in the retrieved artifact at " + retrievedArtifactSymlinkPath, + Arrays.equals(fooBar123ArtifactContentsInCache, retrievedArtifactContent)); + + // let this retrieved artifact file stay and let's now try and resolve a module and then retrieve a different version + // of the same artifact, *without* symlinking enabled, which will end up at this exact location + // org:mod1:2.0 depends on org:foo-bar:1.2.3 + final ResolveReport resolve2Report = ivy.resolve(new File("test/repositories/1/org/mod1/ivys/ivy-5.0.xml").toURI().toURL(), + getResolveOptions(new String[]{"*"})); + assertNotNull("Resolve report isn't available", resolve2Report); + assertNotNull("Module descriptor missing in resolve report", resolve2Report.getModuleDescriptor()); + + // verify the downloaded dependency artifact in cache + final ModuleRevisionId fooBar234Mrid = ModuleRevisionId.newInstance("org", "foo-bar", "2.3.4"); + final ArtifactDownloadReport[] foobar234report = resolve2Report.getArtifactsReports(fooBar234Mrid); + assertNotNull("No artifact download report found for the " + fooBar234Mrid + " dependency", foobar234report); + assertEquals("Unexpected number for artifact download reports for the " + fooBar234Mrid + " dependency", 1, foobar234report.length); + final File foobar234InCache = foobar234report[0].getLocalFile(); + assertNotNull("Local file for " + fooBar234Mrid + " missing in download report", foobar234InCache); + assertTrue("Artifact file for " + fooBar234Mrid + " isn't a file, in cache at " + foobar234InCache, + foobar234InCache.isFile()); + final byte[] foobar234CacheFileContents = Files.readAllBytes(foobar234InCache.toPath()); + assertTrue("Artifact content was empty at " + foobar234CacheFileContents, foobar234CacheFileContents.length > 0); + + // do the retrieve with symlinks disabled + ivy.retrieve(resolve2Report.getModuleDescriptor().getModuleRevisionId(), + getRetrieveOptions().setMakeSymlinks(false).setDestArtifactPattern(retrievePattern)); + // we expect org:foo-bar:2.3.4 to have been retrieved + final Path secondRetrieveArtifactPath = Paths.get(IvyPatternHelper.substitute(retrievePattern, "org", "foo-bar", + "2.3.4", "foo-bar", "jar", "jar", "default")); + assertTrue("Artifact wasn't retrieved to " + secondRetrieveArtifactPath, Files.exists(secondRetrieveArtifactPath)); +// assertFalse("Artifact retrieved at " + secondRetrieveArtifactPath + " wasn't expected to be a " + +// "symlink", Files.isSymbolicLink(secondRetrieveArtifactPath)); + + // get hold of the contents of the retrieved artifact + final byte[] secondRetrievedArtifactContents = Files.readAllBytes(secondRetrieveArtifactPath); + // compare it to the contents of org:foo-bar:2.3.4 artifact in repo cache. Should be the same + assertTrue("Unexpected content in the retrieved artifact at " + secondRetrieveArtifactPath, + Arrays.equals(foobar234CacheFileContents, secondRetrievedArtifactContents)); + + // also make sure that this retrieved content hasn't messed up the other (unrelated) file, in the cache, + // that was previously symlinked to this retrieved path + assertTrue("A second retrieve of an artifact has corrupted an unrelated (previously symlinked) artifact in cache", + Arrays.equals(fooBar123ArtifactContentsInCache, Files.readAllBytes(Paths.get(fooBar123ArtifactInCache.getPath())))); + + + } + private RetrieveOptions getRetrieveOptions() { return new RetrieveOptions(); } http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml ---------------------------------------------------------------------- diff --git a/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml b/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml new file mode 100644 index 0000000..f49b7af --- /dev/null +++ b/test/repositories/1/org/foo-bar/ivys/ivy-2.3.4.xml @@ -0,0 +1,22 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<ivy-module version="1.0"> + <info organisation="org" + module="foo-bar" + revision="2.3.4" + /> +</ivy-module> http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar ---------------------------------------------------------------------- diff --git a/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar b/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar index e69de29..d3cd0ae 100644 Binary files a/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar and b/test/repositories/1/org/foo-bar/jars/foo-bar-1.2.3.jar differ http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar ---------------------------------------------------------------------- diff --git a/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar b/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar new file mode 100644 index 0000000..26ae57a Binary files /dev/null and b/test/repositories/1/org/foo-bar/jars/foo-bar-2.3.4.jar differ http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/mod1/ivys/ivy-5.0.xml ---------------------------------------------------------------------- diff --git a/test/repositories/1/org/mod1/ivys/ivy-5.0.xml b/test/repositories/1/org/mod1/ivys/ivy-5.0.xml new file mode 100644 index 0000000..417a213 --- /dev/null +++ b/test/repositories/1/org/mod1/ivys/ivy-5.0.xml @@ -0,0 +1,25 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<ivy-module version="1.0"> + <info organisation="org" + module="mod1" + revision="5.0" + /> + <dependencies> + <dependency org="org" name="foo-bar" rev="2.3.4" /> + </dependencies> +</ivy-module> http://git-wip-us.apache.org/repos/asf/ant-ivy/blob/40708bdf/test/repositories/1/org/mod1/jars/mod1-5.0.jar ---------------------------------------------------------------------- diff --git a/test/repositories/1/org/mod1/jars/mod1-5.0.jar b/test/repositories/1/org/mod1/jars/mod1-5.0.jar new file mode 100644 index 0000000..e69de29
