This is an automated email from the ASF dual-hosted git repository.
mbien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new 22459cc10a improved maven indexer failure modes in low space situations
new 1e03c87c59 Merge pull request #5655 from
mbien/indexer-no-space-handling
22459cc10a is described below
commit 22459cc10ac87a613a7c230de52bd570329d974d
Author: Michael Bien <[email protected]>
AuthorDate: Sun Mar 12 03:22:01 2023 +0100
improved maven indexer failure modes in low space situations
- indexing will disable itself if the index failed to extract due
to no space left on the device (temp or cache) and notify the user
- take cause and suppressed exceptions into account while scanning
for "no space" messages
- added a fallback to clean up remaining files on exception
- simplified the code a bit
---
.../maven/indexer/NexusRepositoryIndexerImpl.java | 136 +++++++++++++--------
1 file changed, 83 insertions(+), 53 deletions(-)
diff --git
a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
index ec029d90d1..57c410c6d6 100644
---
a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
+++
b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java
@@ -28,22 +28,27 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
-import java.nio.file.FileStore;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
import java.util.zip.ZipError;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.codehaus.plexus.PlexusConstants;
import org.apache.lucene.search.*;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.maven.artifact.Artifact;
@@ -462,13 +467,18 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
removeIndexingContext(ic, false);
}
}
-
- @Messages({"# {0} - folder path",
- "# {1} - repository name",
- "MSG_NoSpace=There is not enough space in {0} to download and
unpack the index for ''{1}''.",
- "# {0} - folder path",
- "# {1} - repository name",
- "MSG_SeemsNoSpace=It seems that there is not enough space in
{0} to download and unpack the index for ''{1}''."})
+
+ @Messages({"# {0} - repository name",
+ "# {1} - cache path",
+ "# {2} - cache free storage",
+ "# {3} - tmp path",
+ "# {4} - tmp free storage",
+ "MSG_NoSpace="
+ +"<html>There is not enough space to download and
unpack the index for ''{0}''.<br/><br/>"
+ +"''{1}'' has {2} MB free<br/>"
+ +"''{3}'' has {4} MB free<br/><br/>"
+ +"Maven indexing is now disabled and can be enabled
again in the maven settings.</html>",
+ })
private void indexLoadedRepo(final RepositoryInfo repo, boolean
updateLocal) throws IOException {
Mutex mutex = getRepoMutex(repo);
assert mutex.isWriteAccess();
@@ -523,10 +533,12 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
p.setProperty("User-Agent", "netBeans/" +
System.getProperty("netbeans.buildnumber"));
httpwagon.setHttpHeaders(p);
}
-
+
+ Path tmpStorage =
Files.createTempDirectory("index-extraction");
ResourceFetcher fetcher = createFetcher(wagon, listener,
wagonAuth, wagonProxy);
listener.setFetcher(fetcher);
IndexUpdateRequest iur = new
IndexUpdateRequest(indexingContext, fetcher);
+ iur.setIndexTempDir(tmpStorage.toFile());
NotifyingIndexCreator nic = null;
for (IndexCreator ic : indexingContext.getIndexCreators())
{
@@ -543,23 +555,20 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
Files.deleteIfExists(getRootGroupCacheFile(repo));
remoteIndexUpdater.fetchAndUpdateIndex(iur);
storeGroupCache(repo, indexingContext);
- } catch (IllegalArgumentException ex) {
- // This exception is raised from the maven-indexer.
- // maven-indexer supported two formats:
- // - legacy/zip: zipped lucene (v2.3) index files
- // - gz: maven-indexer specific file format
- // The legacy format is no longer supported and when
- // the indexer encounters old index files it raises
- // this exception
- //
- // Convert to IOException to utilize the existing error
- // handling paths
+ } catch (IOException | AlreadyClosedException |
IllegalArgumentException ex) {
+ // AlreadyClosedException can happen in low storage
situations when lucene is trying to handle IOEs
+ // IllegalArgumentException signals remote archive
format problems
fetchFailed = true;
throw new IOException("Failed to load maven-index for:
" + indexingContext.getRepositoryUrl(), ex);
- } catch (IOException ex) {
- fetchFailed = true;
- throw ex;
} finally {
+ if (fetchFailed) {
+ try{
+ // make sure no big files remain after
extraction failure
+ cleanupDir(tmpStorage);
+ } catch (IOException ex) {
+ LOGGER.log(Level.WARNING, "cleanup failed");
+ }
+ }
if (nic != null) {
nic.end();
}
@@ -585,45 +594,33 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
}
}
}
- } catch (IOException e) {
+ } catch (IOException e) {
if(e.getCause() instanceof ResourceDoesNotExistException) {
fireChange(repo, () -> repo.fireNoIndex());
}
- File tmpFolder = Places.getCacheDirectory();
- // see also issue #250365
- String noSpaceLeftMsg = null;
- if(e.getMessage() != null && e.getMessage().contains("No space
left on device")) {
- noSpaceLeftMsg =
Bundle.MSG_NoSpace(tmpFolder.getAbsolutePath(), repo.getName());
- }
-
- long downloaded = listener != null ? listener.getUnits() * 1024 :
-1;
- long usableSpace = -1;
- try {
- FileStore store = Files.getFileStore(tmpFolder.toPath());
- usableSpace = store.getUsableSpace();
- } catch (IOException ex) {
- Exceptions.printStackTrace(ex);
- }
- LOGGER.log(Level.INFO, "Downloaded maven index file has size {0}
(zipped). The usable space in {1} is {2}.", new Object[]{downloaded, tmpFolder,
usableSpace});
+ Path tmpFolder = Paths.get(System.getProperty("java.io.tmpdir"));
+ Path cacheFolder = Places.getCacheDirectory().toPath();
- // still might be a problem with a too small tmp,
- // let's try to figure out ...
- if(noSpaceLeftMsg == null && downloaded > -1 && downloaded * 15 >
usableSpace) {
- noSpaceLeftMsg =
Bundle.MSG_SeemsNoSpace(tmpFolder.getAbsolutePath(), repo.getName());
- }
+ long freeTmpSpace = getFreeSpaceInMB(tmpFolder);
+ long freeCacheSpace = getFreeSpaceInMB(cacheFolder);
+
+ if (isNoSpaceLeftOnDevice(e) || freeCacheSpace < 1000 ||
freeTmpSpace < 1000) {
+
+ long downloaded = listener != null ? listener.getUnits() *
1024 : -1;
+ LOGGER.log(Level.INFO, "Downloaded maven index file has size
{0} (zipped). The usable space in {1} is {2} and in {3} MB is {4} MB.",
+ new Object[] {downloaded, cacheFolder, freeCacheSpace,
tmpFolder, freeTmpSpace});
+ LOGGER.log(Level.WARNING, "Download/Extraction failed due to
low storage, indexing is now disabled.", e);
+
+ // disable indexing and tell user about it
+ RepositoryPreferences.setIndexRepositories(false);
- if(noSpaceLeftMsg != null) {
- LOGGER.log(Level.INFO, null, e);
IndexingNotificationProvider np =
Lookup.getDefault().lookup(IndexingNotificationProvider.class);
if(np != null) {
- np.notifyError(noSpaceLeftMsg);
- unloadIndexingContext(repo.getId());
- } else {
- throw e;
+ np.notifyError(Bundle.MSG_NoSpace(repo.getName(),
cacheFolder.toString(), freeCacheSpace, tmpFolder.toString(), freeTmpSpace));
}
- } else {
- throw e;
+ unloadIndexingContext(repo.getId());
}
+ throw e;
} catch (Cancellation x) {
throw new IOException("canceled indexing", x);
} catch (ComponentLookupException x) {
@@ -640,6 +637,15 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
}
}
+ private static boolean isNoSpaceLeftOnDevice(Throwable ex) {
+ String msg = ex.getMessage();
+ Throwable cause = ex.getCause();
+ Throwable[] suppressed = ex.getSuppressed();
+ return (msg != null && msg.contains("No space left on device"))
+ || (cause != null && isNoSpaceLeftOnDevice(cause))
+ || (suppressed.length > 0 &&
Stream.of(suppressed).anyMatch(NexusRepositoryIndexerImpl::isNoSpaceLeftOnDevice));
+ }
+
private static boolean isDiag() {
return Boolean.getBoolean("maven.indexing.diag");
}
@@ -1718,4 +1724,28 @@ public class NexusRepositoryIndexerImpl implements
RepositoryIndexerImplementati
}
}
+ private static void cleanupDir(Path path) throws IOException {
+ Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException
exc) throws IOException {
+ Files.deleteIfExists(dir);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
+ Files.deleteIfExists(file);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ private long getFreeSpaceInMB(Path path) {
+ try {
+ return Files.getFileStore(path).getUsableSpace() / (1024 * 1024);
+ } catch (IOException ignore) {
+ return -1;
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists
