hansva commented on code in PR #9617:
URL: https://github.com/apache/hop/pull/9617#discussion_r4653122104
##########
plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java:
##########
@@ -320,22 +347,51 @@ private void activateStagedFolders(Path staging) throws
HopException {
}
String artifactId = staged.getFileName().toString();
List<String> paths = collectRelativePaths(staged);
- activateStaged(artifactId, paths);
- // The files are in place now; a receipt problem must not stop the
other plugins.
+ // A receipt problem must not stop the other plugins from activating.
+ InstallReceipt receipt = null;
+ boolean unreadable = false;
try {
- InstallReceipt receipt = readReceipt(hopHome, artifactId);
- if (receipt != null) {
- receipt.setPendingActivation(false);
- removeObsoleteFiles(receipt);
- writeReceipt(receipt);
- }
+ receipt = readReceipt(hopHome, artifactId);
} catch (HopException e) {
+ unreadable = true;
log.logError(
- "Activated staged plugin "
+ "Could not read the install receipt of staged plugin "
+ artifactId
- + " but could not update its install receipt; files of a
previous version"
- + " may remain",
+ + "; activating it with a new receipt",
e);
+ moveUnreadableReceiptAside(artifactId);
+ }
+ Map<String, SharedFile> sharedFiles =
+ activateStaged(
+ artifactId,
+ receipt == null || StringUtils.isBlank(receipt.getVersion())
+ ? "unknown"
+ : receipt.getVersion(),
+ paths,
+ receipt == null ? Map.of() : receipt.getSharedFiles());
+ if (unreadable) {
+ // Without a receipt the plugin could not be updated or
uninstalled. The version, the
+ // repository and the previous version's files were only in the
unreadable one.
+ receipt = new InstallReceipt();
+ receipt.setArtifactId(artifactId);
+ receipt.setInstalledAt(Instant.now().toString());
+ receipt.setPaths(paths);
+ receipt.setChecksums(checksums(staged, paths));
Review Comment:
The new receipt has no `obsoletePaths`, so the previous version's files stay
next to the new ones. For example, 1.0.0 with `lib/dep-1.0.jar` is installed
and 2.0.0 is staged. After activation both `dep-1.0.jar` and `dep-2.0.jar` are
in the plugin's lib, and the class versions conflict. Could you work out the
leftovers, for example the files under the plugin's top-level folder that the
staged tree doesn't have? Or at least log clearly that both versions may now
load.
##########
plugins/engines/beam/src/main/java/org/apache/hop/beam/gui/HopBeamGuiPlugin.java:
##########
@@ -336,6 +335,21 @@ public static boolean
isNativeProvidedSparkClientVersion(String version) {
&&
SPARK_CLIENT_VERSION_NATIVE_PROVIDED.equalsIgnoreCase(version.trim());
}
+ /**
+ * The jars under the plugins folder. Folders whose name starts with a dot
are skipped: the
+ * marketplace keeps staged downloads and backed-up jars there ({@code
plugins/.staging}, {@code
+ * plugins/.marketplace}), and those must not end up in a fat jar next to
the live ones.
+ */
+ public static Collection<File> findPluginJars(File pluginsDir) {
Review Comment:
Nit: a new public API on `java.io.File`. CLAUDE.md asks for `FileObject` by
default and keeps `File` for exceptional cases. `FileObject`, or at least
`Path`, would fit better, unless the callers here need `File`.
##########
plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginUninstaller.java:
##########
@@ -51,44 +54,72 @@ public void uninstall(String artifactId) throws
HopException {
+ "'. Only plugins installed via the marketplace can be
uninstalled this way.");
}
- // Delete files deepest-first; leave shared lib/core jars (sticky, may be
used by others).
- // Files of a previous version that an update could not remove yet go too,
unless another
- // marketplace artifact has installed the same file since.
- Set<String> paths = new
LinkedHashSet<>(PluginInstaller.nullSafe(receipt.getPaths()));
+ // Leave shared lib/core jars in place (sticky, may be used by others).
Files of a previous
+ // version that an update could not remove yet go too, unless another
marketplace artifact has
+ // installed the same file since. Those are the files that were in use, so
a failed delete is
+ // logged and kept in the receipt instead of stopping the uninstall half
way.
+ Set<String> obsolete = new LinkedHashSet<>();
if (!PluginInstaller.nullSafe(receipt.getObsoletePaths()).isEmpty()) {
Set<String> otherReceiptPaths =
PluginInstaller.pathsOfOtherReceipts(hopHome, artifactId, log);
- for (String obsolete : receipt.getObsoletePaths()) {
- if (StringUtils.isNotBlank(obsolete)
- &&
!otherReceiptPaths.contains(PluginInstaller.normalizeRelative(obsolete))) {
- paths.add(obsolete);
+ for (String path : receipt.getObsoletePaths()) {
+ if (StringUtils.isNotBlank(path)
+ &&
!otherReceiptPaths.contains(PluginInstaller.normalizeRelative(path))) {
+ obsolete.add(path);
}
}
}
Set<Path> dirs = new HashSet<>();
- for (String relative : paths) {
- if (PluginInstaller.isSharedCorePath(relative)) {
- log.logBasic("Leaving shared lib/core path in place: " + relative);
+ List<String> leftovers = new ArrayList<>();
+ for (String relative : obsolete) {
+ try {
+ removePath(relative, dirs);
+ } catch (IOException e) {
+ log.logError(
+ "Could not remove "
+ + relative
+ + " of a previous version of "
+ + artifactId
+ + " (file in use?). Close Hop and uninstall the plugin again
to remove it. "
+ + e.getMessage());
+ leftovers.add(relative);
+ }
+ }
+ for (String relative : PluginInstaller.nullSafe(receipt.getPaths())) {
+ if (obsolete.contains(relative)) {
continue;
}
- Path path = hopHome.resolve(relative);
try {
- if (Files.isRegularFile(path)) {
- Files.deleteIfExists(path);
- Path parent = path.getParent();
- if (parent != null) {
- dirs.add(parent);
- }
- } else if (Files.isDirectory(path)) {
- dirs.add(path);
- }
+ removePath(relative, dirs);
} catch (IOException e) {
- throw new HopException("Failed to remove " + path, e);
+ throw new HopException("Failed to remove " +
hopHome.resolve(relative), e);
}
}
removeEmptyPluginDirectories(hopHome, dirs, log);
+ if (!leftovers.isEmpty()) {
+ // The receipt now lists only what is left, the files and the plugin
folders that hold
+ // them, so the next uninstall, install or apply retries it.
+ List<String> remainingDirs = new ArrayList<>();
+ for (Path dir : dirs) {
+ if (Files.isDirectory(dir)) {
+ remainingDirs.add(hopHome.relativize(dir).toString().replace('\\',
'/'));
+ }
+ }
+ Collections.sort(remainingDirs);
+ receipt.setArtifactId(artifactId);
+ receipt.setPaths(remainingDirs);
Review Comment:
The kept receipt still has `version`, `groupId`, `checksums` and
`sharedFiles`, so the plugin still counts as installed at that version. At the
next start `removeLeftoverObsoleteFiles` deletes the leftover file but keeps
the receipt.
- Example: install 2.0.0 while a 1.0.0 file is locked, uninstall, restart.
`hop marketplace apply` with the plugin at 2.0.0 now finds a receipt with a
matching version and skips the install. The plugin stays missing while the
marketplace says it is installed.
- A same-version reinstall after this reuses the old `sharedFiles`
(`sameVersion` is true), with REPLACED/CREATED and backup pointers from the
earlier install. The new receipt then says this install replaced and backed up
lib/core jars it never touched.
Suggestion: mark the receipt as an unfinished uninstall, either by clearing
version, checksums and sharedFiles or with a flag. Then delete it once
`removeObsoleteFiles` empties its obsolete list.
##########
plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginInstaller.java:
##########
@@ -57,6 +69,12 @@ public class PluginInstaller {
public static final String STAGING_DIR = "plugins/.staging";
public static final String RECEIPTS_DIR = "plugins/.marketplace";
+ /**
+ * Shared {@code lib/core} files an install replaced, under {@code
<artifactId>/<version>/<path>}.
+ * The plugin scanner skips folders named {@code lib}, so the backed-up jars
are not loaded.
+ */
+ public static final String BACKUP_DIR = RECEIPTS_DIR + "/backup";
Review Comment:
Nit: `backupPath` writes `<artifactId>/<version>/<installId>/<path>`, so the
`<installId>` segment is missing here.
##########
plugins/misc/marketplace/src/main/java/org/apache/hop/marketplace/install/PluginUninstaller.java:
##########
@@ -51,44 +54,72 @@ public void uninstall(String artifactId) throws
HopException {
+ "'. Only plugins installed via the marketplace can be
uninstalled this way.");
}
- // Delete files deepest-first; leave shared lib/core jars (sticky, may be
used by others).
- // Files of a previous version that an update could not remove yet go too,
unless another
- // marketplace artifact has installed the same file since.
- Set<String> paths = new
LinkedHashSet<>(PluginInstaller.nullSafe(receipt.getPaths()));
+ // Leave shared lib/core jars in place (sticky, may be used by others).
Files of a previous
+ // version that an update could not remove yet go too, unless another
marketplace artifact has
+ // installed the same file since. Those are the files that were in use, so
a failed delete is
+ // logged and kept in the receipt instead of stopping the uninstall half
way.
+ Set<String> obsolete = new LinkedHashSet<>();
if (!PluginInstaller.nullSafe(receipt.getObsoletePaths()).isEmpty()) {
Set<String> otherReceiptPaths =
PluginInstaller.pathsOfOtherReceipts(hopHome, artifactId, log);
- for (String obsolete : receipt.getObsoletePaths()) {
- if (StringUtils.isNotBlank(obsolete)
- &&
!otherReceiptPaths.contains(PluginInstaller.normalizeRelative(obsolete))) {
- paths.add(obsolete);
+ for (String path : receipt.getObsoletePaths()) {
+ if (StringUtils.isNotBlank(path)
+ &&
!otherReceiptPaths.contains(PluginInstaller.normalizeRelative(path))) {
+ obsolete.add(path);
}
}
}
Set<Path> dirs = new HashSet<>();
- for (String relative : paths) {
- if (PluginInstaller.isSharedCorePath(relative)) {
- log.logBasic("Leaving shared lib/core path in place: " + relative);
+ List<String> leftovers = new ArrayList<>();
+ for (String relative : obsolete) {
+ try {
+ removePath(relative, dirs);
+ } catch (IOException e) {
+ log.logError(
+ "Could not remove "
+ + relative
+ + " of a previous version of "
+ + artifactId
+ + " (file in use?). Close Hop and uninstall the plugin again
to remove it. "
+ + e.getMessage());
+ leftovers.add(relative);
+ }
+ }
+ for (String relative : PluginInstaller.nullSafe(receipt.getPaths())) {
+ if (obsolete.contains(relative)) {
continue;
}
- Path path = hopHome.resolve(relative);
try {
- if (Files.isRegularFile(path)) {
- Files.deleteIfExists(path);
- Path parent = path.getParent();
- if (parent != null) {
- dirs.add(parent);
- }
- } else if (Files.isDirectory(path)) {
- dirs.add(path);
- }
+ removePath(relative, dirs);
} catch (IOException e) {
- throw new HopException("Failed to remove " + path, e);
+ throw new HopException("Failed to remove " +
hopHome.resolve(relative), e);
}
}
removeEmptyPluginDirectories(hopHome, dirs, log);
+ if (!leftovers.isEmpty()) {
+ // The receipt now lists only what is left, the files and the plugin
folders that hold
+ // them, so the next uninstall, install or apply retries it.
+ List<String> remainingDirs = new ArrayList<>();
+ for (Path dir : dirs) {
+ if (Files.isDirectory(dir)) {
+ remainingDirs.add(hopHome.relativize(dir).toString().replace('\\',
'/'));
+ }
+ }
+ Collections.sort(remainingDirs);
+ receipt.setArtifactId(artifactId);
+ receipt.setPaths(remainingDirs);
+ receipt.setObsoletePaths(leftovers);
+ PluginInstaller.writeReceipt(hopHome, receipt);
+ log.logError(
+ "Uninstalled "
+ + artifactId
+ + " except for files of a previous version that could not be
removed: "
+ + String.join(", ", leftovers));
+ return;
+ }
+
Path receiptFile =
hopHome.resolve(PluginInstaller.RECEIPTS_DIR).resolve(artifactId + ".json");
try {
Files.deleteIfExists(receiptFile);
Review Comment:
This one was already there before this PR: uninstalling a plugin that is
still staged (activateImmediately=false, no restart yet) leaves
`plugins/.staging/<artifactId>`. At the next start `activateStagedFolders`
finds no receipt (`receipt == null`, not unreadable), so it copies the files
into plugins/ without writing a receipt. The plugin is back and `uninstall`
then fails with 'No marketplace install receipt'. Deleting the staging folder
in `uninstall` would fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]