implicitly wrap yaml in bundles when uploading to catalog creates a new bundle when given yaml in catalog; bundle name and version now recommended in the BOM. scan-java option tweaked but in a way that is consistent with the past and sensible in the new world. auto-wrapped bundles are identified with a header so we can simplify their handling in some cases (scanning, search paths).
some things clumsy and need fixed: * uploading a different BOM (or bundle) at the same version says "ignoring because identical"; osgi identicality check should look at contents * auto-wrapped bundles are added to the library search path (no need for this as the bundles are empty) * failed installations keep the bundle installed, blocking subsequent installs; probably don't install unless forced? * warn if different version declared in BOM some things clumsy but we could live with: * containing bundles are added as libraries by munging the yaml; now we have a record to that so can do a bit better than munge yaml * if bundle has no name, a random one is chosen (probably deprecated this behaviour); if you re-submit we don't know it's the same bundle so we don't detect it's a bundle update; this means: * uploading the exact same non-snapshot BOM twice will fail the second time saying the items are different (because containing bundle is different) * uploading an updated item in a snapshot BOM will correctly replace, but the old bundle will still be around (just masked; though we can fix that) Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/d7975d14 Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/d7975d14 Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/d7975d14 Branch: refs/heads/master Commit: d7975d1414269f0f14617d3987170eba729c9c7d Parents: 97b50e4 Author: Alex Heneveld <[email protected]> Authored: Fri Jun 23 10:53:51 2017 +0100 Committer: Alex Heneveld <[email protected]> Committed: Tue Jun 27 11:26:20 2017 +0100 ---------------------------------------------------------------------- .../brooklyn/api/typereg/OsgiBundleWithUrl.java | 5 + .../catalog/CatalogOsgiLibraryTest.java | 11 +- .../CatalogOsgiVersionMoreEntityTest.java | 14 +- .../catalog/CatalogOsgiYamlEntityTest.java | 12 +- .../catalog/SpecParameterParsingOsgiTest.java | 26 ++- .../catalog/internal/BasicBrooklynCatalog.java | 179 ++++++++++++++++--- .../core/catalog/internal/CatalogBundleDto.java | 9 +- .../catalog/internal/CatalogBundleLoader.java | 8 +- .../core/mgmt/ha/OsgiArchiveInstaller.java | 10 +- .../mgmt/ha/OsgiBundleInstallationResult.java | 16 +- .../core/typereg/BasicManagedBundle.java | 2 - .../core/typereg/BasicOsgiBundleWithUrl.java | 7 + .../main/resources/brooklyn/default.catalog.bom | 6 +- .../brooklyn/util/osgi/VersionedName.java | 7 + 14 files changed, 247 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/api/src/main/java/org/apache/brooklyn/api/typereg/OsgiBundleWithUrl.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/brooklyn/api/typereg/OsgiBundleWithUrl.java b/api/src/main/java/org/apache/brooklyn/api/typereg/OsgiBundleWithUrl.java index 970fc68..55b30fc 100644 --- a/api/src/main/java/org/apache/brooklyn/api/typereg/OsgiBundleWithUrl.java +++ b/api/src/main/java/org/apache/brooklyn/api/typereg/OsgiBundleWithUrl.java @@ -18,6 +18,8 @@ */ package org.apache.brooklyn.api.typereg; +import org.apache.brooklyn.util.osgi.VersionedName; + import com.google.common.annotations.Beta; @Beta @@ -36,5 +38,8 @@ public interface OsgiBundleWithUrl { /** @return true if we have a name and version for this bundle; * false if not, e.g. if we only know the URL and we haven't loaded it yet */ public boolean isNameResolved(); + + /** @return the {@link VersionedName} for this bundle, or null if not available */ + public VersionedName getVersionedName(); } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiLibraryTest.java ---------------------------------------------------------------------- diff --git a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiLibraryTest.java b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiLibraryTest.java index aebd80d..87a459e 100644 --- a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiLibraryTest.java +++ b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiLibraryTest.java @@ -25,6 +25,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; +import java.util.Objects; import org.apache.brooklyn.api.catalog.CatalogItem; import org.apache.brooklyn.api.catalog.CatalogItem.CatalogBundle; @@ -58,7 +59,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.io.BaseEncoding; public class CatalogOsgiLibraryTest extends AbstractYamlTest { @@ -421,7 +421,12 @@ public class CatalogOsgiLibraryTest extends AbstractYamlTest { } protected void assertCatalogLibraryUrl(CatalogItem<?,?> item, String expectedUrl) { - CatalogBundle library = Iterables.getOnlyElement(item.getLibraries()); - assertEquals(library.getUrl(), expectedUrl); + for (CatalogBundle b: item.getLibraries()) { + if (Objects.equals(b.getUrl(), expectedUrl)) { + return; + } + } + Assert.fail("No library found with URL "+expectedUrl); } + } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiVersionMoreEntityTest.java ---------------------------------------------------------------------- diff --git a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiVersionMoreEntityTest.java b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiVersionMoreEntityTest.java index 8961c6f..0813d80 100644 --- a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiVersionMoreEntityTest.java +++ b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiVersionMoreEntityTest.java @@ -28,12 +28,14 @@ import org.apache.brooklyn.api.location.Location; import org.apache.brooklyn.api.location.LocationSpec; import org.apache.brooklyn.api.policy.Policy; import org.apache.brooklyn.api.policy.PolicySpec; +import org.apache.brooklyn.api.sensor.Enricher; +import org.apache.brooklyn.api.sensor.EnricherSpec; import org.apache.brooklyn.api.typereg.BrooklynTypeRegistry; import org.apache.brooklyn.api.typereg.ManagedBundle; import org.apache.brooklyn.api.typereg.RegisteredType; import org.apache.brooklyn.camp.brooklyn.AbstractYamlTest; import org.apache.brooklyn.camp.brooklyn.spi.creation.BrooklynEntityMatcher; -import org.apache.brooklyn.core.entity.Entities; +import org.apache.brooklyn.core.catalog.internal.BasicBrooklynCatalog; import org.apache.brooklyn.core.mgmt.ha.OsgiBundleInstallationResult; import org.apache.brooklyn.core.mgmt.internal.ManagementContextInternal; import org.apache.brooklyn.core.mgmt.osgi.OsgiVersionMoreEntityTest; @@ -75,8 +77,8 @@ public class CatalogOsgiVersionMoreEntityTest extends AbstractYamlTest implement // bundle installed Map<String, ManagedBundle> bundles = ((ManagementContextInternal)mgmt()).getOsgiManager().get().getManagedBundles(); - Asserts.assertSize(bundles.keySet(), 1); - Assert.assertEquals(br.getMetadata().getId(), Iterables.getOnlyElement( bundles.keySet() )); + Asserts.assertSize(bundles.keySet(), 1 + (BasicBrooklynCatalog.AUTO_WRAP_CATALOG_YAML_AS_BUNDLE ? 1 : 0)); + Assert.assertTrue(bundles.keySet().contains( br.getMetadata().getId() )); // types installed RegisteredType t = mgmt().getTypeRegistry().get(BROOKLYN_TEST_MORE_ENTITIES_MORE_ENTITY); @@ -96,7 +98,7 @@ public class CatalogOsgiVersionMoreEntityTest extends AbstractYamlTest implement Assert.assertNotNull(item); Assert.assertEquals(item.getVersion(), "1.0"); Assert.assertTrue(RegisteredTypePredicates.IS_ENTITY.apply(item)); - Assert.assertEquals(item.getLibraries().size(), 1); + Assert.assertEquals(item.getLibraries().size(), 1 + (BasicBrooklynCatalog.AUTO_WRAP_CATALOG_YAML_AS_BUNDLE ? 1 : 0)); Entity app = createAndStartApplication("services: [ { type: 'more-entity:1.0' } ]"); Entity moreEntity = Iterables.getOnlyElement(app.getChildren()); @@ -220,7 +222,7 @@ public class CatalogOsgiVersionMoreEntityTest extends AbstractYamlTest implement // this refers to the java item, where the libraries are defined item = mgmt().getTypeRegistry().get("org.apache.brooklyn.test.osgi.entities.more.MoreEntity"); Assert.assertEquals(item.getVersion(), "2.0.test_java"); - Assert.assertEquals(item.getLibraries().size(), 2); + Assert.assertEquals(item.getLibraries().size(), 2 + (BasicBrooklynCatalog.AUTO_WRAP_CATALOG_YAML_AS_BUNDLE ? 1 : 0)); Entity app = createAndStartApplication("services: [ { type: 'more-entity:2.0.test' } ]"); Entity moreEntity = Iterables.getOnlyElement(app.getChildren()); @@ -247,7 +249,7 @@ public class CatalogOsgiVersionMoreEntityTest extends AbstractYamlTest implement // this refers to the java item, where the libraries are defined item = mgmt().getTypeRegistry().get("org.apache.brooklyn.test.osgi.entities.more.MorePolicy"); Assert.assertEquals(item.getVersion(), "2.0.test_java"); - Assert.assertEquals(item.getLibraries().size(), 2); + Assert.assertEquals(item.getLibraries().size(), 2 + (BasicBrooklynCatalog.AUTO_WRAP_CATALOG_YAML_AS_BUNDLE ? 1 : 0)); Entity app = createAndStartApplication( "services: ", http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiYamlEntityTest.java ---------------------------------------------------------------------- diff --git a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiYamlEntityTest.java b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiYamlEntityTest.java index fe86ed0..a1bd800 100644 --- a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiYamlEntityTest.java +++ b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/CatalogOsgiYamlEntityTest.java @@ -243,9 +243,9 @@ public class CatalogOsgiYamlEntityTest extends AbstractYamlTest { " - name: io.brooklyn.brooklyn-test-osgi-entities", " item:", " type: " + SIMPLE_ENTITY_TYPE); - fail(); - } catch (NullPointerException e) { - Assert.assertEquals(e.getMessage(), "both name and version are required"); + Asserts.shouldHaveFailedPreviously(); + } catch (Exception e) { + Asserts.expectedFailureContainsIgnoreCase(e, "both name and version are required"); } try { addCatalogItems( @@ -257,9 +257,9 @@ public class CatalogOsgiYamlEntityTest extends AbstractYamlTest { " - version: " + OsgiStandaloneTest.BROOKLYN_TEST_OSGI_ENTITIES_VERSION, " item:", " type: " + SIMPLE_ENTITY_TYPE); - fail(); - } catch (NullPointerException e) { - Assert.assertEquals(e.getMessage(), "both name and version are required"); + Asserts.shouldHaveFailedPreviously(); + } catch (Exception e) { + Asserts.expectedFailureContainsIgnoreCase(e, "both name and version are required"); } } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/SpecParameterParsingOsgiTest.java ---------------------------------------------------------------------- diff --git a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/SpecParameterParsingOsgiTest.java b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/SpecParameterParsingOsgiTest.java index 814aed7..557ebc5 100644 --- a/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/SpecParameterParsingOsgiTest.java +++ b/camp/camp-brooklyn/src/test/java/org/apache/brooklyn/camp/brooklyn/catalog/SpecParameterParsingOsgiTest.java @@ -29,6 +29,7 @@ import org.apache.brooklyn.api.internal.AbstractBrooklynObjectSpec; import org.apache.brooklyn.api.objs.SpecParameter; import org.apache.brooklyn.api.typereg.RegisteredType; import org.apache.brooklyn.camp.brooklyn.AbstractYamlTest; +import org.apache.brooklyn.core.BrooklynVersion; import org.apache.brooklyn.core.config.ConfigKeys; import org.apache.brooklyn.core.entity.AbstractEntity; import org.apache.brooklyn.core.objs.BasicSpecParameter; @@ -39,6 +40,7 @@ import org.testng.Assert; import org.testng.annotations.Test; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; public class SpecParameterParsingOsgiTest extends AbstractYamlTest { @@ -82,16 +84,34 @@ public class SpecParameterParsingOsgiTest extends AbstractYamlTest { TestResourceUnavailableException.throwIfResourceUnavailable(getClass(), OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_V2_PATH); addCatalogItems("brooklyn.catalog:", + " bundle: test-items", + " version: 2.0-test_java", " items:", " - scanJavaAnnotations: true", - " version: 2.0.test_java", + " item:", + " id: here-item", + " type: "+OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_MORE_ENTITY, " libraries:", " - classpath://" + OsgiTestResources.BROOKLYN_TEST_OSGI_ENTITIES_PATH, " - classpath://" + OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_V2_PATH); + RegisteredType hereItem = mgmt().getTypeRegistry().get("here-item"); + assertEquals(hereItem.getVersion(), "2.0-test_java"); + assertEquals(hereItem.getLibraries().size(), 3); + assertEquals(hereItem.getContainingBundle(), "test-items:2.0-test_java"); + RegisteredType item = mgmt().getTypeRegistry().get(OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_MORE_ENTITY); - assertEquals(item.getVersion(), "2.0.test_java"); - assertEquals(item.getLibraries().size(), 2); + // since 0.12.0 items now installed with version from bundle, not inherited from the version here + assertEquals(item.getVersion(), BrooklynVersion.get()); + // since 0.12.0 library bundles (correctly) don't inherit libraries from caller + assertEquals(item.getLibraries().size(), 1); + assertEquals(Iterables.getOnlyElement(item.getLibraries()).getVersionedName().toString(), + OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_SYMBOLIC_NAME_FULL+":"+"0.2.0"); + + assertEquals(item.getContainingBundle(), OsgiTestResources.BROOKLYN_TEST_MORE_ENTITIES_SYMBOLIC_NAME_FULL+":"+"0.2.0"); + + // TODO assertions above should be in separate test + AbstractBrooklynObjectSpec<?,?> spec = createSpec(item); List<SpecParameter<?>> inputs = spec.getParameters(); if (inputs.isEmpty()) Assert.fail("no inputs (if you're in the IDE, mvn clean install may need to be run to rebuild osgi test JARs)"); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java index a787789..d003ce9 100644 --- a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/BasicBrooklynCatalog.java @@ -21,12 +21,20 @@ package org.apache.brooklyn.core.catalog.internal; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.jar.Attributes; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; import javax.annotation.Nullable; @@ -40,9 +48,13 @@ import org.apache.brooklyn.api.location.LocationSpec; import org.apache.brooklyn.api.mgmt.ManagementContext; import org.apache.brooklyn.api.mgmt.classloading.BrooklynClassLoadingContext; import org.apache.brooklyn.api.typereg.ManagedBundle; +import org.apache.brooklyn.api.typereg.OsgiBundleWithUrl; import org.apache.brooklyn.core.catalog.CatalogPredicates; import org.apache.brooklyn.core.catalog.internal.CatalogClasspathDo.CatalogScanningModes; import org.apache.brooklyn.core.location.BasicLocationRegistry; +import org.apache.brooklyn.core.mgmt.ha.OsgiBundleInstallationResult; +import org.apache.brooklyn.core.mgmt.ha.OsgiBundleInstallationResult.ResultCode; +import org.apache.brooklyn.core.mgmt.ha.OsgiManager; import org.apache.brooklyn.core.mgmt.internal.CampYamlParser; import org.apache.brooklyn.core.mgmt.internal.ManagementContextInternal; import org.apache.brooklyn.core.typereg.BrooklynTypePlanTransformer; @@ -52,6 +64,7 @@ import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.collections.MutableSet; import org.apache.brooklyn.util.core.ResourceUtils; import org.apache.brooklyn.util.core.flags.TypeCoercions; +import org.apache.brooklyn.util.core.osgi.BundleMaker; import org.apache.brooklyn.util.core.task.Tasks; import org.apache.brooklyn.util.exceptions.Exceptions; import org.apache.brooklyn.util.exceptions.UserFacingException; @@ -60,15 +73,18 @@ import org.apache.brooklyn.util.javalang.AggregateClassLoader; import org.apache.brooklyn.util.javalang.JavaClassNames; import org.apache.brooklyn.util.javalang.LoadedClassLoader; import org.apache.brooklyn.util.osgi.VersionedName; +import org.apache.brooklyn.util.text.Identifiers; import org.apache.brooklyn.util.text.Strings; import org.apache.brooklyn.util.time.Duration; import org.apache.brooklyn.util.time.Time; import org.apache.brooklyn.util.yaml.Yamls; import org.apache.brooklyn.util.yaml.Yamls.YamlExtract; +import org.osgi.framework.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; @@ -86,8 +102,19 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { public static final String POLICIES_KEY = "brooklyn.policies"; public static final String ENRICHERS_KEY = "brooklyn.enrichers"; public static final String LOCATIONS_KEY = "brooklyn.locations"; - public static final String NO_VERSION = "0.0.0.SNAPSHOT"; - + public static final String NO_VERSION = "0.0.0-SNAPSHOT"; + + public static final String CATALOG_BOM = "catalog.bom"; + // should always be 1.0; see bottom of + // http://www.eclipse.org/virgo/documentation/virgo-documentation-3.7.0.M01/docs/virgo-user-guide/html/ch02s02.html + // (some things talk of 2.0, but haven't investigated that) + public static final String OSGI_MANIFEST_VERSION_VALUE = "1.0"; + + /** Header on bundle indicating it is a wrapped BOM with no other resources */ + public static final String BROOKLYN_WRAPPED_BOM_BUNDLE = "Brooklyn-Wrapped-BOM"; + @VisibleForTesting + public static final boolean AUTO_WRAP_CATALOG_YAML_AS_BUNDLE = true; + private static final Logger log = LoggerFactory.getLogger(BasicBrooklynCatalog.class); public static class BrooklynLoaderTracker { @@ -415,9 +442,9 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { return (Maybe) getFirstAs(map, Map.class, firstKey, otherKeys); } - private List<CatalogItemDtoAbstract<?,?>> collectCatalogItems(String yaml) { + private List<CatalogItemDtoAbstract<?,?>> collectCatalogItems(String yaml, ManagedBundle containingBundle) { List<CatalogItemDtoAbstract<?, ?>> result = MutableList.of(); - collectCatalogItems(yaml, result, ImmutableMap.of()); + collectCatalogItems(yaml, containingBundle, result, ImmutableMap.of()); return result; } @@ -449,7 +476,7 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { return new VersionedName(bundle, version); } - private void collectCatalogItems(String yaml, List<CatalogItemDtoAbstract<?, ?>> result, Map<?, ?> parentMeta) { + private void collectCatalogItems(String yaml, ManagedBundle containingBundle, List<CatalogItemDtoAbstract<?, ?>> result, Map<?, ?> parentMeta) { Map<?,?> itemDef = Yamls.getAs(Yamls.parseAll(yaml), Map.class); Map<?,?> catalogMetadata = getFirstAsMap(itemDef, "brooklyn.catalog").orNull(); if (catalogMetadata==null) @@ -457,7 +484,7 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { catalogMetadata = MutableMap.copyOf(catalogMetadata); collectCatalogItems(Yamls.getTextOfYamlAtPath(yaml, "brooklyn.catalog").getMatchedYamlTextOrWarn(), - catalogMetadata, result, parentMeta); + containingBundle, catalogMetadata, result, parentMeta, 0); itemDef.remove("brooklyn.catalog"); catalogMetadata.remove("item"); @@ -472,12 +499,12 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { if (rootItemYaml.startsWith(match)) rootItemYaml = Strings.removeFromStart(rootItemYaml, match); else rootItemYaml = Strings.replaceAllNonRegex(rootItemYaml, "\n"+match, ""); } - collectCatalogItems("item:\n"+makeAsIndentedObject(rootItemYaml), rootItem, result, catalogMetadata); + collectCatalogItems("item:\n"+makeAsIndentedObject(rootItemYaml), containingBundle, rootItem, result, catalogMetadata, 1); } } @SuppressWarnings("unchecked") - private void collectCatalogItems(String sourceYaml, Map<?,?> itemMetadata, List<CatalogItemDtoAbstract<?, ?>> result, Map<?,?> parentMetadata) { + private void collectCatalogItems(String sourceYaml, ManagedBundle containingBundle, Map<?,?> itemMetadata, List<CatalogItemDtoAbstract<?, ?>> result, Map<?,?> parentMetadata, int depth) { if (sourceYaml==null) sourceYaml = new Yaml().dump(itemMetadata); @@ -528,13 +555,35 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { if (scanJavaAnnotations==null || !scanJavaAnnotations) { // don't scan } else { - // scan for annotations: if libraries here, scan them; if inherited libraries error; else scan classpath - if (!libraryBundlesNew.isEmpty()) { - result.addAll(scanAnnotationsFromBundles(mgmt, libraryBundlesNew, catalogMetadata)); - } else if (libraryBundles.isEmpty()) { - result.addAll(scanAnnotationsFromLocal(mgmt, catalogMetadata)); + if (isNoBundleOrSimpleWrappingBundle(containingBundle)) { + // BOMs wrapped in JARs, or without JARs, have special treatment + if (isLibrariesMoreThanJustContainingBundle(libraryBundlesNew, containingBundle)) { + // legacy mode, since 0.12.0, scan libraries referenced in a legacy non-bundle BOM + log.warn("Deprecated use of scanJavaAnnotations to scan other libraries ("+libraryBundlesNew+"); libraries should declare they scan themselves"); + result.addAll(scanAnnotationsFromBundles(mgmt, libraryBundlesNew, catalogMetadata)); + } else if (!isLibrariesMoreThanJustContainingBundle(libraryBundles, containingBundle)) { + // for default catalog, no libraries declared, we want to scan local classpath + // bundle should be named "brooklyn-default-catalog" + if (containingBundle!=null && !containingBundle.getSymbolicName().contains("brooklyn-default-catalog")) { + // a user uplaoded a BOM trying to tell us to do a local java scan; previously supported but becoming unsupported + log.warn("Deprecated use of scanJavaAnnotations in non-Java BOM outwith the default catalog setup"); + } else if (depth>0) { + // since 0.12.0, require this to be right next to where libraries are defined, or at root + log.warn("Deprecated use of scanJavaAnnotations declared in item; should be declared at the top level of the BOM"); + } + result.addAll(scanAnnotationsFromLocal(mgmt, catalogMetadata)); + } else { + throw new IllegalStateException("Cannot scan for Java catalog items when libraries declared on an ancestor; scanJavaAnnotations should be specified alongside brooklyn.libraries (or ideally those libraries should specify to scan)"); + } } else { - throw new IllegalStateException("Cannot scan catalog node no local bundles, and with inherited bundles we will not scan the classpath"); + if (depth>0) { + // since 0.12.0, require this to be right next to where libraries are defined, or at root + log.warn("Deprecated use of scanJavaAnnotations declared in item; should be declared at the top level of the BOM"); + } + // normal JAR install, only scan that bundle (the one containing the catalog.bom) + result.addAll(scanAnnotationsFromBundles(mgmt, MutableList.of(containingBundle), catalogMetadata)); + // TODO above (scanning a ZIP uploaded) won't work yet because scan routines need a URL + // TODO are libraries installed properly, such that they are now managed and their catalog.bom's are scanned ? } } @@ -546,18 +595,18 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { int count = 0; for (Object ii: checkType(items, "items", List.class)) { if (ii instanceof String) { - collectUrlReferencedCatalogItems((String) ii, result, catalogMetadata); + collectUrlReferencedCatalogItems((String) ii, containingBundle, result, catalogMetadata); } else { Map<?,?> i = checkType(ii, "entry in items list", Map.class); collectCatalogItems(Yamls.getTextOfYamlAtPath(sourceYaml, "items", count).getMatchedYamlTextOrWarn(), - i, result, catalogMetadata); + containingBundle, i, result, catalogMetadata, depth+1); } count++; } } if (url != null) { - collectUrlReferencedCatalogItems(checkType(url, "include in catalog meta", String.class), result, catalogMetadata); + collectUrlReferencedCatalogItems(checkType(url, "include in catalog meta", String.class), containingBundle, result, catalogMetadata); } if (item==null) return; @@ -594,7 +643,7 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { itemType = planInterpreter.getCatalogItemType(); Map<?, ?> itemAsMap = planInterpreter.getItem(); // the "plan yaml" includes the services: ... or brooklyn.policies: ... outer key, - // as opposed to the rawer { type: xxx } map without that outer key which is valid as item input + // as opposed to the rawer { type: foo } map without that outer key which is valid as item input // TODO this plan yaml is needed for subsequent reconstruction; would be nicer if it weren't! // if symname not set, infer from: id, then name, then item id, then item name @@ -738,7 +787,21 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { result.add(dto); } - private void collectUrlReferencedCatalogItems(String url, List<CatalogItemDtoAbstract<?, ?>> result, Map<Object, Object> parentMeta) { + private boolean isLibrariesMoreThanJustContainingBundle(Collection<CatalogBundle> library, ManagedBundle containingBundle) { + if (library==null) return false; + if (containingBundle==null) return !library.isEmpty(); + if (library.size()>1) return true; + CatalogBundle li = Iterables.getOnlyElement(library); + return !containingBundle.getVersionedName().equalsOsgi(li.getVersionedName()); + } + + private boolean isNoBundleOrSimpleWrappingBundle(ManagedBundle b) { + if (b==null) return true; + String wrapped = ((ManagementContextInternal)mgmt).getOsgiManager().get().findBundle(b).get().getHeaders().get(BROOKLYN_WRAPPED_BOM_BUNDLE); + return wrapped!=null && wrapped.equalsIgnoreCase("true"); + } + + private void collectUrlReferencedCatalogItems(String url, ManagedBundle containingBundle, List<CatalogItemDtoAbstract<?, ?>> result, Map<Object, Object> parentMeta) { @SuppressWarnings("unchecked") List<?> parentLibrariesRaw = MutableList.copyOf(getFirstAs(parentMeta, List.class, "brooklyn.libraries", "libraries").orNull()); Collection<CatalogBundle> parentLibraries = CatalogItemDtoAbstract.parseLibraries(parentLibrariesRaw); @@ -750,7 +813,7 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { Exceptions.propagateIfFatal(e); throw new IllegalStateException("Remote catalog url " + url + " can't be fetched.", e); } - collectCatalogItems(yaml, result, parentMeta); + collectCatalogItems(yaml, containingBundle, result, parentMeta); } @SuppressWarnings("unchecked") @@ -774,14 +837,16 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { return scanAnnotationsInternal(mgmt, new CatalogDo(dto), catalogMetadata); } - private Collection<CatalogItemDtoAbstract<?, ?>> scanAnnotationsFromBundles(ManagementContext mgmt, Collection<CatalogBundle> libraries, Map<?, ?> catalogMetadata) { + private Collection<CatalogItemDtoAbstract<?, ?>> scanAnnotationsFromBundles(ManagementContext mgmt, Collection<? extends OsgiBundleWithUrl> libraries, Map<?, ?> catalogMetadata) { CatalogDto dto = CatalogDto.newNamedInstance("Bundles Scanned Catalog", "All annotated Brooklyn entities detected in bundles", "scanning-bundles-classpath-"+libraries.hashCode()); List<String> urls = MutableList.of(); - for (CatalogBundle b: libraries) { + for (OsgiBundleWithUrl b: libraries) { // TODO currently does not support pre-installed bundles identified by name:version // (ie where URL not supplied) if (Strings.isNonBlank(b.getUrl())) { urls.add(b.getUrl()); + } else { + log.warn("scanJavaAnnotations does not apply to pre-installed bundles; skipping "+b); } } @@ -1060,28 +1125,84 @@ public class BasicBrooklynCatalog implements BrooklynCatalog { @Override public List<? extends CatalogItem<?,?>> addItems(String yaml) { - return addItems(yaml, null); + return addItems(yaml, false); } @Override - public List<? extends CatalogItem<?, ?>> addItems(String yaml, ManagedBundle bundle) { - return addItems(yaml, bundle, false); - } - - @Override public CatalogItem<?,?> addItem(String yaml, boolean forceUpdate) { return Iterables.getOnlyElement(addItems(yaml, forceUpdate)); } @Override public List<? extends CatalogItem<?,?>> addItems(String yaml, boolean forceUpdate) { + Maybe<OsgiManager> osgiManager = ((ManagementContextInternal)mgmt).getOsgiManager(); + if (osgiManager.isPresent() && AUTO_WRAP_CATALOG_YAML_AS_BUNDLE) { + // TODO wrap in a bundle to be managed; need to get bundle and version from yaml + Map<?, ?> cm = BasicBrooklynCatalog.getCatalogMetadata(yaml); + VersionedName vn = BasicBrooklynCatalog.getVersionedName( cm, false ); + if (vn==null) { + // for better legacy compatibiity, if id specified at root use that + String id = (String) cm.get("id"); + if (Strings.isNonBlank(id)) { + vn = VersionedName.fromString(id); + } + vn = new VersionedName(vn!=null && Strings.isNonBlank(vn.getSymbolicName()) ? vn.getSymbolicName() : "brooklyn-catalog-bom-"+Identifiers.makeRandomId(8), + vn!=null && vn.getVersionString()!=null ? vn.getVersionString() : getFirstAs(cm, String.class, "version").or(NO_VERSION)); + } + Manifest mf = new Manifest(); + mf.getMainAttributes().putValue(Constants.BUNDLE_SYMBOLICNAME, vn.getSymbolicName()); + mf.getMainAttributes().putValue(Constants.BUNDLE_VERSION, vn.getOsgiVersionString() ); + mf.getMainAttributes().putValue(Attributes.Name.MANIFEST_VERSION.toString(), OSGI_MANIFEST_VERSION_VALUE); + mf.getMainAttributes().putValue(BROOKLYN_WRAPPED_BOM_BUNDLE, Boolean.TRUE.toString()); + + BundleMaker bm = new BundleMaker(mgmt); + File bf = bm.createTempBundle(vn.getSymbolicName(), mf, MutableMap.of( + new ZipEntry(CATALOG_BOM), (InputStream) new ByteArrayInputStream(yaml.getBytes())) ); + + OsgiBundleInstallationResult result = null; + try { + result = osgiManager.get().install(null, new FileInputStream(bf), true, true, forceUpdate).get(); + } catch (FileNotFoundException e) { + throw Exceptions.propagate(e); + } + bf.delete(); + if (result.getCode().isError() || result.getCode()==ResultCode.IGNORING_BUNDLE_AREADY_INSTALLED) { + // if we're wrapping YAML then we don't allow equivalent YAML to be pasted + // TODO remove this once we have better bundle equivalence checks + throw new IllegalStateException(result.getMessage()); + } + return toItems(result.getCatalogItemsInstalled()); + + // TODO check if we've overridden all items pertaining to an older anonymous catalog.bom bundle + // we could remove references to that anonymous bundle; + // without this currently we leak bundles as bom's are replaced + // (because we persist each item as well as the bundle, and we use the item XML on rebind, + // rather than rereading the catalog.bom from the bundle, there isn't currently a risk of loading + // any of those overwritten items; however probably wise in future to require a bundle ID) + } + // fallback to non-OSGi for tests and other environments return addItems(yaml, null, forceUpdate); } + @SuppressWarnings("deprecation") + private List<CatalogItem<?,?>> toItems(Iterable<String> itemIds) { + List<CatalogItem<?,?>> result = MutableList.of(); + for (String id: itemIds) { + // TODO prefer to use RegisteredType, but that's an API change here + result.add(CatalogUtils.getCatalogItemOptionalVersion(mgmt, id)); + } + return result; + } + + @Override + public List<? extends CatalogItem<?, ?>> addItems(String yaml, ManagedBundle bundle) { + return addItems(yaml, bundle, false); + } + private List<? extends CatalogItem<?,?>> addItems(String yaml, ManagedBundle bundle, boolean forceUpdate) { log.debug("Adding manual catalog item to "+mgmt+": "+yaml); checkNotNull(yaml, "yaml"); - List<CatalogItemDtoAbstract<?, ?>> result = collectCatalogItems(yaml); + List<CatalogItemDtoAbstract<?, ?>> result = collectCatalogItems(yaml, bundle); // do this at the end for atomic updates; if there are intra-yaml references, we handle them specially for (CatalogItemDtoAbstract<?, ?> item: result) { http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleDto.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleDto.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleDto.java index dbbfc76..2006d5b 100644 --- a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleDto.java +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleDto.java @@ -23,6 +23,7 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import org.apache.brooklyn.api.catalog.CatalogItem.CatalogBundle; +import org.apache.brooklyn.util.osgi.VersionedName; import org.apache.brooklyn.util.text.BrooklynVersionSyntax; public class CatalogBundleDto implements CatalogBundle { @@ -67,7 +68,13 @@ public class CatalogBundleDto implements CatalogBundle { public String getOsgiVersionString() { return version==null ? version : BrooklynVersionSyntax.toValidOsgiVersion(version); } - + + @Override + public VersionedName getVersionedName() { + if (!isNameResolved()) return null; + return new VersionedName(getSymbolicName(), getSuppliedVersionString()); + } + @Override public String getUrl() { return url; http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleLoader.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleLoader.java index 4bf8824..8d3179d 100644 --- a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleLoader.java +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogBundleLoader.java @@ -121,6 +121,7 @@ public class CatalogBundleLoader { } } + // TODO remove; now that the bundle is passed through we can add it in the catalog private String addLibraryDetails(Bundle bundle, String bomText) { @SuppressWarnings("unchecked") final Map<String, Object> bom = (Map<String, Object>) Iterables.getOnlyElement(Yamls.parseAll(bomText)); @@ -142,7 +143,12 @@ public class CatalogBundleLoader { private void addLibraryDetails(Bundle bundle, Map<String, Object> catalog) { if (!catalog.containsKey(CatalogBundleLoader.BROOKLYN_LIBRARIES)) { - catalog.put(CatalogBundleLoader.BROOKLYN_LIBRARIES, MutableList.of()); + if (catalog.containsKey("libraries")) { + // legacy name + catalog.put(CatalogBundleLoader.BROOKLYN_LIBRARIES, catalog.remove("libraries")); + } else { + catalog.put(CatalogBundleLoader.BROOKLYN_LIBRARIES, MutableList.of()); + } } final Object librarySpec = catalog.get(CatalogBundleLoader.BROOKLYN_LIBRARIES); if (!(librarySpec instanceof List)) { http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiArchiveInstaller.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiArchiveInstaller.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiArchiveInstaller.java index e4ffb54..9ef04c8 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiArchiveInstaller.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiArchiveInstaller.java @@ -59,10 +59,6 @@ class OsgiArchiveInstaller { private static final Logger log = LoggerFactory.getLogger(OsgiArchiveInstaller.class); - // must be 1.0; see bottom of - // http://www.eclipse.org/virgo/documentation/virgo-documentation-3.7.0.M01/docs/virgo-user-guide/html/ch02s02.html - private static final String OSGI_MANIFEST_VERSION_VALUE = "1.0"; - final private OsgiManager osgiManager; private ManagedBundle suppliedKnownBundleMetadata; private InputStream zipIn; @@ -185,9 +181,9 @@ class OsgiArchiveInstaller { } catch (IOException e) { throw new IllegalArgumentException("Invalid ZIP/JAR archive: "+e); } - ZipEntry bom = zf.getEntry("catalog.bom"); + ZipEntry bom = zf.getEntry(BasicBrooklynCatalog.CATALOG_BOM); if (bom==null) { - bom = zf.getEntry("/catalog.bom"); + bom = zf.getEntry("/"+BasicBrooklynCatalog.CATALOG_BOM); } if (bom==null) { if (isCatalogBomRequired) { @@ -231,7 +227,7 @@ class OsgiArchiveInstaller { throw new IllegalArgumentException("Missing bundle version in BOM or MANIFEST"); } if (discoveredManifest.getMainAttributes().getValue(Attributes.Name.MANIFEST_VERSION)==null) { - discoveredManifest.getMainAttributes().putValue(Attributes.Name.MANIFEST_VERSION.toString(), OSGI_MANIFEST_VERSION_VALUE); + discoveredManifest.getMainAttributes().putValue(Attributes.Name.MANIFEST_VERSION.toString(), BasicBrooklynCatalog.OSGI_MANIFEST_VERSION_VALUE); manifestNeedsUpdating = true; } if (manifestNeedsUpdating) { http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiBundleInstallationResult.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiBundleInstallationResult.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiBundleInstallationResult.java index c3a725a..9e845ad 100644 --- a/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiBundleInstallationResult.java +++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/ha/OsgiBundleInstallationResult.java @@ -37,11 +37,17 @@ public class OsgiBundleInstallationResult { Runnable deferredStart; public enum ResultCode { - INSTALLED_NEW_BUNDLE, - UPDATED_EXISTING_BUNDLE, - IGNORING_BUNDLE_AREADY_INSTALLED, - ERROR_PREPARING_BUNDLE, - ERROR_INSTALLING_BUNDLE + INSTALLED_NEW_BUNDLE(false), + UPDATED_EXISTING_BUNDLE(false), + // TODO if bundle installed is different to bundle supplied we should flag an error + IGNORING_BUNDLE_AREADY_INSTALLED(false), + ERROR_PREPARING_BUNDLE(true), + ERROR_INSTALLING_BUNDLE(true); + + final boolean isError; + ResultCode(boolean isError) { this.isError = isError; } + + public boolean isError() { return isError; } } final List<String> catalogItemsInstalled = MutableList.of(); http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/typereg/BasicManagedBundle.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/typereg/BasicManagedBundle.java b/core/src/main/java/org/apache/brooklyn/core/typereg/BasicManagedBundle.java index 350507c..d480ed5 100644 --- a/core/src/main/java/org/apache/brooklyn/core/typereg/BasicManagedBundle.java +++ b/core/src/main/java/org/apache/brooklyn/core/typereg/BasicManagedBundle.java @@ -55,8 +55,6 @@ public class BasicManagedBundle extends AbstractBrooklynObject implements Manage Preconditions.checkNotNull(name, "Either a URL or both name and version are required"); Preconditions.checkNotNull(version, "Either a URL or both name and version are required"); } - Version.parseVersion(version); - this.symbolicName = name; this.version = version; this.url = url; http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/core/src/main/java/org/apache/brooklyn/core/typereg/BasicOsgiBundleWithUrl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/typereg/BasicOsgiBundleWithUrl.java b/core/src/main/java/org/apache/brooklyn/core/typereg/BasicOsgiBundleWithUrl.java index 264b96c..9001ad4 100644 --- a/core/src/main/java/org/apache/brooklyn/core/typereg/BasicOsgiBundleWithUrl.java +++ b/core/src/main/java/org/apache/brooklyn/core/typereg/BasicOsgiBundleWithUrl.java @@ -20,6 +20,7 @@ package org.apache.brooklyn.core.typereg; import org.apache.brooklyn.api.catalog.CatalogItem.CatalogBundle; import org.apache.brooklyn.api.typereg.OsgiBundleWithUrl; +import org.apache.brooklyn.util.osgi.VersionedName; import org.apache.brooklyn.util.text.BrooklynVersionSyntax; import com.google.common.base.MoreObjects; @@ -79,6 +80,12 @@ public class BasicOsgiBundleWithUrl implements CatalogBundle, OsgiBundleWithUrl } @Override + public VersionedName getVersionedName() { + if (!isNameResolved()) return null; + return new VersionedName(getSymbolicName(), getSuppliedVersionString()); + } + + @Override public String getUrl() { return url; } http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/server-cli/src/main/resources/brooklyn/default.catalog.bom ---------------------------------------------------------------------- diff --git a/server-cli/src/main/resources/brooklyn/default.catalog.bom b/server-cli/src/main/resources/brooklyn/default.catalog.bom index 5a5d1b2..d4d3797 100644 --- a/server-cli/src/main/resources/brooklyn/default.catalog.bom +++ b/server-cli/src/main/resources/brooklyn/default.catalog.bom @@ -3,11 +3,13 @@ # and templates to get started using Brooklyn brooklyn.catalog: + bundle: brooklyn-default-catalog version: "0.12.0-SNAPSHOT" # BROOKLYN_VERSION - items: # load everything in the classpath with a @Catalog annotation - - scanJavaAnnotations: true + scanJavaAnnotations: true + + items: - id: server itemType: entity http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/d7975d14/utils/common/src/main/java/org/apache/brooklyn/util/osgi/VersionedName.java ---------------------------------------------------------------------- diff --git a/utils/common/src/main/java/org/apache/brooklyn/util/osgi/VersionedName.java b/utils/common/src/main/java/org/apache/brooklyn/util/osgi/VersionedName.java index 9ae70df..98d6759 100644 --- a/utils/common/src/main/java/org/apache/brooklyn/util/osgi/VersionedName.java +++ b/utils/common/src/main/java/org/apache/brooklyn/util/osgi/VersionedName.java @@ -96,6 +96,13 @@ public class VersionedName { } @Nullable + public String getOsgiVersionString() { + Version ov = getOsgiVersion(); + if (ov==null) return null; + return ov.toString(); + } + + @Nullable public String getVersionString() { return v; }
