http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDo.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDo.java new file mode 100644 index 0000000..2bf9cca --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDo.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.util.Arrays; +import java.util.Set; + +import javax.annotation.Nullable; + +import org.reflections.util.ClasspathHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.brooklyn.api.catalog.Catalog; +import org.apache.brooklyn.api.catalog.CatalogItem; +import org.apache.brooklyn.api.entity.Application; +import org.apache.brooklyn.api.entity.Entity; +import org.apache.brooklyn.api.entity.proxying.ImplementedBy; +import org.apache.brooklyn.api.location.Location; +import org.apache.brooklyn.api.policy.Policy; +import org.apache.brooklyn.core.management.internal.ManagementContextInternal; + +import brooklyn.entity.basic.ApplicationBuilder; +import brooklyn.util.ResourceUtils; +import brooklyn.util.exceptions.Exceptions; +import brooklyn.util.javalang.AggregateClassLoader; +import brooklyn.util.javalang.ReflectionScanner; +import brooklyn.util.javalang.UrlClassLoader; +import brooklyn.util.os.Os; +import brooklyn.util.stream.Streams; +import brooklyn.util.text.Strings; +import brooklyn.util.time.Time; + +import com.google.common.annotations.Beta; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Iterables; + +public class CatalogClasspathDo { + + public static enum CatalogScanningModes { + /** the classpath is not scanned; + * for any catalog which is presented over the internet this is recommended (to prevent loading) and is the default; + * (you should explicitly list the items to include; it may be useful to autogenerate it by using a local catalog + * scanning with ANNOTATIONS, viwing that by running mgmt.getCatalog().toXmlString(), + * then editing the resulting XML (e.g. setting the classpath and removing the scan attribute) */ + NONE, + + /** types in the classpath are scanned for annotations indicating inclusion in the catalog ({@link Catalog}); + * this is the default if no catalog is supplied, scanning the local classpath */ + ANNOTATIONS, + + @Beta + /** all catalog-friendly types are included, + * even if not annotated for inclusion in the catalog; useful for quick hacking, + * or a classpath (and possibly in future a regex, if added) which is known to have only good things in it; + * however the precise semantics of what is included is subject to change, + * and it is strongly recommended to use the {@link Catalog} annotation and scan for annotations + * <p> + * a catalog-friendly type is currently defined as: + * any concrete non-anonymous (and not a non-static inner) class implementing Entity or Policy; + * and additionally for entities and applications, an interface with the {@link ImplementedBy} annotation; + * note that this means classes done "properly" with both an interface and an implementation + * will be included twice, once as interface and once as implementation; + * this guarantees inclusion of anything previously included (implementations; + * and this will be removed from catalog in future likely), + * plus things now done properly (which will become the only way in the future) + **/ + TYPES + } + + private static final Logger log = LoggerFactory.getLogger(CatalogClasspathDo.class); + + private final CatalogDo catalog; + private final CatalogClasspathDto classpath; + private final CatalogScanningModes scanMode; + + boolean isLoaded = false; + private URL[] urls; + + private final AggregateClassLoader classloader = AggregateClassLoader.newInstanceWithNoLoaders(); + private volatile boolean classloaderLoaded = false; + + public CatalogClasspathDo(CatalogDo catalog) { + this.catalog = Preconditions.checkNotNull(catalog, "catalog"); + this.classpath = catalog.dto.classpath; + this.scanMode = (classpath != null) ? classpath.scan : null; + } + + /** causes all scanning-based classpaths to scan the classpaths + * (but does _not_ load all JARs) */ + // TODO this does a Java scan; we also need an OSGi scan which uses the OSGi classloaders when loading for scanning and resolving dependencies + synchronized void load() { + if (classpath == null || isLoaded) return; + + if (classpath.getEntries() == null) { + urls = new URL[0]; + } else { + urls = new URL[classpath.getEntries().size()]; + for (int i=0; i<urls.length; i++) { + try { + String u = classpath.getEntries().get(i); + if (u.startsWith("classpath:")) { + // special support for classpath: url's + // TODO put convenience in ResourceUtils for extracting to a normal url + // (or see below) + InputStream uin = ResourceUtils.create(this).getResourceFromUrl(u); + File f = Os.newTempFile("brooklyn-catalog-"+u, null); + FileOutputStream fout = new FileOutputStream(f); + try { + Streams.copy(uin, fout); + } finally { + Streams.closeQuietly(fout); + Streams.closeQuietly(uin); + } + u = f.toURI().toString(); + } + urls[i] = new URL(u); + + // TODO potential disk leak above as we have no way to know when the temp file can be removed earlier than server shutdown; + // a better way to handle this is to supply a stream handler (but URLConnection is a little bit hard to work with): +// urls[i] = new URL(null, classpath.getEntries().get(i) // (handy construtor for reparsing urls, without splitting into uri first) +// , new URLStreamHandler() { +// @Override +// protected URLConnection openConnection(URL u) throws IOException { +// new ResourceUtils(null). ??? +// } +// }); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + log.error("Error loading URL "+classpath.getEntries().get(i)+" in definition of catalog "+catalog+"; skipping definition"); + throw Exceptions.propagate(e); + } + } + } + + // prefix is supported (but not really used yet) -- + // seems to have _better_ URL-discovery with prefixes + // (might also offer regex ? but that is post-load filter as opposed to native optimisation) + String prefix = null; + + if (scanMode==null || scanMode==CatalogScanningModes.NONE) + return; + + Stopwatch timer = Stopwatch.createStarted(); + ReflectionScanner scanner = null; + if (!catalog.isLocal()) { + log.warn("Scanning not supported for remote catalogs; ignoring scan request in "+catalog); + } else if (classpath.getEntries() == null || classpath.getEntries().isEmpty()) { + // scan default classpath: + ClassLoader baseCL = null; + Iterable<URL> baseCP = null; + if (catalog.mgmt instanceof ManagementContextInternal) { + baseCL = ((ManagementContextInternal)catalog.mgmt).getBaseClassLoader(); + baseCP = ((ManagementContextInternal)catalog.mgmt).getBaseClassPathForScanning(); + } + scanner = new ReflectionScanner(baseCP, prefix, baseCL, catalog.getRootClassLoader()); + if (scanner.getSubTypesOf(Entity.class).isEmpty()) { + try { + ((ManagementContextInternal)catalog.mgmt).setBaseClassPathForScanning(ClasspathHelper.forJavaClassPath()); + log.debug("Catalog scan of default classloader returned nothing; reverting to java.class.path"); + baseCP = ((ManagementContextInternal)catalog.mgmt).getBaseClassPathForScanning(); + scanner = new ReflectionScanner(baseCP, prefix, baseCL, catalog.getRootClassLoader()); + } catch (Exception e) { + log.info("Catalog scan is empty, and unable to use java.class.path (base classpath is "+baseCP+"): "+e); + Exceptions.propagateIfFatal(e); + } + } + } else { + // scan specified jars: + scanner = new ReflectionScanner(urls==null || urls.length==0 ? null : Arrays.asList(urls), prefix, getLocalClassLoader()); + } + + if (scanner!=null) { + int count = 0, countApps = 0; + if (scanMode==CatalogScanningModes.ANNOTATIONS) { + Set<Class<?>> catalogClasses = scanner.getTypesAnnotatedWith(Catalog.class); + for (Class<?> c: catalogClasses) { + try { + CatalogItem<?,?> item = addCatalogEntry(c); + count++; + if (CatalogTemplateItemDto.class.isInstance(item)) countApps++; + } catch (Exception e) { + log.warn("Failed to add catalog entry for "+c+"; continuing scan...", e); + } + } + } else if (scanMode==CatalogScanningModes.TYPES) { + Iterable<Class<?>> entities = this.excludeInvalidClasses( + Iterables.concat(scanner.getSubTypesOf(Entity.class), + // not sure why we have to look for sub-types of Application, + // they should be picked up as sub-types of Entity, but in maven builds (only!) + // they are not -- i presume a bug in scanner + scanner.getSubTypesOf(Application.class), + scanner.getSubTypesOf(ApplicationBuilder.class))); + for (Class<?> c: entities) { + if (Application.class.isAssignableFrom(c) || ApplicationBuilder.class.isAssignableFrom(c)) { + addCatalogEntry(new CatalogTemplateItemDto(), c); + countApps++; + } else { + addCatalogEntry(new CatalogEntityItemDto(), c); + } + count++; + } + Iterable<Class<? extends Policy>> policies = this.excludeInvalidClasses(scanner.getSubTypesOf(Policy.class)); + for (Class<?> c: policies) { + addCatalogEntry(new CatalogPolicyItemDto(), c); + count++; + } + + Iterable<Class<? extends Location>> locations = this.excludeInvalidClasses(scanner.getSubTypesOf(Location.class)); + for (Class<?> c: locations) { + addCatalogEntry(new CatalogLocationItemDto(), c); + count++; + } + } else { + throw new IllegalStateException("Unsupported catalog scan mode "+scanMode+" for "+this); + } + log.debug("Catalog '"+catalog.dto.name+"' classpath scan completed: loaded "+ + count+" item"+Strings.s(count)+" ("+countApps+" app"+Strings.s(countApps)+") in "+Time.makeTimeStringRounded(timer)); + } + + isLoaded = true; + } + + /** removes inner classes (non-static nesteds) and others; + * bear in mind named ones will be hard to instantiate without the outer class instance) */ + private <T> Iterable<Class<? extends T>> excludeInvalidClasses(Iterable<Class<? extends T>> input) { + Predicate<Class<? extends T>> f = new Predicate<Class<? extends T>>() { + @Override + public boolean apply(@Nullable Class<? extends T> input) { + if (input==null) return false; + if (input.isLocalClass() || input.isAnonymousClass()) return false; + if (Modifier.isAbstract(input.getModifiers())) { + if (input.getAnnotation(ImplementedBy.class)==null) + return false; + } + // non-abstract top-level classes are okay + if (!input.isMemberClass()) return true; + if (!Modifier.isStatic(input.getModifiers())) return false; + // nested classes only okay if static + return true; + } + }; + return Iterables.filter(input, f); + } + + /** augments the given item with annotations and class data for the given class, then adds to catalog + * @deprecated since 0.7.0 the classpath DO is replaced by libraries */ + @Deprecated + public CatalogItem<?,?> addCatalogEntry(Class<?> c) { + if (Application.class.isAssignableFrom(c)) return addCatalogEntry(new CatalogTemplateItemDto(), c); + if (ApplicationBuilder.class.isAssignableFrom(c)) return addCatalogEntry(new CatalogTemplateItemDto(), c); + if (Entity.class.isAssignableFrom(c)) return addCatalogEntry(new CatalogEntityItemDto(), c); + if (Policy.class.isAssignableFrom(c)) return addCatalogEntry(new CatalogPolicyItemDto(), c); + if (Location.class.isAssignableFrom(c)) return addCatalogEntry(new CatalogLocationItemDto(), c); + throw new IllegalStateException("Cannot add "+c+" to catalog: unsupported type "+c.getName()); + } + + /** augments the given item with annotations and class data for the given class, then adds to catalog + * @deprecated since 0.7.0 the classpath DO is replaced by libraries */ + @Deprecated + public CatalogItem<?,?> addCatalogEntry(CatalogItemDtoAbstract<?,?> item, Class<?> c) { + Catalog annotations = c.getAnnotation(Catalog.class); + item.setSymbolicName(c.getName()); + item.setJavaType(c.getName()); + item.setDisplayName(firstNonEmpty(c.getSimpleName(), c.getName())); + if (annotations!=null) { + item.setDisplayName(firstNonEmpty(annotations.name(), item.getDisplayName())); + item.setDescription(firstNonEmpty(annotations.description())); + item.setIconUrl(firstNonEmpty(annotations.iconUrl())); + } + if (log.isTraceEnabled()) + log.trace("adding to catalog: "+c+" (from catalog "+catalog+")"); + catalog.addEntry(item); + return item; + } + + private static String firstNonEmpty(String ...candidates) { + for (String c: candidates) + if (c!=null && !c.isEmpty()) return c; + return null; + } + + /** returns classloader for the entries specified here */ + public ClassLoader getLocalClassLoader() { + if (!classloaderLoaded) loadLocalClassLoader(); + return classloader; + } + + protected synchronized void loadLocalClassLoader() { + if (classloaderLoaded) return; + if (urls==null) return; + classloader.addFirst(new UrlClassLoader(urls)); + classloaderLoaded = true; + return; + } + + /** adds the given URL as something this classloader will load + * (however no scanning is done) */ + public void addToClasspath(URL u, boolean updateDto) { + if (updateDto) classpath.getEntries().add(u.toExternalForm()); + addToClasspath(new UrlClassLoader(u)); + } + + /** adds the given URL as something this classloader will load + * (however no scanning is done). + * <p> + * the DTO will _not_ be updated. */ + public void addToClasspath(ClassLoader loader) { + classloader.addFirst(loader); + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDto.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDto.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDto.java new file mode 100644 index 0000000..5779a4e --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogClasspathDto.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.brooklyn.core.catalog.internal.CatalogClasspathDo.CatalogScanningModes; + +public class CatalogClasspathDto { + + /** whether/what to scan; defaults to 'none' */ + CatalogScanningModes scan; + private List<String> entries; + + public synchronized void addEntry(String url) { + if (entries==null) + entries = new CopyOnWriteArrayList<String>(); + + entries.add(url); + } + + public synchronized List<String> getEntries() { + return entries; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDo.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDo.java new file mode 100644 index 0000000..8e53e28 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDo.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.brooklyn.api.management.ManagementContext; +import org.apache.brooklyn.core.catalog.internal.CatalogClasspathDo.CatalogScanningModes; +import org.apache.brooklyn.core.management.internal.ManagementContextInternal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import brooklyn.util.collections.MutableList; +import brooklyn.util.exceptions.Exceptions; +import brooklyn.util.javalang.AggregateClassLoader; +import brooklyn.util.net.Urls; +import brooklyn.util.time.CountdownTimer; +import brooklyn.util.time.Duration; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class CatalogDo { + + private static final Logger log = LoggerFactory.getLogger(CatalogDo.class); + + volatile boolean isLoaded = false; + final CatalogDto dto; + ManagementContext mgmt = null; + CatalogDo parent = null; + + List<CatalogDo> childrenCatalogs = new ArrayList<CatalogDo>(); + CatalogClasspathDo classpath; + private Map<String, CatalogItemDo<?,?>> cacheById; + + AggregateClassLoader childrenClassLoader = AggregateClassLoader.newInstanceWithNoLoaders(); + ClassLoader recursiveClassLoader; + + protected CatalogDo(CatalogDto dto) { + this.dto = Preconditions.checkNotNull(dto); + } + + public CatalogDo(ManagementContext mgmt, CatalogDto dto) { + this(dto); + this.mgmt = mgmt; + } + + boolean isLoaded() { + return isLoaded; + } + + /** Calls {@link #load(CatalogDo)} with a null parent. */ + public CatalogDo load() { + return load(null); + } + + /** Calls {@link #load(ManagementContext, CatalogDo)} with the catalog's existing management context. */ + public CatalogDo load(CatalogDo parent) { + return load(mgmt, parent); + } + + /** Calls {@link #load(ManagementContext, CatalogDo, boolean)} failing on load errors. */ + public synchronized CatalogDo load(ManagementContext mgmt, CatalogDo parent) { + return load(mgmt, parent, true); + } + + /** causes all URL-based catalogs to have their manifests loaded, + * and all scanning-based classpaths to scan the classpaths + * (but does not load all JARs) + */ + public synchronized CatalogDo load(ManagementContext mgmt, CatalogDo parent, boolean failOnLoadError) { + if (isLoaded()) { + if (mgmt!=null && !Objects.equal(mgmt, this.mgmt)) { + throw new IllegalStateException("Cannot set mgmt "+mgmt+" on "+this+" after catalog is loaded"); + } + log.debug("Catalog "+this+" is already loaded"); + return this; + } + loadThisCatalog(mgmt, parent, failOnLoadError); + loadChildrenCatalogs(failOnLoadError); + buildCaches(); + return this; + } + + protected synchronized void loadThisCatalog(ManagementContext mgmt, CatalogDo parent, boolean failOnLoadError) { + if (isLoaded()) return; + CatalogUtils.logDebugOrTraceIfRebinding(log, "Loading catalog {} into {}", this, parent); + if (this.parent!=null && !this.parent.equals(parent)) + log.warn("Catalog "+this+" being initialised with different parent "+parent+" when already parented by "+this.parent, new Throwable("source of reparented "+this)); + if (this.mgmt!=null && !this.mgmt.equals(mgmt)) + log.warn("Catalog "+this+" being initialised with different mgmt "+mgmt+" when already managed by "+this.mgmt, new Throwable("source of reparented "+this)); + this.parent = parent; + this.mgmt = mgmt; + dto.populate(); + loadCatalogClasspath(); + loadCatalogItems(failOnLoadError); + isLoaded = true; + synchronized (this) { + notifyAll(); + } + } + + private void loadCatalogClasspath() { + try { + classpath = new CatalogClasspathDo(this); + classpath.load(); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + log.error("Unable to load catalog "+this+" (ignoring): "+e); + log.info("Trace for failure to load "+this+": "+e, e); + } + } + + private void loadCatalogItems(boolean failOnLoadError) { + Iterable<CatalogItemDtoAbstract<?, ?>> entries = dto.getUniqueEntries(); + if (entries!=null) { + for (CatalogItemDtoAbstract<?,?> entry : entries) { + try { + CatalogUtils.installLibraries(mgmt, entry.getLibraries()); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + if (failOnLoadError) { + Exceptions.propagate(e); + } else { + log.error("Loading bundles for catalog item " + entry + " failed: " + e.getMessage(), e); + } + } + } + } + } + + public boolean blockIfNotLoaded(Duration timeout) throws InterruptedException { + if (isLoaded()) return true; + synchronized (this) { + if (isLoaded()) return true; + CountdownTimer timer = CountdownTimer.newInstanceStarted(timeout); + while (!isLoaded()) + if (!timer.waitOnForExpiry(this)) + return false; + return true; + } + } + + protected void loadChildrenCatalogs(boolean failOnLoadError) { + if (dto.catalogs!=null) { + for (CatalogDto child: dto.catalogs) { + loadCatalog(child, failOnLoadError); + } + } + } + + CatalogDo loadCatalog(CatalogDto child, boolean failOnLoadError) { + CatalogDo childL = new CatalogDo(child); + childrenCatalogs.add(childL); + childL.load(mgmt, this, failOnLoadError); + childrenClassLoader.addFirst(childL.getRecursiveClassLoader()); + clearCache(false); + return childL; + } + + protected Map<String, CatalogItemDo<?,?>> getIdCache() { + Map<String, CatalogItemDo<?,?>> cache = this.cacheById; + if (cache==null) cache = buildCaches(); + return cache; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected synchronized Map<String, CatalogItemDo<?,?>> buildCaches() { + if (cacheById != null) return cacheById; + CatalogUtils.logDebugOrTraceIfRebinding(log, "Building cache for {}", this); + if (!isLoaded()) + log.debug("Catalog not fully loaded when loading cache of "+this); + + Map<String, CatalogItemDo<?,?>> cache = new LinkedHashMap<String, CatalogItemDo<?,?>>(); + + // build the cache; first from children catalogs, then from local entities + // so that root and near-root takes precedence over deeper items; + // and go through in reverse order so that things at the top of the file take precedence + // (both in the cache and in the aggregate class loader); + // however anything added _subsequently_ will take precedence (again in both) + if (dto.catalogs!=null) { + List<CatalogDo> catalogsReversed = new ArrayList<CatalogDo>(childrenCatalogs); + Collections.reverse(catalogsReversed); + for (CatalogDo child: catalogsReversed) { + cache.putAll(child.getIdCache()); + } + } + if (dto.getUniqueEntries()!=null) { + List<CatalogItemDtoAbstract<?,?>> entriesReversed = MutableList.copyOf(dto.getUniqueEntries()); + Collections.reverse(entriesReversed); + for (CatalogItemDtoAbstract<?,?> entry: entriesReversed) + cache.put(entry.getId(), new CatalogItemDo(this, entry)); + } + this.cacheById = cache; + return cache; + } + + protected synchronized void clearCache(boolean deep) { + this.cacheById = null; + if (deep) { + for (CatalogDo child : childrenCatalogs) { + child.clearCache(true); + } + } + clearParentCache(); + } + protected void clearParentCache() { + if (this.parent!=null) + this.parent.clearCache(false); + } + + /** + * Adds the given entry to the catalog, with no enrichment. + * Callers may prefer {@link CatalogClasspathDo#addCatalogEntry(CatalogItemDtoAbstract, Class)} + */ + public synchronized void addEntry(CatalogItemDtoAbstract<?,?> entry) { + dto.addEntry(entry); + + // could do clearCache(false); but this is slightly more efficient... + if (cacheById != null) { + @SuppressWarnings({ "unchecked", "rawtypes" }) + CatalogItemDo<?, ?> cdo = new CatalogItemDo(this, entry); + cacheById.put(entry.getId(), cdo); + } + clearParentCache(); + + if (mgmt != null) { + mgmt.getRebindManager().getChangeListener().onManaged(entry); + } + } + + /** + * Removes the given entry from the catalog. + */ + public synchronized void deleteEntry(CatalogItemDtoAbstract<?, ?> entry) { + dto.removeEntry(entry); + + // could do clearCache(false); but this is slightly more efficient... + if (cacheById != null) { + cacheById.remove(entry.getId()); + } + clearParentCache(); + + if (mgmt != null) { + // TODO: Can the entry be in more than one catalogue? The management context has no notion of + // catalogue hierarchy so this will effectively remove it from all catalogues. + // (YES- we're assuming ID's are unique across all catalogues; if not, things get out of sync; + // however see note at top of BasicBrooklynCatalog -- + // manualCatalog and OSGi is used for everything now except legacy XML trees) + mgmt.getRebindManager().getChangeListener().onUnmanaged(entry); + } + } + + /** returns loaded catalog, if this has been loaded */ + CatalogDo addCatalog(CatalogDto child) { + if (dto.catalogs == null) + dto.catalogs = new ArrayList<CatalogDto>(); + dto.catalogs.add(child); + if (!isLoaded()) + return null; + return loadCatalog(child, true); + } + + /** adds the given urls; filters out any nulls supplied */ + public synchronized void addToClasspath(String ...urls) { + if (dto.classpath == null) + dto.classpath = new CatalogClasspathDto(); + for (String url: urls) { + if (url!=null) + dto.classpath.addEntry(url); + } + if (isLoaded()) + throw new IllegalStateException("dynamic classpath entry value update not supported"); + // easy enough to add, just support unload+reload (and can also allow dynamic setScan below) + // but more predictable if we don't; the one exception is in the manualAdditionsCatalog + // where BasicBrooklynCatalog reaches in and updates the DTO and/or CompositeClassLoader directly, if necessary +// for (String url: urls) +// loadedClasspath.addEntry(url); + } + + public synchronized void setClasspathScanForEntities(CatalogScanningModes value) { + if (dto.classpath == null) + dto.classpath = new CatalogClasspathDto(); + dto.classpath.scan = value; + if (isLoaded()) + throw new IllegalStateException("dynamic classpath scan value update not supported"); + // easy enough to add, see above + } + + @Override + public String toString() { + String size = cacheById == null ? "not yet loaded" : "size " + cacheById.size(); + return "Loaded:" + dto + "(" + size + ")"; + } + + /** is "local" if it and all ancestors are not based on any remote urls */ + public boolean isLocal() { + if (dto.url != null) { + String proto = Urls.getProtocol(dto.url); + if (proto != null) { + // 'file' is the only protocol accepted as "local" + if (!"file".equals(proto)) return false; + } + } + return parent == null || parent.isLocal(); + } + + /** classloader for only the entries in this catalog's classpath */ + public ClassLoader getLocalClassLoader() { + if (classpath != null) return classpath.getLocalClassLoader(); + return null; + } + + /** recursive classloader is the local classloader plus all children catalog's classloader */ + public ClassLoader getRecursiveClassLoader() { + if (recursiveClassLoader == null) loadRecursiveClassLoader(); + return recursiveClassLoader; + } + + protected synchronized void loadRecursiveClassLoader() { + if (recursiveClassLoader!=null) return; + AggregateClassLoader cl = AggregateClassLoader.newInstanceWithNoLoaders(); + cl.addFirst(childrenClassLoader); + ClassLoader local = getLocalClassLoader(); + if (local != null) cl.addFirst(local); + if (parent == null) { + // we are root. include the mgmt base classloader and/or standard class loaders + ClassLoader base = mgmt != null ? ((ManagementContextInternal)mgmt).getBaseClassLoader() : null; + if (base != null) cl.addFirst(base); + else { + cl.addFirst(getClass().getClassLoader()); + cl.addFirst(Object.class.getClassLoader()); + } + } + recursiveClassLoader = cl; + } + + /** the root classloader is the recursive CL from the outermost catalog + * (which includes the base classloader from the mgmt context, if set) */ + public ClassLoader getRootClassLoader() { + if (parent != null) return parent.getRootClassLoader(); + return getRecursiveClassLoader(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDto.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDto.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDto.java new file mode 100644 index 0000000..847f114 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDto.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.io.InputStream; +import java.io.StringReader; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.brooklyn.api.catalog.CatalogItem; + +import brooklyn.util.ResourceUtils; +import brooklyn.util.collections.MutableList; +import brooklyn.util.collections.MutableMap; +import brooklyn.util.exceptions.Exceptions; +import brooklyn.util.exceptions.PropagatedRuntimeException; +import brooklyn.util.stream.Streams; + +import com.google.common.annotations.Beta; +import com.google.common.base.Objects; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.Lists; + +@Beta +public class CatalogDto { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogDto.class); + + String id; + String url; + String contents; + String contentsDescription; + String name; + String description; + CatalogClasspathDto classpath; + private List<CatalogItemDtoAbstract<?,?>> entries = null; + + // for thread-safety, any dynamic additions to this should be handled by a method + // in this class which does copy-on-write + List<CatalogDto> catalogs = null; + + public static CatalogDto newDefaultLocalScanningDto(CatalogClasspathDo.CatalogScanningModes scanMode) { + CatalogDo result = new CatalogDo( + newNamedInstance("Local Scanned Catalog", "All annotated Brooklyn entities detected in the default classpath", "scanning-local-classpath") ); + result.setClasspathScanForEntities(scanMode); + return result.dto; + } + + /** @deprecated since 0.7.0 use {@link #newDtoFromXmlUrl(String)} if you must, but note the xml format itself is deprecated */ + @Deprecated + public static CatalogDto newDtoFromUrl(String url) { + return newDtoFromXmlUrl(url); + } + + /** @deprecated since 0.7.0 the xml format is deprecated; use YAML parse routines on BasicBrooklynCatalog */ + @Deprecated + public static CatalogDto newDtoFromXmlUrl(String url) { + if (LOG.isDebugEnabled()) LOG.debug("Retrieving catalog from: {}", url); + try { + InputStream source = ResourceUtils.create().getResourceFromUrl(url); + String contents = Streams.readFullyString(source); + return newDtoFromXmlContents(contents, url); + } catch (Throwable t) { + Exceptions.propagateIfFatal(t); + throw new PropagatedRuntimeException("Unable to retrieve catalog from " + url + ": " + t, t); + } + } + + /** @deprecated since 0.7.0 the xml format is deprecated; use YAML parse routines on BasicBrooklynCatalog */ + @Deprecated + public static CatalogDto newDtoFromXmlContents(String xmlContents, String originDescription) { + CatalogDto result = (CatalogDto) new CatalogXmlSerializer().deserialize(new StringReader(xmlContents)); + result.contentsDescription = originDescription; + + if (LOG.isDebugEnabled()) LOG.debug("Retrieved catalog from: {}", originDescription); + return result; + } + + /** + * Creates a DTO. + * <p> + * The way contents is treated may change; thus this (and much of catalog) should be treated as beta. + * + * @param name + * @param description + * @param optionalContentsDescription optional description of contents; if null, we normally expect source 'contents' to be set later; + * if the DTO has no 'contents' (ie XML source) then a description should be supplied so we know who is populating it + * (e.g. manual additions); without this, warnings may be generated + * + * @return a new Catalog DTO + */ + public static CatalogDto newNamedInstance(String name, String description, String optionalContentsDescription) { + CatalogDto result = new CatalogDto(); + result.name = name; + result.description = description; + if (optionalContentsDescription!=null) result.contentsDescription = optionalContentsDescription; + return result; + } + + /** Used when caller wishes to create an explicitly empty catalog */ + public static CatalogDto newEmptyInstance(String optionalContentsDescription) { + CatalogDto result = new CatalogDto(); + if (optionalContentsDescription!=null) result.contentsDescription = optionalContentsDescription; + return result; + } + + public static CatalogDto newLinkedInstance(String url) { + CatalogDto result = new CatalogDto(); + result.contentsDescription = url; + result.contents = ResourceUtils.create().getResourceAsString(url); + return result; + } + + /** @deprecated since 0.7.0 use {@link #newDtoFromCatalogItems(Collection, String)}, supplying a description for tracking */ + @Deprecated + public static CatalogDto newDtoFromCatalogItems(Collection<CatalogItem<?, ?>> entries) { + return newDtoFromCatalogItems(entries, null); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public static CatalogDto newDtoFromCatalogItems(Collection<CatalogItem<?, ?>> entries, String description) { + CatalogDto result = new CatalogDto(); + result.contentsDescription = description; + // Weird casts because compiler does not seem to like + // .copyInto(Lists.<CatalogItemDtoAbstract<?, ?>>newArrayListWithExpectedSize(entries.size())); + result.entries = (List<CatalogItemDtoAbstract<?, ?>>) (List) FluentIterable.from(entries) + .filter(CatalogItemDtoAbstract.class) + .copyInto(Lists.newArrayListWithExpectedSize(entries.size())); + return result; + } + + void populate() { + if (contents==null) { + if (url != null) { + contents = ResourceUtils.create().getResourceAsString(url); + contentsDescription = url; + } else if (contentsDescription==null) { + LOG.debug("Catalog DTO has no contents and no description; ignoring call to populate it. Description should be set to suppress this message."); + return; + } else { + LOG.trace("Nothing needs doing (no contents or URL) for catalog with contents described as "+contentsDescription+"."); + return; + } + } + + CatalogDto remoteDto = newDtoFromXmlContents(contents, contentsDescription); + try { + copyFrom(remoteDto, true); + } catch (Exception e) { + Exceptions.propagate(e); + } + } + + /** + * @throws NullPointerException If source is null (and !skipNulls) + */ + void copyFrom(CatalogDto source, boolean skipNulls) throws IllegalAccessException { + if (source==null) { + if (skipNulls) return; + throw new NullPointerException("source DTO is null, when copying to "+this); + } + + if (!skipNulls || source.id != null) id = source.id; + if (!skipNulls || source.contentsDescription != null) contentsDescription = source.contentsDescription; + if (!skipNulls || source.contents != null) contents = source.contents; + if (!skipNulls || source.name != null) name = source.name; + if (!skipNulls || source.description != null) description = source.description; + if (!skipNulls || source.classpath != null) classpath = source.classpath; + if (!skipNulls || source.entries != null) entries = source.entries; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .omitNullValues() + .add("name", name) + .add("id", id) + .add("contentsDescription", contentsDescription) + .toString(); + } + + // temporary fix for issue where entries might not be unique + Iterable<CatalogItemDtoAbstract<?, ?>> getUniqueEntries() { + if (entries==null) return null; + Map<String, CatalogItemDtoAbstract<?, ?>> result = getEntriesMap(); + return result.values(); + } + + private Map<String, CatalogItemDtoAbstract<?, ?>> getEntriesMap() { + if (entries==null) return null; + Map<String,CatalogItemDtoAbstract<?, ?>> result = MutableMap.of(); + for (CatalogItemDtoAbstract<?,?> entry: entries) { + result.put(entry.getId(), entry); + } + return result; + } + + void removeEntry(CatalogItemDtoAbstract<?, ?> entry) { + if (entries!=null) + entries.remove(entry); + } + + void addEntry(CatalogItemDtoAbstract<?, ?> entry) { + if (entries == null) entries = MutableList.of(); + CatalogItemDtoAbstract<?, ?> oldEntry = getEntriesMap().get(entry.getId()); + entries.add(entry); + if (oldEntry!=null) + removeEntry(oldEntry); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDtoUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDtoUtils.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDtoUtils.java new file mode 100644 index 0000000..e2df123 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogDtoUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.io.InputStream; +import java.io.InputStreamReader; + +import org.apache.brooklyn.core.catalog.internal.CatalogClasspathDo.CatalogScanningModes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import brooklyn.util.ResourceUtils; +import brooklyn.util.exceptions.Exceptions; + +public class CatalogDtoUtils { + + private static final Logger log = LoggerFactory.getLogger(CatalogDtoUtils.class); + + public static CatalogDto newDefaultLocalScanningDto(CatalogScanningModes scanMode) { + return CatalogDto.newDefaultLocalScanningDto(scanMode); + } + + /** throws if there are any problems in retrieving or copying */ + public static void populateFromUrl(CatalogDto dto, String url) { + CatalogDto remoteDto = newDtoFromUrl(url); + try { + copyDto(remoteDto, dto, true); + } catch (Exception e) { + Exceptions.propagate(e); + } + } + + /** does a shallow copy. + * "skipNulls" means not to copy any fields from the source which are null */ + static void copyDto(CatalogDto source, CatalogDto target, boolean skipNulls) throws IllegalArgumentException, IllegalAccessException { + target.copyFrom(source, skipNulls); + } + + public static CatalogDto newDtoFromUrl(String url) { + if (log.isDebugEnabled()) log.debug("Retrieving catalog from: {}", url); + try { + InputStream source = ResourceUtils.create().getResourceFromUrl(url); + CatalogDto result = (CatalogDto) new CatalogXmlSerializer().deserialize(new InputStreamReader(source)); + if (log.isDebugEnabled()) log.debug("Retrieved catalog from: {}", url); + return result; + } catch (Throwable t) { + log.debug("Unable to retrieve catalog from: "+url+" ("+t+")"); + throw Exceptions.propagate(t); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogEntityItemDto.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogEntityItemDto.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogEntityItemDto.java new file mode 100644 index 0000000..ec7cd90 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogEntityItemDto.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import org.apache.brooklyn.api.entity.Entity; +import org.apache.brooklyn.api.entity.proxying.EntitySpec; + + +public class CatalogEntityItemDto extends CatalogItemDtoAbstract<Entity,EntitySpec<?>> { + + @Override + public CatalogItemType getCatalogItemType() { + return CatalogItemType.ENTITY; + } + + @Override + public Class<Entity> getCatalogItemJavaType() { + return Entity.class; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + @Override + public Class<EntitySpec<?>> getSpecType() { + return (Class)EntitySpec.class; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogInitialization.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogInitialization.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogInitialization.java new file mode 100644 index 0000000..047a168 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogInitialization.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.io.File; +import java.util.Collection; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.brooklyn.api.catalog.CatalogItem; +import org.apache.brooklyn.api.management.ManagementContext; +import org.apache.brooklyn.api.management.ha.ManagementNodeState; +import org.apache.brooklyn.core.management.ManagementContextInjectable; +import org.apache.brooklyn.core.management.internal.ManagementContextInternal; + +import brooklyn.config.BrooklynServerConfig; +import brooklyn.util.ResourceUtils; +import brooklyn.util.collections.MutableList; +import brooklyn.util.exceptions.Exceptions; +import brooklyn.util.exceptions.FatalRuntimeException; +import brooklyn.util.exceptions.RuntimeInterruptedException; +import brooklyn.util.flags.TypeCoercions; +import brooklyn.util.guava.Maybe; +import brooklyn.util.javalang.JavaClassNames; +import brooklyn.util.os.Os; +import brooklyn.util.text.Strings; + +import com.google.common.annotations.Beta; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +@Beta +public class CatalogInitialization implements ManagementContextInjectable { + + /* + + A1) if not persisting, go to B1 + A2) if --catalog-reset, delete persisted catalog items + A3) if there is a persisted catalog (and it wasn't not deleted by A2), read it and go to C1 + A4) go to B1 + + B1) look for --catalog-initial, if so read it, then go to C1 + B2) look for BrooklynServerConfig.BROOKLYN_CATALOG_URL, if so, read it, supporting YAML or XML (warning if XML), then go to C1 + B3) look for ~/.brooklyn/catalog.bom, if exists, read it then go to C1 + B4) look for ~/.brooklyn/brooklyn.xml, if exists, warn, read it then go to C1 + B5) read all classpath://brooklyn/default.catalog.bom items, if they exist (and for now they will) + B6) go to C1 + + C1) if --catalog-add, read and add those items + + D1) if persisting, read the rest of the persisted items (entities etc) + + */ + + private static final Logger log = LoggerFactory.getLogger(CatalogInitialization.class); + + private String initialUri; + private boolean reset; + private String additionsUri; + private boolean force; + + private boolean disallowLocal = false; + private List<Function<CatalogInitialization, Void>> callbacks = MutableList.of(); + private boolean + /** has run an unofficial initialization (i.e. an early load, triggered by an early read of the catalog) */ + hasRunUnofficialInitialization = false, + /** has run an official initialization, but it is not a permanent one (e.g. during a hot standby mode, or a run failed) */ + hasRunTransientOfficialInitialization = false, + /** has run an official initialization which is permanent (node is master, and the new catalog is now set) */ + hasRunFinalInitialization = false; + /** is running a populate method; used to prevent recursive loops */ + private boolean isPopulating = false; + + private ManagementContext managementContext; + private boolean isStartingUp = false; + private boolean failOnStartupErrors = false; + + private Object populatingCatalogMutex = new Object(); + + public CatalogInitialization(String initialUri, boolean reset, String additionUri, boolean force) { + this.initialUri = initialUri; + this.reset = reset; + this.additionsUri = additionUri; + this.force = force; + } + + public CatalogInitialization() { + this(null, false, null, false); + } + + @Override + public void injectManagementContext(ManagementContext managementContext) { + Preconditions.checkNotNull(managementContext, "management context"); + if (this.managementContext!=null && managementContext!=this.managementContext) + throw new IllegalStateException("Cannot switch management context, from "+this.managementContext+" to "+managementContext); + this.managementContext = managementContext; + } + + /** Called by the framework to set true while starting up, and false afterwards, + * in order to assist in appropriate logging and error handling. */ + public void setStartingUp(boolean isStartingUp) { + this.isStartingUp = isStartingUp; + } + + public void setFailOnStartupErrors(boolean startupFailOnCatalogErrors) { + this.failOnStartupErrors = startupFailOnCatalogErrors; + } + + public CatalogInitialization addPopulationCallback(Function<CatalogInitialization, Void> callback) { + callbacks.add(callback); + return this; + } + + public ManagementContext getManagementContext() { + return Preconditions.checkNotNull(managementContext, "management context has not been injected into "+this); + } + + public boolean isInitialResetRequested() { + return reset; + } + + /** Returns true if the canonical initialization has completed, + * that is, an initialization which is done when a node is rebinded as master + * (or an initialization done by the startup routines when not running persistence); + * see also {@link #hasRunAnyInitialization()}. */ + public boolean hasRunFinalInitialization() { return hasRunFinalInitialization; } + /** Returns true if an official initialization has run, + * even if it was a transient run, e.g. so that the launch sequence can tell whether rebind has triggered initialization */ + public boolean hasRunOfficialInitialization() { return hasRunFinalInitialization || hasRunTransientOfficialInitialization; } + /** Returns true if the initializer has run at all, + * including transient initializations which might be needed before a canonical becoming-master rebind, + * for instance because the catalog is being accessed before loading rebind information + * (done by {@link #populateUnofficial(BasicBrooklynCatalog)}) */ + public boolean hasRunAnyInitialization() { return hasRunFinalInitialization || hasRunTransientOfficialInitialization || hasRunUnofficialInitialization; } + + /** makes or updates the mgmt catalog, based on the settings in this class + * @param nodeState the management node for which this is being read; if master, then we expect this run to be the last one, + * and so subsequent applications should ignore any initialization data (e.g. on a subsequent promotion to master, + * after a master -> standby -> master cycle) + * @param needsInitialItemsLoaded whether the catalog needs the initial items loaded + * @param needsAdditionalItemsLoaded whether the catalog needs the additions loaded + * @param optionalExcplicitItemsForResettingCatalog + * if supplied, the catalog is reset to contain only these items, before calling any other initialization + * for use primarily when rebinding + */ + public void populateCatalog(ManagementNodeState nodeState, boolean needsInitialItemsLoaded, boolean needsAdditionsLoaded, Collection<CatalogItem<?, ?>> optionalExcplicitItemsForResettingCatalog) { + if (log.isDebugEnabled()) { + String message = "Populating catalog for "+nodeState+", needsInitial="+needsInitialItemsLoaded+", needsAdditional="+needsAdditionsLoaded+", explicitItems="+(optionalExcplicitItemsForResettingCatalog==null ? "null" : optionalExcplicitItemsForResettingCatalog.size())+"; from "+JavaClassNames.callerNiceClassAndMethod(1); + if (!ManagementNodeState.isHotProxy(nodeState)) { + log.debug(message); + } else { + // in hot modes, make this message trace so we don't get too much output then + log.trace(message); + } + } + synchronized (populatingCatalogMutex) { + try { + if (hasRunFinalInitialization() && (needsInitialItemsLoaded || needsAdditionsLoaded)) { + // if we have already run "final" then we should only ever be used to reset the catalog, + // not to initialize or add; e.g. we are being given a fixed list on a subsequent master rebind after the initial master rebind + log.warn("Catalog initialization called to populate initial, even though it has already run the final official initialization"); + } + isPopulating = true; + BasicBrooklynCatalog catalog = (BasicBrooklynCatalog) managementContext.getCatalog(); + if (!catalog.getCatalog().isLoaded()) { + catalog.load(); + } else { + if (needsInitialItemsLoaded && hasRunAnyInitialization()) { + // an indication that something caused it to load early; not severe, but unusual + if (hasRunTransientOfficialInitialization) { + log.debug("Catalog initialization now populating, but has noted a previous official run which was not final (probalby loaded while in a standby mode, or a previous run failed); overwriting any items installed earlier"); + } else { + log.warn("Catalog initialization now populating, but has noted a previous unofficial run (it may have been an early web request); overwriting any items installed earlier"); + } + catalog.reset(ImmutableList.<CatalogItem<?,?>>of()); + } + } + + populateCatalogImpl(catalog, needsInitialItemsLoaded, needsAdditionsLoaded, optionalExcplicitItemsForResettingCatalog); + if (nodeState == ManagementNodeState.MASTER) { + // TODO ideally this would remain false until it has *persisted* the changed catalog; + // if there is a subsequent startup failure the forced additions will not be persisted, + // but nor will they be loaded on a subsequent run. + // callers will have to restart a brooklyn, or reach into this class to change this field, + // or (recommended) manually adjust the catalog. + // TODO also, if a node comes up in standby, the addition might not take effector for a while + // + // however since these options are mainly for use on the very first brooklyn run, it's not such a big deal; + // once up and running the typical way to add items is via the REST API + hasRunFinalInitialization = true; + } + } finally { + if (!hasRunFinalInitialization) { + hasRunTransientOfficialInitialization = true; + } + isPopulating = false; + } + } + } + + private void populateCatalogImpl(BasicBrooklynCatalog catalog, boolean needsInitialItemsLoaded, boolean needsAdditionsLoaded, Collection<CatalogItem<?, ?>> optionalItemsForResettingCatalog) { + applyCatalogLoadMode(); + + if (optionalItemsForResettingCatalog!=null) { + catalog.reset(optionalItemsForResettingCatalog); + } + + if (needsInitialItemsLoaded) { + populateInitial(catalog); + } + + if (needsAdditionsLoaded) { + populateAdditions(catalog); + populateViaCallbacks(catalog); + } + } + + private enum PopulateMode { YAML, XML, AUTODETECT } + + protected void populateInitial(BasicBrooklynCatalog catalog) { + if (disallowLocal) { + if (!hasRunFinalInitialization()) { + log.debug("CLI initial catalog not being read when local catalog load mode is disallowed."); + } + return; + } + +// B1) look for --catalog-initial, if so read it, then go to C1 +// B2) look for BrooklynServerConfig.BROOKLYN_CATALOG_URL, if so, read it, supporting YAML or XML (warning if XML), then go to C1 +// B3) look for ~/.brooklyn/catalog.bom, if exists, read it then go to C1 +// B4) look for ~/.brooklyn/brooklyn.xml, if exists, warn, read it then go to C1 +// B5) read all classpath://brooklyn/default.catalog.bom items, if they exist (and for now they will) +// B6) go to C1 + + if (initialUri!=null) { + populateInitialFromUri(catalog, initialUri, PopulateMode.AUTODETECT); + return; + } + + String catalogUrl = managementContext.getConfig().getConfig(BrooklynServerConfig.BROOKLYN_CATALOG_URL); + if (Strings.isNonBlank(catalogUrl)) { + populateInitialFromUri(catalog, catalogUrl, PopulateMode.AUTODETECT); + return; + } + + catalogUrl = Os.mergePaths(BrooklynServerConfig.getMgmtBaseDir( managementContext.getConfig() ), "catalog.bom"); + if (new File(catalogUrl).exists()) { + populateInitialFromUri(catalog, new File(catalogUrl).toURI().toString(), PopulateMode.YAML); + return; + } + + catalogUrl = Os.mergePaths(BrooklynServerConfig.getMgmtBaseDir( managementContext.getConfig() ), "catalog.xml"); + if (new File(catalogUrl).exists()) { + populateInitialFromUri(catalog, new File(catalogUrl).toURI().toString(), PopulateMode.XML); + return; + } + + // otherwise look for for classpath:/brooklyn/default.catalog.bom -- + // there is one on the classpath which says to scan, and provides a few templates; + // if one is supplied by user in the conf/ dir that will override the item from the classpath + // (TBD - we might want to scan for all such bom's?) + + catalogUrl = "classpath:/brooklyn/default.catalog.bom"; + if (new ResourceUtils(this).doesUrlExist(catalogUrl)) { + populateInitialFromUri(catalog, catalogUrl, PopulateMode.YAML); + return; + } + + log.info("No catalog found on classpath or specified; catalog will not be initialized."); + return; + } + + private void populateInitialFromUri(BasicBrooklynCatalog catalog, String catalogUrl, PopulateMode mode) { + log.debug("Loading initial catalog from {}", catalogUrl); + + Exception problem = null; + Object result = null; + + String contents = null; + try { + contents = new ResourceUtils(this).getResourceAsString(catalogUrl); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + if (problem==null) problem = e; + } + + if (contents!=null && (mode==PopulateMode.YAML || mode==PopulateMode.AUTODETECT)) { + // try YAML first + try { + catalog.reset(MutableList.<CatalogItem<?,?>>of()); + result = catalog.addItems(contents); + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + if (problem==null) problem = e; + } + } + + if (result==null && contents!=null && (mode==PopulateMode.XML || mode==PopulateMode.AUTODETECT)) { + // then try XML + try { + populateInitialFromUriXml(catalog, catalogUrl, contents); + // clear YAML problem + problem = null; + } catch (Exception e) { + Exceptions.propagateIfFatal(e); + if (problem==null) problem = e; + } + } + + if (result!=null) { + log.debug("Loaded initial catalog from {}: {}", catalogUrl, result); + } + if (problem!=null) { + log.warn("Error importing catalog from " + catalogUrl + ": " + problem, problem); + // TODO inform mgmt of error + } + + } + + // deprecated XML format + @SuppressWarnings("deprecation") + private void populateInitialFromUriXml(BasicBrooklynCatalog catalog, String catalogUrl, String contents) { + CatalogDto dto = CatalogDto.newDtoFromXmlContents(contents, catalogUrl); + if (dto!=null) { + catalog.reset(dto); + } + } + + boolean hasRunAdditions = false; + protected void populateAdditions(BasicBrooklynCatalog catalog) { + if (Strings.isNonBlank(additionsUri)) { + if (disallowLocal) { + if (!hasRunAdditions) { + log.warn("CLI additions supplied but not supported when catalog load mode disallows local loads; ignoring."); + } + return; + } + if (!hasRunAdditions) { + log.debug("Adding to catalog from CLI: "+additionsUri+" (force: "+force+")"); + } + Iterable<? extends CatalogItem<?, ?>> items = catalog.addItems( + new ResourceUtils(this).getResourceAsString(additionsUri), force); + + if (!hasRunAdditions) + log.debug("Added to catalog from CLI: "+items); + else + log.debug("Added to catalog from CLI: count "+Iterables.size(items)); + + hasRunAdditions = true; + } + } + + protected void populateViaCallbacks(BasicBrooklynCatalog catalog) { + for (Function<CatalogInitialization, Void> callback: callbacks) + callback.apply(this); + } + + private Object setFromCLMMutex = new Object(); + private boolean setFromCatalogLoadMode = false; + + /** @deprecated since introduced in 0.7.0, only for legacy compatibility with + * {@link CatalogLoadMode} {@link BrooklynServerConfig#CATALOG_LOAD_MODE}, + * allowing control of catalog loading from a brooklyn property */ + @Deprecated + public void applyCatalogLoadMode() { + synchronized (setFromCLMMutex) { + if (setFromCatalogLoadMode) return; + setFromCatalogLoadMode = true; + Maybe<Object> clmm = ((ManagementContextInternal)managementContext).getConfig().getConfigRaw(BrooklynServerConfig.CATALOG_LOAD_MODE, false); + if (clmm.isAbsent()) return; + org.apache.brooklyn.core.catalog.CatalogLoadMode clm = TypeCoercions.coerce(clmm.get(), org.apache.brooklyn.core.catalog.CatalogLoadMode.class); + log.warn("Legacy CatalogLoadMode "+clm+" set: applying, but this should be changed to use new CLI --catalogXxx commands"); + switch (clm) { + case LOAD_BROOKLYN_CATALOG_URL: + reset = true; + break; + case LOAD_BROOKLYN_CATALOG_URL_IF_NO_PERSISTED_STATE: + // now the default + break; + case LOAD_PERSISTED_STATE: + disallowLocal = true; + break; + } + } + } + + /** Creates the catalog based on parameters set here, if not yet loaded, + * but ignoring persisted state and warning if persistence is on and we are starting up + * (because the official persistence is preferred and the catalog will be subsequently replaced); + * for use when the catalog is accessed before persistence is completed. + * <p> + * This method is primarily used during testing, which in many cases does not enforce the full startup order + * and which wants a local catalog in any case. It may also be invoked if a client requests the catalog + * while the server is starting up. */ + public void populateUnofficial(BasicBrooklynCatalog catalog) { + synchronized (populatingCatalogMutex) { + // check isPopulating in case this method gets called from inside another populate call + if (hasRunAnyInitialization() || isPopulating) return; + log.debug("Populating catalog unofficially ("+catalog+")"); + isPopulating = true; + try { + if (isStartingUp) { + log.warn("Catalog access requested when not yet initialized; populating best effort rather than through recommended pathway. Catalog data may be replaced subsequently."); + } + populateCatalogImpl(catalog, true, true, null); + } finally { + hasRunUnofficialInitialization = true; + isPopulating = false; + } + } + } + + public void handleException(Throwable throwable, Object details) { + if (throwable instanceof InterruptedException) + throw new RuntimeInterruptedException((InterruptedException) throwable); + if (throwable instanceof RuntimeInterruptedException) + throw (RuntimeInterruptedException) throwable; + + log.error("Error loading catalog item '"+details+"': "+throwable); + log.debug("Trace for error loading catalog item '"+details+"': "+throwable, throwable); + + // TODO give more detail when adding + ((ManagementContextInternal)getManagementContext()).errors().add(throwable); + + if (isStartingUp && failOnStartupErrors) { + throw new FatalRuntimeException("Unable to load catalog item '"+details+"': "+throwable, throwable); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemBuilder.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemBuilder.java new file mode 100644 index 0000000..e157a51 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemBuilder.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.util.Collection; +import java.util.Collections; + +import org.apache.brooklyn.api.catalog.CatalogItem.CatalogBundle; + +import com.google.common.base.Preconditions; + +public class CatalogItemBuilder<CatalogItemType extends CatalogItemDtoAbstract<?, ?>> { + private CatalogItemType dto; + + public static CatalogItemBuilder<CatalogEntityItemDto> newEntity(String symbolicName, String version) { + return new CatalogItemBuilder<CatalogEntityItemDto>(new CatalogEntityItemDto()) + .symbolicName(symbolicName) + .version(version); + } + + public static CatalogItemBuilder<CatalogTemplateItemDto> newTemplate(String symbolicName, String version) { + return new CatalogItemBuilder<CatalogTemplateItemDto>(new CatalogTemplateItemDto()) + .symbolicName(symbolicName) + .version(version); + } + + public static CatalogItemBuilder<CatalogPolicyItemDto> newPolicy(String symbolicName, String version) { + return new CatalogItemBuilder<CatalogPolicyItemDto>(new CatalogPolicyItemDto()) + .symbolicName(symbolicName) + .version(version); + } + + public static CatalogItemBuilder<CatalogLocationItemDto> newLocation(String symbolicName, String version) { + return new CatalogItemBuilder<CatalogLocationItemDto>(new CatalogLocationItemDto()) + .symbolicName(symbolicName) + .version(version); + } + + public CatalogItemBuilder(CatalogItemType dto) { + this.dto = dto; + this.dto.setLibraries(Collections.<CatalogBundle>emptyList()); + } + + public CatalogItemBuilder<CatalogItemType> symbolicName(String symbolicName) { + dto.setSymbolicName(symbolicName); + return this; + } + + @Deprecated + public CatalogItemBuilder<CatalogItemType> javaType(String javaType) { + dto.setJavaType(javaType); + return this; + } + + /** @deprecated since 0.7.0 use {@link #displayName}*/ + @Deprecated + public CatalogItemBuilder<CatalogItemType> name(String name) { + return displayName(name); + } + + public CatalogItemBuilder<CatalogItemType> displayName(String displayName) { + dto.setDisplayName(displayName); + return this; + } + + public CatalogItemBuilder<CatalogItemType> description(String description) { + dto.setDescription(description); + return this; + } + + public CatalogItemBuilder<CatalogItemType> iconUrl(String iconUrl) { + dto.setIconUrl(iconUrl); + return this; + } + + public CatalogItemBuilder<CatalogItemType> version(String version) { + dto.setVersion(version); + return this; + } + + public CatalogItemBuilder<CatalogItemType> deprecated(boolean deprecated) { + dto.setDeprecated(deprecated); + return this; + } + + public CatalogItemBuilder<CatalogItemType> libraries(Collection<CatalogBundle> libraries) { + dto.setLibraries(libraries); + return this; + } + + public CatalogItemBuilder<CatalogItemType> plan(String yaml) { + dto.setPlanYaml(yaml); + return this; + } + + public CatalogItemType build() { + Preconditions.checkNotNull(dto.getSymbolicName()); + Preconditions.checkNotNull(dto.getVersion()); + + if (dto.getLibraries() == null) { + dto.setLibraries(Collections.<CatalogBundle>emptyList()); + } + + CatalogItemType ret = dto; + + //prevent mutations through the builder + dto = null; + + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/6caee589/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemComparator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemComparator.java b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemComparator.java new file mode 100644 index 0000000..6832503 --- /dev/null +++ b/core/src/main/java/org/apache/brooklyn/core/catalog/internal/CatalogItemComparator.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.core.catalog.internal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.brooklyn.api.catalog.CatalogItem; + +import brooklyn.util.text.NaturalOrderComparator; + +/** + * Largest version first order. + * + * When using the comparator to sort - first using symbolicName + * and if equal puts larger versions first, snapshots at the back. + */ +public class CatalogItemComparator<T,SpecT> implements Comparator<CatalogItem<T, SpecT>> { + private static final String SNAPSHOT = "SNAPSHOT"; + + public static final CatalogItemComparator<?, ?> INSTANCE = new CatalogItemComparator<Object, Object>(); + + @SuppressWarnings("unchecked") + public static <T,SpecT> CatalogItemComparator<T,SpecT> getInstance() { + return (CatalogItemComparator<T, SpecT>) INSTANCE; + } + + @Override + public int compare(CatalogItem<T, SpecT> o1, CatalogItem<T, SpecT> o2) { + int symbolicNameComparison = o1.getSymbolicName().compareTo(o2.getSymbolicName()); + if (symbolicNameComparison != 0) { + return symbolicNameComparison; + } else { + String v1 = o1.getVersion(); + String v2 = o2.getVersion(); + + boolean isV1Snapshot = v1.toUpperCase().contains(SNAPSHOT); + boolean isV2Snapshot = v2.toUpperCase().contains(SNAPSHOT); + if (isV1Snapshot == isV2Snapshot) { + String[] v1Parts = split(v1); + String[] v2Parts = split(v2); + return -compare(v1Parts, v2Parts); + } else if (isV1Snapshot) { + return 1; + } else { + return -1; + } + } + } + + private String[] split(String v) { + Collection<String> parts = new ArrayList<String>(); + int startPos = 0; + int delimPos; + while ((delimPos = v.indexOf('.', startPos)) != -1) { + String part = v.substring(startPos, delimPos); + if (parse(part) != -1) { + parts.add(part); + } else { + break; + } + startPos = delimPos+1; + } + String remaining = v.substring(startPos); + parts.addAll(Arrays.asList(remaining.split("[^\\d]", 2))); + return parts.toArray(new String[parts.size()]); + } + + private int compare(String[] v1Parts, String[] v2Parts) { + int len = Math.max(v1Parts.length, v2Parts.length); + for (int i = 0; i < len; i++) { + if (i == v1Parts.length) { + return isNumber(v2Parts[i]) ? -1 : 1; + } + if (i == v2Parts.length) { + return isNumber(v1Parts[i]) ? 1 : -1; + } + + String p1 = v1Parts[i]; + String p2 = v2Parts[i]; + int n1 = parse(p1); + int n2 = parse(p2); + if (n1 != -1 && n2 != -1) { + if (n1 != n2) { + return compare(n1, n2); + } + } else if (n1 == -1 && n2 != -1) { + return -1; + } else if (n1 != -1 && n2 == -1) { + return 1; + } else { + int cmp = NaturalOrderComparator.INSTANCE.compare(p1, p2); + if (cmp < 0) { + return -1; + } else if (cmp > 0) { + return 1; + } + } + } + return 0; + } + + private boolean isNumber(String v) { + return parse(v) != -1; + } + + //Replace with Integer.compare in J7 + private int compare(int n1, int n2) { + if (n1 == n2) { + return 0; + } else if (n1 < n2) { + return -1; + } else { + return 1; + } + } + + private int parse(String p) { + try { + return Integer.parseInt(p); + } catch (NumberFormatException e) { + return -1; + } + } +}
