This is an automated email from the ASF dual-hosted git repository. rombert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/sling-org-apache-sling-distribution-avro-serializer.git
commit 23e0d7acb72bbbe6ba70a45418700b7cd1eaa340 Author: Tommaso Teofili <[email protected]> AuthorDate: Thu Feb 23 15:28:47 2017 +0000 SLING-6325 - split Avro and Kryo serializers into own bundles git-svn-id: https://svn.apache.org/repos/asf/sling/trunk@1784154 13f79535-47bb-0310-9956-ffa450edef68 --- pom.xml | 340 +++++++++++++++++++ src/main/avro/shallowresource.avsc | 11 + .../impl/avro/AvroContentSerializer.java | 234 +++++++++++++ .../AvroDistributionContentSerializerFactory.java | 91 +++++ .../impl/avro/AvroShallowResource.java | 376 +++++++++++++++++++++ .../impl/avro/AvroContentSerializerTest.java | 177 ++++++++++ src/test/resources/avro/dp.avro | Bin 0 -> 564 bytes 7 files changed, 1229 insertions(+) diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5ae5a3b --- /dev/null +++ b/pom.xml @@ -0,0 +1,340 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <!-- ======================================================================= --> + <!-- P A R E N T P R O J E C T --> + <!-- ======================================================================= --> + <parent> + <groupId>org.apache.sling</groupId> + <artifactId>sling</artifactId> + <version>26</version> + </parent> + + <!-- ======================================================================= --> + <!-- P R O J E C T --> + <!-- ======================================================================= --> + <artifactId>org.apache.sling.distribution.avro-serializer</artifactId> + <version>0.0.9-SNAPSHOT</version> + <packaging>bundle</packaging> + + <name>Apache Sling Distribution Avro Serializer</name> + <description> + The Apache Sling Distribution Avro Serializer extensions bundle provides an Avro serialization implementation for Sling Content Distribution + </description> + + <scm> + <connection>scm:svn:http://svn.apache.org/repos/asf/sling/trunk/contrib/extensions/distribution/avro-serializer</connection> + <developerConnection>scm:svn:https://svn.apache.org/repos/asf/sling/trunk/contrib/extensions/distribution/avro-serializer</developerConnection> + <url>http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/distribution/avro-serializer</url> + </scm> + + <!-- ======================================================================= --> + <!-- B U I L D --> + <!-- ======================================================================= --> + <build> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/avro</directory> + </resource> + </resources> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>**/*.avsc</exclude> + <exclude>**/*.avro</exclude> + </excludes> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.avro</groupId> + <artifactId>avro-maven-plugin</artifactId> + <version>1.7.7</version> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>schema</goal> + </goals> + <configuration> + <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory> + <outputDirectory>${project.basedir}/src/main/java/</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-scr-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.sling</groupId> + <artifactId>maven-sling-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <instructions> + <Bundle-SymbolicName>org.apache.sling.distribution.extensions</Bundle-SymbolicName> + <Embed-Dependency>avro,avro-ipc, + jackson-core-asl, + jackson-mapper-asl, + paranamer, + commons-compress, + org.apache.sling.jcr.resource;inline="org/apache/sling/jcr/resource/internal/helper/jcr/JcrNodeResource.*| + org/apache/sling/jcr/resource/internal/helper/jcr/JcrItemResource.*| + org/apache/sling/jcr/resource/internal/helper/LazyInputStream.*| + org/apache/sling/jcr/resource/internal/HelperData*| + org/apache/sling/jcr/resource/internal/JcrModifiableValueMap*|" + org/apache/sling/jcr/resource/internal/JcrValueMap*|" + org/apache/sling/jcr/resource/internal/helper/jcr/PathMapper*|" + org/apache/sling/jcr/resource/internal/helper/JcrPropertyMapCacheEntry*" + </Embed-Dependency> + <Import-Package> + !sun.*, + !org.joda.time.*, + !org.tukaani.xz, + !org.xerial.snappy, + !sun.misc, + !javax.inject, + !org.apache.velocity.*, + !org.jboss.netty.*, + !org.mortbay.jetty.*, + !org.mortbay.resource.*, + !org.apache.sling.api.resource, + !org.apache.sling.distribution.component.*, + !org.apache.sling.distribution.packaging.*, + !org.apache.sling.distribution.serialization.impl.vlt.*, + * + </Import-Package> + <DynamicImport-Package> + org.apache.sling.api.resource + </DynamicImport-Package> + </instructions> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + </plugin> + </plugins> + </build> + + <!-- ======================================================================= --> + <!-- D E P E N D E N C I E S --> + <!-- ======================================================================= --> + <dependencies> + <!-- TESTING --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>1.9.5</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>jackrabbit-spi-commons</artifactId> + <version>2.6.4</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.testing.resourceresolver-mock</artifactId> + <version>1.1.12</version> + <scope>test</scope> + </dependency> + <!-- SLING --> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.api</artifactId> + <version>2.7.0</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.distribution.api</artifactId> + <version>0.3.0</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.distribution.core</artifactId> + <version>0.2.5-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.commons.osgi</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.commons.scheduler</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.jcr.api</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.event</artifactId> + <version>3.3.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.hc.core</artifactId> + <version>1.0.6</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.settings</artifactId> + <version>1.3.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.commons.json</artifactId> + <version>2.0.8</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.event.dea</artifactId> + <version>1.0.0</version> + <scope>provided</scope> + </dependency> + <!-- LOGGING --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.6.2</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>1.6.2</version> + <scope>runtime</scope> + </dependency> + <!-- SPECs --> + <dependency> + <groupId>javax.jcr</groupId> + <artifactId>jcr</artifactId> + <version>2.0</version> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.core</artifactId> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.compendium</artifactId> + </dependency> + <!-- JACKRABBIT --> + <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>jackrabbit-jcr-commons</artifactId> + <version>2.6.2</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.jackrabbit</groupId> + <artifactId>jackrabbit-api</artifactId> + <version>2.6.2</version> + </dependency> + <!-- COMMONS --> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + <version>2.0.0</version> + <scope>provided</scope> + </dependency> + + <!-- avro --> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.7.7</version> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-ipc</artifactId> + <version>1.7.7</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + <version>1.9.13</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <version>1.9.13</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.thoughtworks.paranamer</groupId> + <artifactId>paranamer</artifactId> + <version>2.3</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <version>1.4.1</version> + <scope>provided</scope> + </dependency> + + </dependencies> + <reporting> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + <version>2.5.3</version> + </plugin> + </plugins> + </reporting> +</project> diff --git a/src/main/avro/shallowresource.avsc b/src/main/avro/shallowresource.avsc new file mode 100644 index 0000000..3430342 --- /dev/null +++ b/src/main/avro/shallowresource.avsc @@ -0,0 +1,11 @@ +{"namespace": "org.apache.sling.distribution.serialization.impl.avro", + "type": "record", + "name": "AvroShallowResource", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "valueMap", "type": {"type": "map", "values": ["int","long","float","double","string","boolean","bytes",{"type":"array","items":["int","long","float","double","string","boolean","bytes"]}]}}, + {"name": "path", "type": ["string", "null"]}, + {"name": "children", "type": [{"type": "array", "items": "AvroShallowResource"},"null"]}, + {"name": "resourceType", "type": ["string", "null"]} + ] +} \ No newline at end of file diff --git a/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializer.java b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializer.java new file mode 100644 index 0000000..5c21d8d --- /dev/null +++ b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializer.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sling.distribution.serialization.impl.avro; + +import javax.annotation.Nonnull; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collection; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.file.SeekableByteArrayInput; +import org.apache.avro.generic.GenericData; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.util.Utf8; +import org.apache.commons.io.IOUtils; +import org.apache.sling.api.resource.PersistenceException; +import org.apache.sling.api.resource.Resource; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.api.resource.ValueMap; +import org.apache.sling.distribution.common.DistributionException; +import org.apache.sling.distribution.serialization.DistributionContentSerializer; +import org.apache.sling.distribution.serialization.DistributionExportFilter; +import org.apache.sling.distribution.serialization.DistributionExportOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Apache Avro based {@link DistributionContentSerializer} + */ +public class AvroContentSerializer implements DistributionContentSerializer { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final String name; + private final Schema schema; + private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss.sss+hh:mm"); + + public AvroContentSerializer(String name) { + try { + schema = new Schema.Parser().parse(getClass().getResourceAsStream("/shallowresource.avsc")); + } catch (IOException e) { + throw new RuntimeException(e); + } + this.name = name; + } + + @Override + public void exportToStream(ResourceResolver resourceResolver, DistributionExportOptions options, OutputStream outputStream) throws DistributionException { + + DatumWriter<AvroShallowResource> datumWriter = new SpecificDatumWriter<AvroShallowResource>(AvroShallowResource.class); + DataFileWriter<AvroShallowResource> writer = new DataFileWriter<AvroShallowResource>(datumWriter); + try { + writer.create(schema, outputStream); + } catch (IOException e) { + throw new DistributionException(e); + } + + try { + DistributionExportFilter filter = options.getFilter(); + for (DistributionExportFilter.TreeFilter treeFilter : filter.getNodeFilters()) { + String path = treeFilter.getPath(); + Resource resource = resourceResolver.getResource(path); + AvroShallowResource avroShallowResource = getAvroShallowResource(treeFilter, filter.getPropertyFilter(), + resource); + writer.append(avroShallowResource); + } + outputStream.flush(); + } catch (Exception e) { + throw new DistributionException(e); + } finally { + try { + writer.close(); + } catch (IOException e) { + // do nothing + } + } + } + + @Override + public void importFromStream(ResourceResolver resourceResolver, InputStream stream) throws DistributionException { + try { + byte[] bin = IOUtils.toByteArray(stream); // TODO : avoid byte[] conversion + Collection<AvroShallowResource> avroShallowResources = readAvroResources(bin); + for (AvroShallowResource ar : avroShallowResources) { + persistResource(resourceResolver, ar); + } + resourceResolver.commit(); + } catch (Exception e) { + throw new DistributionException(e); + } + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isRequestFiltering() { + return false; + } + + private AvroShallowResource getAvroShallowResource(DistributionExportFilter.TreeFilter nodeFilter, + DistributionExportFilter.TreeFilter propertyFilter, + Resource resource) throws IOException { + AvroShallowResource avroShallowResource = new AvroShallowResource(); + avroShallowResource.setName("avro_" + System.nanoTime()); + avroShallowResource.setPath(resource.getPath()); + avroShallowResource.setResourceType(resource.getResourceType()); + ValueMap valueMap = resource.getValueMap(); + Map<CharSequence, Object> map = new HashMap<CharSequence, Object>(); + for (Map.Entry<String, Object> entry : valueMap.entrySet()) { + String property = entry.getKey(); + if (propertyFilter.matches(property)) { + Object value = entry.getValue(); + if (value instanceof GregorianCalendar) { + value = dateFormat.format(((GregorianCalendar) value).getTime()); + } else if (value instanceof Object[]) { + Object[] ar = (Object[]) value; + value = Arrays.asList(ar); + } else if (value instanceof InputStream) { + value = ByteBuffer.wrap(IOUtils.toByteArray(((InputStream) value))); + } + map.put(property, value); + } + } + avroShallowResource.setValueMap(map); + List<AvroShallowResource> children = new LinkedList<AvroShallowResource>(); + for (Resource child : resource.getChildren()) { + if (nodeFilter.matches(child.getPath())) { + children.add(getAvroShallowResource(nodeFilter, propertyFilter, child)); + } + } + avroShallowResource.setChildren(children); + return avroShallowResource; + } + + private Collection<AvroShallowResource> readAvroResources(byte[] bytes) throws IOException { + DatumReader<AvroShallowResource> datumReader = new SpecificDatumReader<AvroShallowResource>(AvroShallowResource.class); + DataFileReader<AvroShallowResource> dataFileReader = new DataFileReader<AvroShallowResource>(new SeekableByteArrayInput(bytes), datumReader); + Collection<AvroShallowResource> avroResources = new LinkedList<AvroShallowResource>(); + try { + for (AvroShallowResource avroResource : dataFileReader) { + avroResources.add(avroResource); + } + } finally { + dataFileReader.close(); + } + return avroResources; + } + + private void persistResource(@Nonnull ResourceResolver resourceResolver, AvroShallowResource r) throws PersistenceException { + String path = r.getPath().toString().trim(); + String name = path.substring(path.lastIndexOf('/') + 1); + String substring = path.substring(0, path.lastIndexOf('/')); + String parentPath = substring.length() == 0 ? "/" : substring; + Map<String, Object> map = new HashMap<String, Object>(); + Map<CharSequence, Object> valueMap = r.getValueMap(); + for (Map.Entry<CharSequence, Object> entry : valueMap.entrySet()) { + Object value = entry.getValue(); + if (value instanceof GenericData.Array) { + GenericData.Array array = (GenericData.Array) value; + String[] s = new String[array.size()]; + for (int i = 0; i < s.length; i++) { + Object gd = array.get(i); + s[i] = gd.toString(); + } + value = s; + } else if (value instanceof Utf8) { + value = value.toString(); + } else if (value instanceof ByteBuffer) { + byte[] bytes = ((ByteBuffer) value).array(); + value = new BufferedInputStream(new ByteArrayInputStream(bytes)); + } + map.put(entry.getKey().toString(), value); + } + Resource existingResource = resourceResolver.getResource(path); + if (existingResource != null) { + resourceResolver.delete(existingResource); + } + Resource parent = resourceResolver.getResource(parentPath); + if (parent == null) { + parent = createParent(resourceResolver, parentPath); + } + Resource createdResource = resourceResolver.create(parent, name, map); + log.debug("created resource {}", createdResource); + for (AvroShallowResource child : r.getChildren()) { + persistResource(createdResource.getResourceResolver(), child); + } + } + + private Resource createParent(ResourceResolver resourceResolver, String path) throws PersistenceException { + String parentPath = path.substring(0, path.lastIndexOf('/')); + String name = path.substring(path.lastIndexOf('/') + 1); + Resource parentResource = resourceResolver.getResource(parentPath); + if (parentResource == null) { + parentResource = createParent(resourceResolver, parentPath); + } + Map<String, Object> properties = new HashMap<String, Object>(); + return resourceResolver.create(parentResource, name, properties); + } +} diff --git a/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroDistributionContentSerializerFactory.java b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroDistributionContentSerializerFactory.java new file mode 100644 index 0000000..351a1b7 --- /dev/null +++ b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroDistributionContentSerializerFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sling.distribution.serialization.impl.avro; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; + +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.Service; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.osgi.PropertiesUtil; +import org.apache.sling.distribution.common.DistributionException; +import org.apache.sling.distribution.serialization.DistributionContentSerializer; +import org.apache.sling.distribution.serialization.DistributionExportOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Factory for {@link DistributionContentSerializer}s based on Apache Avro. + */ +@Component(metatype = true, + label = "Apache Sling Distribution Packaging - Avro Serialization Format Factory", + description = "OSGi configuration for Avro serializers", + configurationFactory = true, + specVersion = "1.1", + policy = ConfigurationPolicy.REQUIRE +) +@Service(DistributionContentSerializer.class) +@Property(name = "webconsole.configurationFactory.nameHint", value = "Content serializer name: {name}") +public class AvroDistributionContentSerializerFactory implements DistributionContentSerializer { + + /** + * name of this package builder. + */ + @Property(label = "Name", description = "The name of the avro format.") + public static final String NAME = "name"; + + private DistributionContentSerializer format; + + private final Logger log = LoggerFactory.getLogger(getClass()); + + @Activate + public void activate(Map<String, Object> config) { + + String name = PropertiesUtil.toString(config.get(NAME), null); + + format = new AvroContentSerializer(name); + log.info("started avro content serializer {}", name); + } + + + @Override + public void exportToStream(ResourceResolver resourceResolver, DistributionExportOptions options, OutputStream outputStream) throws DistributionException { + format.exportToStream(resourceResolver, options, outputStream); + } + + @Override + public void importFromStream(ResourceResolver resourceResolver, InputStream stream) throws DistributionException { + format.importFromStream(resourceResolver, stream); + } + + @Override + public String getName() { + return format.getName(); + } + + @Override + public boolean isRequestFiltering() { + return format.isRequestFiltering(); + } +} diff --git a/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroShallowResource.java b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroShallowResource.java new file mode 100644 index 0000000..1139e24 --- /dev/null +++ b/src/main/java/org/apache/sling/distribution/serialization/impl/avro/AvroShallowResource.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sling.distribution.serialization.impl.avro; +@SuppressWarnings("all") [email protected] +public class AvroShallowResource extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"AvroShallowResource\",\"namespace\":\"org.apache.sling.distribution.serialization.impl.avro\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"valueMap\",\"type\":{\"type\":\"map\",\"values\":[\"int\",\"long\",\"float\",\"double\",\"string\",\"boolean\",\"bytes\",{\"type\":\"array\",\"items\":[\"int\",\"long\",\"float\",\"double\",\"string\",\"b [...] + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + @Deprecated + public java.lang.CharSequence name; + @Deprecated + public java.util.Map<java.lang.CharSequence,java.lang.Object> valueMap; + @Deprecated + public java.lang.CharSequence path; + @Deprecated + public java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> children; + @Deprecated + public java.lang.CharSequence resourceType; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use <code>newBuilder()</code>. + */ + public AvroShallowResource() {} + + /** + * All-args constructor. + */ + public AvroShallowResource(java.lang.CharSequence name, java.util.Map<java.lang.CharSequence,java.lang.Object> valueMap, java.lang.CharSequence path, java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> children, java.lang.CharSequence resourceType) { + this.name = name; + this.valueMap = valueMap; + this.path = path; + this.children = children; + this.resourceType = resourceType; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return name; + case 1: return valueMap; + case 2: return path; + case 3: return children; + case 4: return resourceType; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: name = (java.lang.CharSequence)value$; break; + case 1: valueMap = (java.util.Map<java.lang.CharSequence,java.lang.Object>)value$; break; + case 2: path = (java.lang.CharSequence)value$; break; + case 3: children = (java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource>)value$; break; + case 4: resourceType = (java.lang.CharSequence)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'name' field. + */ + public java.lang.CharSequence getName() { + return name; + } + + /** + * Sets the value of the 'name' field. + * @param value the value to set. + */ + public void setName(java.lang.CharSequence value) { + this.name = value; + } + + /** + * Gets the value of the 'valueMap' field. + */ + public java.util.Map<java.lang.CharSequence,java.lang.Object> getValueMap() { + return valueMap; + } + + /** + * Sets the value of the 'valueMap' field. + * @param value the value to set. + */ + public void setValueMap(java.util.Map<java.lang.CharSequence,java.lang.Object> value) { + this.valueMap = value; + } + + /** + * Gets the value of the 'path' field. + */ + public java.lang.CharSequence getPath() { + return path; + } + + /** + * Sets the value of the 'path' field. + * @param value the value to set. + */ + public void setPath(java.lang.CharSequence value) { + this.path = value; + } + + /** + * Gets the value of the 'children' field. + */ + public java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> getChildren() { + return children; + } + + /** + * Sets the value of the 'children' field. + * @param value the value to set. + */ + public void setChildren(java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> value) { + this.children = value; + } + + /** + * Gets the value of the 'resourceType' field. + */ + public java.lang.CharSequence getResourceType() { + return resourceType; + } + + /** + * Sets the value of the 'resourceType' field. + * @param value the value to set. + */ + public void setResourceType(java.lang.CharSequence value) { + this.resourceType = value; + } + + /** Creates a new AvroShallowResource RecordBuilder */ + public static org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder newBuilder() { + return new org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder(); + } + + /** Creates a new AvroShallowResource RecordBuilder by copying an existing Builder */ + public static org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder newBuilder(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder other) { + return new org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder(other); + } + + /** Creates a new AvroShallowResource RecordBuilder by copying an existing AvroShallowResource instance */ + public static org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder newBuilder(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource other) { + return new org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder(other); + } + + /** + * RecordBuilder for AvroShallowResource instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<AvroShallowResource> + implements org.apache.avro.data.RecordBuilder<AvroShallowResource> { + + private java.lang.CharSequence name; + private java.util.Map<java.lang.CharSequence,java.lang.Object> valueMap; + private java.lang.CharSequence path; + private java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> children; + private java.lang.CharSequence resourceType; + + /** Creates a new Builder */ + private Builder() { + super(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.SCHEMA$); + } + + /** Creates a Builder by copying an existing Builder */ + private Builder(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder other) { + super(other); + if (isValidValue(fields()[0], other.name)) { + this.name = data().deepCopy(fields()[0].schema(), other.name); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.valueMap)) { + this.valueMap = data().deepCopy(fields()[1].schema(), other.valueMap); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.path)) { + this.path = data().deepCopy(fields()[2].schema(), other.path); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.children)) { + this.children = data().deepCopy(fields()[3].schema(), other.children); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.resourceType)) { + this.resourceType = data().deepCopy(fields()[4].schema(), other.resourceType); + fieldSetFlags()[4] = true; + } + } + + /** Creates a Builder by copying an existing AvroShallowResource instance */ + private Builder(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource other) { + super(org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.SCHEMA$); + if (isValidValue(fields()[0], other.name)) { + this.name = data().deepCopy(fields()[0].schema(), other.name); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.valueMap)) { + this.valueMap = data().deepCopy(fields()[1].schema(), other.valueMap); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.path)) { + this.path = data().deepCopy(fields()[2].schema(), other.path); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.children)) { + this.children = data().deepCopy(fields()[3].schema(), other.children); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.resourceType)) { + this.resourceType = data().deepCopy(fields()[4].schema(), other.resourceType); + fieldSetFlags()[4] = true; + } + } + + /** Gets the value of the 'name' field */ + public java.lang.CharSequence getName() { + return name; + } + + /** Sets the value of the 'name' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder setName(java.lang.CharSequence value) { + validate(fields()[0], value); + this.name = value; + fieldSetFlags()[0] = true; + return this; + } + + /** Checks whether the 'name' field has been set */ + public boolean hasName() { + return fieldSetFlags()[0]; + } + + /** Clears the value of the 'name' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder clearName() { + name = null; + fieldSetFlags()[0] = false; + return this; + } + + /** Gets the value of the 'valueMap' field */ + public java.util.Map<java.lang.CharSequence,java.lang.Object> getValueMap() { + return valueMap; + } + + /** Sets the value of the 'valueMap' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder setValueMap(java.util.Map<java.lang.CharSequence,java.lang.Object> value) { + validate(fields()[1], value); + this.valueMap = value; + fieldSetFlags()[1] = true; + return this; + } + + /** Checks whether the 'valueMap' field has been set */ + public boolean hasValueMap() { + return fieldSetFlags()[1]; + } + + /** Clears the value of the 'valueMap' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder clearValueMap() { + valueMap = null; + fieldSetFlags()[1] = false; + return this; + } + + /** Gets the value of the 'path' field */ + public java.lang.CharSequence getPath() { + return path; + } + + /** Sets the value of the 'path' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder setPath(java.lang.CharSequence value) { + validate(fields()[2], value); + this.path = value; + fieldSetFlags()[2] = true; + return this; + } + + /** Checks whether the 'path' field has been set */ + public boolean hasPath() { + return fieldSetFlags()[2]; + } + + /** Clears the value of the 'path' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder clearPath() { + path = null; + fieldSetFlags()[2] = false; + return this; + } + + /** Gets the value of the 'children' field */ + public java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> getChildren() { + return children; + } + + /** Sets the value of the 'children' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder setChildren(java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource> value) { + validate(fields()[3], value); + this.children = value; + fieldSetFlags()[3] = true; + return this; + } + + /** Checks whether the 'children' field has been set */ + public boolean hasChildren() { + return fieldSetFlags()[3]; + } + + /** Clears the value of the 'children' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder clearChildren() { + children = null; + fieldSetFlags()[3] = false; + return this; + } + + /** Gets the value of the 'resourceType' field */ + public java.lang.CharSequence getResourceType() { + return resourceType; + } + + /** Sets the value of the 'resourceType' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder setResourceType(java.lang.CharSequence value) { + validate(fields()[4], value); + this.resourceType = value; + fieldSetFlags()[4] = true; + return this; + } + + /** Checks whether the 'resourceType' field has been set */ + public boolean hasResourceType() { + return fieldSetFlags()[4]; + } + + /** Clears the value of the 'resourceType' field */ + public org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource.Builder clearResourceType() { + resourceType = null; + fieldSetFlags()[4] = false; + return this; + } + + @Override + public AvroShallowResource build() { + try { + AvroShallowResource record = new AvroShallowResource(); + record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]); + record.valueMap = fieldSetFlags()[1] ? this.valueMap : (java.util.Map<java.lang.CharSequence,java.lang.Object>) defaultValue(fields()[1]); + record.path = fieldSetFlags()[2] ? this.path : (java.lang.CharSequence) defaultValue(fields()[2]); + record.children = fieldSetFlags()[3] ? this.children : (java.util.List<org.apache.sling.distribution.serialization.impl.avro.AvroShallowResource>) defaultValue(fields()[3]); + record.resourceType = fieldSetFlags()[4] ? this.resourceType : (java.lang.CharSequence) defaultValue(fields()[4]); + return record; + } catch (Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } +} diff --git a/src/test/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializerTest.java b/src/test/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializerTest.java new file mode 100644 index 0000000..f8d4811 --- /dev/null +++ b/src/test/java/org/apache/sling/distribution/serialization/impl/avro/AvroContentSerializerTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sling.distribution.serialization.impl.avro; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; + +import org.apache.sling.api.resource.Resource; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.distribution.DistributionRequest; +import org.apache.sling.distribution.DistributionRequestType; +import org.apache.sling.distribution.SimpleDistributionRequest; +import org.apache.sling.distribution.packaging.DistributionPackage; +import org.apache.sling.distribution.packaging.DistributionPackageBuilder; +import org.apache.sling.distribution.packaging.impl.FileDistributionPackageBuilder; +import org.apache.sling.distribution.serialization.DistributionContentSerializer; +import org.apache.sling.distribution.serialization.DistributionExportFilter; +import org.apache.sling.distribution.serialization.DistributionExportOptions; +import org.apache.sling.testing.resourceresolver.MockHelper; +import org.apache.sling.testing.resourceresolver.MockResourceResolverFactory; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Tests for {@link AvroContentSerializer} + */ +public class AvroContentSerializerTest { + + private MockHelper helper; + private ResourceResolver resourceResolver; + + @Before + public void setUp() throws Exception { + resourceResolver = new MockResourceResolverFactory().getResourceResolver(null); + helper = MockHelper.create(resourceResolver).resource("/libs").p("prop", "value") + .resource("sub").p("sub", "hello") + .resource(".sameLevel") + .resource("/apps").p("foo", "baa"); + helper.commit(); + } + + @Test + public void testExtractDeep() throws Exception { + AvroContentSerializer avroContentSerializer = new AvroContentSerializer("avro"); + DistributionRequest request = new SimpleDistributionRequest(DistributionRequestType.ADD, true, "/libs"); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + NavigableMap<String, List<String>> nodeFilters = new TreeMap<String, List<String>>(); + NavigableMap<String, List<String>> propertyFilters = new TreeMap<String, List<String>>(); + try { + DistributionExportFilter filter = DistributionExportFilter.createFilter(request, nodeFilters, propertyFilters); + avroContentSerializer.exportToStream(resourceResolver, new DistributionExportOptions(request, filter), outputStream); + byte[] bytes = outputStream.toByteArray(); + assertNotNull(bytes); + assertTrue(bytes.length > 0); + } finally { + outputStream.close(); + } + } + + @Test + public void testExtractShallow() throws Exception { + AvroContentSerializer avroContentSerializer = new AvroContentSerializer("avro"); + DistributionRequest request = new SimpleDistributionRequest(DistributionRequestType.ADD, "/libs"); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + NavigableMap<String, List<String>> nodeFilters = new TreeMap<String, List<String>>(); + NavigableMap<String, List<String>> propertyFilters = new TreeMap<String, List<String>>(); + try { + DistributionExportFilter filter = DistributionExportFilter.createFilter(request, nodeFilters, propertyFilters); + avroContentSerializer.exportToStream(resourceResolver, new DistributionExportOptions(request, filter), outputStream); + byte[] bytes = outputStream.toByteArray(); + assertNotNull(bytes); + assertTrue(bytes.length > 0); + } finally { + outputStream.close(); + } + } + + @Test + public void testImport() throws Exception { + AvroContentSerializer avroContentSerializer = new AvroContentSerializer("avro"); + InputStream inputStream = getClass().getResourceAsStream("/avro/dp.avro"); + avroContentSerializer.importFromStream(resourceResolver, inputStream); + } + + @Test + public void testBuildAndInstallOnSingleDeepPath() throws Exception { + String type = "avro"; + DistributionContentSerializer contentSerializer = new AvroContentSerializer(type); + String tempFilesFolder = "target"; + String[] nodeFilters = new String[0]; + String[] propertyFilters = new String[0]; + DistributionPackageBuilder packageBuilder = new FileDistributionPackageBuilder(type, contentSerializer, + tempFilesFolder, null, nodeFilters, propertyFilters); + DistributionRequest request = new SimpleDistributionRequest(DistributionRequestType.ADD, true, "/libs"); + DistributionPackage distributionPackage = packageBuilder.createPackage(resourceResolver, request); + + Resource resource = resourceResolver.getResource("/libs/sub"); + resourceResolver.delete(resource); + resourceResolver.commit(); + + assertTrue(packageBuilder.installPackage(resourceResolver, distributionPackage)); + + assertNotNull(resourceResolver.getResource("/libs")); + assertNotNull(resourceResolver.getResource("/libs/sub")); + assertNotNull(resourceResolver.getResource("/libs/sameLevel")); + } + + @Test + public void testBuildAndInstallOnSingleShallowPath() throws Exception { + String type = "avro"; + DistributionContentSerializer contentSerializer = new AvroContentSerializer(type); + String tempFilesFolder = "target"; + String[] nodeFilters = new String[0]; + String[] propertyFilters = new String[0]; + DistributionPackageBuilder packageBuilder = new FileDistributionPackageBuilder(type, contentSerializer, + tempFilesFolder, null, nodeFilters, propertyFilters); + DistributionRequest request = new SimpleDistributionRequest(DistributionRequestType.ADD, "/libs/sub"); + DistributionPackage distributionPackage = packageBuilder.createPackage(resourceResolver, request); + + Resource resource = resourceResolver.getResource("/libs/sub"); + resourceResolver.delete(resource); + resourceResolver.commit(); + + assertTrue(packageBuilder.installPackage(resourceResolver, distributionPackage)); + + assertNotNull(resourceResolver.getResource("/libs")); + assertNotNull(resourceResolver.getResource("/libs/sub")); + assertNotNull(resourceResolver.getResource("/libs/sameLevel")); + } + + @Test + public void testBuildAndInstallOnMultipleShallowPaths() throws Exception { + String type = "avro"; + DistributionContentSerializer contentSerializer = new AvroContentSerializer(type); + String tempFilesFolder = "target"; + String[] nodeFilters = new String[0]; + String[] propertyFilters = new String[0]; + DistributionPackageBuilder packageBuilder = new FileDistributionPackageBuilder(type, contentSerializer, + tempFilesFolder, null, nodeFilters, propertyFilters); + DistributionRequest request = new SimpleDistributionRequest(DistributionRequestType.ADD, "/libs/sub", "/libs/sameLevel"); + DistributionPackage distributionPackage = packageBuilder.createPackage(resourceResolver, request); + + Resource resource = resourceResolver.getResource("/libs/sub"); + resourceResolver.delete(resource); + resource = resourceResolver.getResource("/libs/sameLevel"); + resourceResolver.delete(resource); + resourceResolver.commit(); + + assertTrue(packageBuilder.installPackage(resourceResolver, distributionPackage)); + + assertNotNull(resourceResolver.getResource("/libs")); + assertNotNull(resourceResolver.getResource("/libs/sub")); + assertNotNull(resourceResolver.getResource("/libs/sameLevel")); + } +} diff --git a/src/test/resources/avro/dp.avro b/src/test/resources/avro/dp.avro new file mode 100644 index 0000000..51c61a2 Binary files /dev/null and b/src/test/resources/avro/dp.avro differ -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
