Author: npeltier Date: Tue Sep 12 09:32:00 2017 New Revision: 1808091 URL: http://svn.apache.org/viewvc?rev=1808091&view=rev Log: SLING-7099 introducing csv pipe
- adding also unit test (using repository based input stream for now), - adding PipeBuilder api, and fixing javadoc issues Added: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv Modified: sling/trunk/contrib/extensions/sling-pipes/pom.xml sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java Modified: sling/trunk/contrib/extensions/sling-pipes/pom.xml URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/pom.xml?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/pom.xml (original) +++ sling/trunk/contrib/extensions/sling-pipes/pom.xml Tue Sep 12 09:32:00 2017 @@ -96,6 +96,16 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <!-- test csv files can't have licenses embedded --> + <exclude>src/test/resources/**/*.csv</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java (original) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java Tue Sep 12 09:32:00 2017 @@ -30,8 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -40,13 +38,14 @@ import java.util.regex.Pattern; /** * Input Stream based pipe, coming from web, from request, resource tree, web + * binding is updated by the returned iterator */ public abstract class AbstractInputStreamPipe extends BasePipe { private static Logger LOGGER = LoggerFactory.getLogger(AbstractInputStreamPipe.class); public final String REMOTE_START = "http"; - protected final Pattern VALID_PATH = Pattern.compile("/([\\w\\d]+/)+[\\w\\d]+"); + protected final Pattern VALID_PATH = Pattern.compile("/([\\w\\d\\.-_]+/)+[\\w\\d\\.-_]+"); public static final Object BINDING_IS = "org.apache.sling.pipes.RequestInputStream"; @@ -92,15 +91,14 @@ public abstract class AbstractInputStrea } } else if (VALID_PATH.matcher(expr).find() && resolver.getResource(expr) != null - && resolver.getResource(expr).adaptTo(File.class) != null) { - return new FileInputStream(resolver.getResource(expr).adaptTo(File.class)); + && resolver.getResource(expr).adaptTo(InputStream.class) != null) { + return resolver.getResource(expr).adaptTo(InputStream.class); } else if (getBindings().getBindings().get(BINDING_IS) != null){ return (InputStream)getBindings().getBindings().get(BINDING_IS); } return new ByteArrayInputStream(expr.getBytes(StandardCharsets.UTF_8)); } - @Override public Object getOutputBinding() { return binding; Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java (original) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java Tue Sep 12 09:32:00 2017 @@ -87,8 +87,15 @@ public interface PipeBuilder { PipeBuilder rm(); /** + * attach a csv pipe to the current context + * @param expr csv expr or URL or path in the resource tree + * @return updated instance of PipeBuilder + */ + PipeBuilder csv(String expr); + + /** * attach a json pipe to the current context - * @param expr json expr or URL + * @param expr json expr or URL or path in the resource tree * @return updated instance of PipeBuilder */ PipeBuilder json(String expr); @@ -124,8 +131,9 @@ public interface PipeBuilder { * attach a reference pipe to the current context * @param expr reference * @return updated instance of PipeBuilder + * @throws IllegalAccessException in case it's called with wrong # of arguments */ - PipeBuilder ref(String expr) throws IllegalAccessException; + PipeBuilder ref(String expr)throws IllegalAccessException; /** * parameterized current pipe in the context Added: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java?rev=1808091&view=auto ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java (added) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java Tue Sep 12 09:32:00 2017 @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sling.pipes.internal; + +import org.apache.commons.lang3.StringUtils; +import org.apache.sling.api.resource.Resource; +import org.apache.sling.pipes.AbstractInputStreamPipe; +import org.apache.sling.pipes.Plumber; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Csv input stream pipe, similar at what + */ +public class CsvPipe extends AbstractInputStreamPipe { + private static Logger logger = LoggerFactory.getLogger(JsonPipe.class); + public static final String RESOURCE_TYPE = RT_PREFIX + "csv"; + + + protected static final String PN_SEPARATOR = "separator"; + + protected static final String DEFAULT_SEPARATOR = ","; + + BufferedReader reader; + + String nextLine = null; + + int index = 0; + + public CsvPipe(Plumber plumber, Resource resource) throws Exception { + super(plumber, resource); + } + + @Override + public Iterator<Resource> getOutput(InputStream inputStream) { + Iterator<Resource> output = EMPTY_ITERATOR; + String separator = properties.get(PN_SEPARATOR, DEFAULT_SEPARATOR); + try { + reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + String headersLine = reader.readLine(); + final String[] headers = headersLine.split(separator); + if (headers.length > 0){ + nextLine = reader.readLine(); + output = new Iterator<Resource>() { + @Override + public boolean hasNext() { + return StringUtils.isNotBlank(nextLine); + } + @Override + public Resource next() { + try { + String[] values = nextLine.split(separator); + if (values.length < headers.length){ + throw new IllegalArgumentException("wrong format line " + index + " should have at least the same number of columns than the headers"); + } + Map<String, String> map = new HashMap<>(); + for (int i = 0; i < headers.length; i ++){ + map.put(headers[i], values[i]); + } + binding = map; + nextLine = reader.readLine(); + } catch (Exception e) { + logger.error("Unable to retrieve {}nth line of csv file", index, e); + nextLine = null; + } + return getInput(); + } + }; + } + } catch (IllegalArgumentException iae){ + logger.error("unable to correctly process csv file", iae); + } catch (IOException e){ + logger.error("unable to process csv file", e); + } + return output; + } +} \ No newline at end of file Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java (original) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java Tue Sep 12 09:32:00 2017 @@ -130,6 +130,12 @@ public class PipeBuilderImpl implements return pipe(TraversePipe.RESOURCE_TYPE); } + + @Override + public PipeBuilder csv(String expr) { + return pipeWithExpr(CsvPipe.RESOURCE_TYPE, expr); + } + @Override public PipeBuilder json(String expr) { return pipeWithExpr(JsonPipe.RESOURCE_TYPE, expr); @@ -172,8 +178,8 @@ public class PipeBuilderImpl implements /** * Checks arguments and throws exception if there is an issue - * @param params - * @throws IllegalArgumentException + * @param params arguments to check + * @throws IllegalArgumentException exception thrown in case arguments are wrong */ protected void checkArguments(Object... params) throws IllegalArgumentException { if (params.length % 2 > 0){ @@ -183,8 +189,8 @@ public class PipeBuilderImpl implements /** * write key/value pairs into a map - * @param map - * @param params + * @param map target map + * @param params key/value pairs to write into the map */ protected void writeToMap(Map map, Object... params){ for (int i = 0; i < params.length; i += 2){ @@ -245,6 +251,7 @@ public class PipeBuilderImpl implements * @param type type of the node to be created * @param data map of properties to add * @throws PersistenceException in case configuration resource couldn't be persisted + * @return resource created */ protected Resource createResource(ResourceResolver resolver, String path, String type, Map data) throws PersistenceException { return ResourceUtil.getOrCreateResource(resolver, path, data, type, false); @@ -257,10 +264,11 @@ public class PipeBuilderImpl implements /** * Persist a step at a given path - * @param path - * @param step + * @param path path into which step should be persisted + * @param parentType type of the parent resource + * @param step step to persist * @return created resource - * @throws PersistenceException + * @throws PersistenceException in case persistence fails */ protected Resource persistStep(String path, String parentType, Step step) throws PersistenceException { Resource resource = createResource(resolver, path, parentType, step.properties); Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java (original) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java Tue Sep 12 09:32:00 2017 @@ -119,6 +119,7 @@ public class PlumberImpl implements Plum registerPipe(FilterPipe.RESOURCE_TYPE, FilterPipe.class); registerPipe(NotPipe.RESOURCE_TYPE, NotPipe.class); registerPipe(TraversePipe.RESOURCE_TYPE, TraversePipe.class); + registerPipe(CsvPipe.RESOURCE_TYPE, CsvPipe.class); } @Reference(policy= ReferencePolicy.DYNAMIC, cardinality= ReferenceCardinality.OPTIONAL) Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java?rev=1808091&r1=1808090&r2=1808091&view=diff ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java (original) +++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java Tue Sep 12 09:32:00 2017 @@ -139,7 +139,7 @@ public class PlumberServlet extends Slin } } - if (request.getParameterMap().containsValue(PARAM_FILE)){ + if (request.getRequestParameterMap() != null && request.getRequestParameterMap().containsKey(PARAM_FILE)){ bindings.put(AbstractInputStreamPipe.BINDING_IS, request.getRequestParameter(PARAM_FILE).getInputStream()); } Added: sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java?rev=1808091&view=auto ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java (added) +++ sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java Tue Sep 12 09:32:00 2017 @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sling.pipes.internal; + +import org.apache.commons.collections.IteratorUtils; +import org.apache.sling.api.resource.Resource; +import org.apache.sling.pipes.AbstractPipeTest; +import org.apache.sling.pipes.Pipe; +import org.junit.Test; + +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +/** + * testing csv pipe + */ +public class CsvPipeTest extends AbstractPipeTest{ + + @Test + public void getOutput() throws Exception { + String csvPath = "/content/test/standardTest.csv"; + context.load().binaryFile("/standardTest.csv", csvPath); + Pipe pipe = plumber.newPipe(context.resourceResolver()) + .csv(csvPath).name("csv") + .mkdir(PATH_FRUITS + "/csv/${csv.fruit}-${csv.color}-${csv.id}").build(); + Iterator<Resource> output = pipe.getOutput(); + List<Resource> resources = IteratorUtils.toList(output); + List<String> paths = resources.stream().map( resource -> resource.getPath()).collect(Collectors.toList()); + assertEquals("there should be 3 elements", 3, paths.size()); + assertEquals("first should be /content/fruits/csv/apple-green-1", "/content/fruits/csv/apple-green-1", paths.get(0)); + assertEquals("second should be /content/fruits/csv/banana-yellow-2", "/content/fruits/csv/banana-yellow-2", paths.get(1)); + assertEquals("first should be /content/fruits/csv/plum-purple-3", "/content/fruits/csv/plum-purple-3", paths.get(2)); + } +} \ No newline at end of file Added: sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv?rev=1808091&view=auto ============================================================================== --- sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv (added) +++ sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv Tue Sep 12 09:32:00 2017 @@ -0,0 +1,4 @@ +fruit,color,id +apple,green,1 +banana,yellow,2 +plum,purple,3 \ No newline at end of file