http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorResource.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorResource.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorResource.java new file mode 100644 index 0000000..4bb39de --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorResource.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.standard.api.processor; + +import java.util.HashMap; +import java.util.Map; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.ext.Provider; + +import org.apache.nifi.web.ComponentDescriptor; +import org.apache.nifi.web.ComponentDetails; +import org.apache.nifi.web.NiFiWebConfigurationContext; +import org.apache.nifi.web.NiFiWebConfigurationRequestContext; +import org.apache.nifi.web.NiFiWebRequestContext; + + +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; +import org.mockito.Mockito; + +import com.bazaarvoice.jolt.JsonUtils; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.core.ClassNamesResourceConfig; +import com.sun.jersey.spi.container.servlet.WebComponent; +import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider; +import com.sun.jersey.test.framework.AppDescriptor; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; +import com.sun.jersey.test.framework.spi.container.TestContainerFactory; +import com.sun.jersey.test.framework.spi.container.inmemory.InMemoryTestContainerFactory; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; + +public class TestProcessorResource extends JerseyTest { + + public static final ServletContext servletContext = mock(ServletContext.class); + public static final HttpServletRequest requestContext = mock(HttpServletRequest.class); + + @Override + protected AppDescriptor configure() { + return new WebAppDescriptor.Builder() + .initParam(WebComponent.RESOURCE_CONFIG_CLASS, ClassNamesResourceConfig.class.getName()) + .initParam(ClassNamesResourceConfig.PROPERTY_CLASSNAMES, + ProcessorResource.class.getName() + + ";" + TestProcessorResource.MockServletContext.class.getName() + + ";" + TestProcessorResource.MockRequestContext.class.getName()+";") + .initParam("com.sun.jersey.api.json.POJOMappingFeature", "true") + .build(); + } + + @Override + public TestContainerFactory getTestContainerFactory() { + return new InMemoryTestContainerFactory(); + } + + + @Test + public void testSetProperties() throws JSONException{ + + final NiFiWebConfigurationContext niFiWebConfigurationContext = mock(NiFiWebConfigurationContext.class); + final Map<String,String> properties = new HashMap<>(); + properties.put("jolt-transform","jolt-transform-chain"); + final ComponentDetails componentDetails = new ComponentDetails.Builder().properties(properties).build(); + + Mockito.when(servletContext.getAttribute(Mockito.anyString())).thenReturn(niFiWebConfigurationContext); + Mockito.when(niFiWebConfigurationContext.updateComponent(any(NiFiWebConfigurationRequestContext.class),any(String.class),any(Map.class))).thenReturn(componentDetails); + + ClientResponse response = client().resource(getBaseURI()).path("/standard/processor/properties") + .queryParam("processorId","1") + .queryParam("clientId","1") + .queryParam("revisionId","1") + .type(MediaType.APPLICATION_JSON_TYPE) + .put(ClientResponse.class,JsonUtils.toJsonString(properties)); + assertNotNull(response); + JSONObject jsonObject = response.getEntity(JSONObject.class); + assertNotNull(jsonObject); + assertTrue(jsonObject.getJSONObject("properties").get("jolt-transform").equals("jolt-transform-chain")); + } + + + @Test + public void testGetProcessorDetails() { + final NiFiWebConfigurationContext niFiWebConfigurationContext = mock(NiFiWebConfigurationContext.class); + final Map<String,String> allowableValues = new HashMap<>(); + final ComponentDescriptor descriptor = new ComponentDescriptor.Builder().name("test-name").allowableValues(allowableValues).build(); + final Map<String,ComponentDescriptor> descriptors = new HashMap<>(); + descriptors.put("jolt-transform",descriptor); + final ComponentDetails componentDetails = new ComponentDetails.Builder().name("mytransform").type("org.apache.nifi.processors.standard.JoltTransformJSON") + .descriptors(descriptors) + .build(); + + Mockito.when(servletContext.getAttribute(Mockito.anyString())).thenReturn(niFiWebConfigurationContext); + Mockito.when(niFiWebConfigurationContext.getComponentDetails(any(NiFiWebRequestContext.class))).thenReturn(componentDetails); + + JSONObject value = client().resource(getBaseURI()).path("/standard/processor/details").queryParam("processorId","1").get(JSONObject.class); + assertNotNull(value); + + try{ + assertTrue(value.get("name").equals("mytransform")); + } catch (Exception e){ + fail("Failed due to: " + e.toString()); + } + + } + + @Provider + public static class MockServletContext extends SingletonTypeInjectableProvider<Context, ServletContext> { + + public MockServletContext(){ + super(ServletContext.class, servletContext); + } + + } + + @Provider + public static class MockRequestContext extends SingletonTypeInjectableProvider<Context, HttpServletRequest> { + + public MockRequestContext(){ + super(HttpServletRequest.class, requestContext); + } + + } + + +}
http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorWebUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorWebUtils.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorWebUtils.java new file mode 100644 index 0000000..87a058f --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/processor/TestProcessorWebUtils.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.standard.api.processor; + + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.CacheControl; +import javax.ws.rs.core.Response; + +import org.apache.nifi.web.ComponentDetails; +import org.apache.nifi.web.HttpServletConfigurationRequestContext; +import org.apache.nifi.web.HttpServletRequestContext; +import org.apache.nifi.web.NiFiWebConfigurationContext; +import org.apache.nifi.web.NiFiWebRequestContext; +import org.apache.nifi.web.standard.api.processor.ProcessorWebUtils; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +public class TestProcessorWebUtils { + + @Test + public void testGetComponentDetailsForProcessor(){ + HttpServletRequest request = mock(HttpServletRequest.class); + NiFiWebConfigurationContext configurationContext = mock(NiFiWebConfigurationContext.class); + when(configurationContext.getComponentDetails(any(HttpServletRequestContext.class))).thenReturn(new ComponentDetails.Builder().build()); + ComponentDetails componentDetails = ProcessorWebUtils.getComponentDetails(configurationContext,"1",request); + assertNotNull(componentDetails); + + } + + @Test + public void testGetComponentDetailsForProcessorWithSpecificClientRevision(){ + NiFiWebConfigurationContext configurationContext = mock(NiFiWebConfigurationContext.class); + when(configurationContext.getComponentDetails(any(HttpServletConfigurationRequestContext.class))).thenReturn(new ComponentDetails.Builder().build()); + ComponentDetails componentDetails = ProcessorWebUtils.getComponentDetails(configurationContext,"1",1L, "client1",mock(HttpServletRequest.class)); + assertNotNull(componentDetails); + } + + @Test + public void testApplyCacheControl(){ + Response.ResponseBuilder response = mock(Response.ResponseBuilder.class); + ProcessorWebUtils.applyCacheControl(response); + verify(response).cacheControl(any(CacheControl.class)); + } + + @Test + @SuppressWarnings("unchecked") + public void testGetRequestContextForProcessor() throws NoSuchMethodException, IOException,InvocationTargetException, IllegalAccessException{ + Method method = ProcessorWebUtils.class.getDeclaredMethod("getRequestContext", String.class, HttpServletRequest.class); + method.setAccessible(true); + NiFiWebRequestContext requestContext = (NiFiWebRequestContext) method.invoke(null,"1",mock(HttpServletRequest.class)); + assertTrue(requestContext instanceof HttpServletRequestContext); + assertTrue(requestContext.getId().equals("1")); + + } + + @Test + @SuppressWarnings("unchecked") + public void testGetRequestContextForProcessorWithSpecificClientRevision() throws NoSuchMethodException, IOException,InvocationTargetException, IllegalAccessException{ + Method method = ProcessorWebUtils.class.getDeclaredMethod("getRequestContext", String.class, Long.class, String.class, HttpServletRequest.class); + method.setAccessible(true); + NiFiWebRequestContext requestContext = (NiFiWebRequestContext) method.invoke(null,"1",1L, "client1",mock(HttpServletRequest.class)); + assertTrue(requestContext instanceof HttpServletConfigurationRequestContext); + assertTrue(requestContext.getId().equals("1")); + assertTrue(((HttpServletConfigurationRequestContext)requestContext).getRevision().getClientId().equals("client1")); + assertTrue(((HttpServletConfigurationRequestContext)requestContext).getRevision().getVersion().equals(1L)); + } + + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java new file mode 100644 index 0000000..ba00a15 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/src/test/java/org/apache/nifi/web/standard/api/transformjson/TestTransformJSONResource.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.standard.api.transformjson; + +import junit.framework.TestCase; + +import java.util.HashMap; +import java.util.Map; + + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Context; +import javax.ws.rs.ext.Provider; + + +import org.apache.nifi.web.ComponentDetails; +import org.apache.nifi.web.NiFiWebConfigurationContext; +import org.apache.nifi.web.NiFiWebRequestContext; +import org.apache.nifi.web.standard.api.transformjson.dto.JoltSpecificationDTO; +import org.apache.nifi.web.standard.api.transformjson.dto.ValidationDTO; + + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.Mockito; + + +import com.bazaarvoice.jolt.Diffy; +import com.bazaarvoice.jolt.JsonUtils; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.core.ClassNamesResourceConfig; + + +import com.sun.jersey.spi.container.servlet.WebComponent; +import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider; +import com.sun.jersey.test.framework.AppDescriptor; +import com.sun.jersey.test.framework.JerseyTest; +import com.sun.jersey.test.framework.WebAppDescriptor; +import com.sun.jersey.test.framework.spi.container.TestContainerFactory; +import com.sun.jersey.test.framework.spi.container.inmemory.InMemoryTestContainerFactory; + + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; + + +public class TestTransformJSONResource extends JerseyTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + public static final ServletContext servletContext = mock(ServletContext.class); + public static final HttpServletRequest requestContext = mock(HttpServletRequest.class); + + @Override + protected AppDescriptor configure() { + return new WebAppDescriptor.Builder() + .initParam(WebComponent.RESOURCE_CONFIG_CLASS, ClassNamesResourceConfig.class.getName()) + .initParam(ClassNamesResourceConfig.PROPERTY_CLASSNAMES, + TransformJSONResource.class.getName() + ";" + MockServletContext.class.getName() + ";" + MockRequestContext.class.getName()+";") + .initParam("com.sun.jersey.api.json.POJOMappingFeature", "true") + .build(); + } + + @Override + public TestContainerFactory getTestContainerFactory() { + return new InMemoryTestContainerFactory(); + } + + + @Test + public void testValidateWithInvalidSpec() { + + final NiFiWebConfigurationContext niFiWebConfigurationContext = mock(NiFiWebConfigurationContext.class); + final Map<String,String> properties = new HashMap<>(); + properties.put("jolt-transform","jolt-transform-chain"); + final ComponentDetails componentDetails = new ComponentDetails.Builder().properties(properties).build(); + Mockito.when(servletContext.getAttribute(Mockito.anyString())).thenReturn(niFiWebConfigurationContext); + Mockito.when(niFiWebConfigurationContext.getComponentDetails(any(NiFiWebRequestContext.class))).thenReturn(componentDetails); + + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-chain","[]"); + ValidationDTO validate = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + + assertNotNull(validate); + assertTrue(!validate.isValid()); + + } + + @Test + public void testValidateWithValidSpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-remove","{\"rating\": {\"quality\": \"\"} }"); + ValidationDTO validation = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + TestCase.assertNotNull(validation); + assertTrue(validation.isValid()); + } + + @Test + public void testValidateWithValidEmptySpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-sort",""); + ValidationDTO validation = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + TestCase.assertNotNull(validation); + assertTrue(validation.isValid()); + } + + @Test + public void testValidateWithInvalidEmptySpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-remove",""); + ValidationDTO validation = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + TestCase.assertNotNull(validation); + assertTrue(!validation.isValid()); + } + + @Test + public void testValidateWithValidNullSpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-sort",null); + ValidationDTO validation = client().resource(getBaseURI()).path("/standard/transformjson/validate").post(ValidationDTO.class, joltSpecificationDTO); + TestCase.assertNotNull(validation); + assertTrue(validation.isValid()); + } + + + @Test + public void testExecuteWithInvalidSpec() { + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-remove", "{\"rating\": {\"quality\": \"\"} }"); + exception.expect(UniformInterfaceException.class); + client().resource(getBaseURI()).path("/standard/transformjson/execute").post(joltSpecificationDTO); + } + + @Test + public void testExecuteWithValidSpec() { + final Diffy diffy = new Diffy(); + JoltSpecificationDTO joltSpecificationDTO = new JoltSpecificationDTO("jolt-transform-remove","{\"rating\": {\"quality\": \"\"} }"); + String inputJson = "{\"rating\":{\"quality\":2,\"count\":1}}"; + joltSpecificationDTO.setInput(inputJson); + String responseString = client().resource(getBaseURI()).path("/standard/transformjson/execute").post(String.class, joltSpecificationDTO); + Object transformedJson = JsonUtils.jsonToObject(responseString); + Object compareJson = JsonUtils.jsonToObject("{\"rating\":{\"count\":1}}"); + assertNotNull(transformedJson); + assertTrue(diffy.diff(compareJson, transformedJson).isEmpty()); + } + + @Provider + public static class MockServletContext extends SingletonTypeInjectableProvider<Context, ServletContext> { + + public MockServletContext(){ + super(ServletContext.class, servletContext); + } + + } + + @Provider + public static class MockRequestContext extends SingletonTypeInjectableProvider<Context, HttpServletRequest> { + + public MockRequestContext(){ + super(HttpServletRequest.class, requestContext); + } + + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml index e95b071..03bbaba 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml @@ -49,5 +49,10 @@ <artifactId>nifi-standard-content-viewer</artifactId> <type>war</type> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-jolt-transform-json-ui</artifactId> + <type>war</type> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE index 18c9df1..32b25cf 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/LICENSE @@ -350,4 +350,72 @@ This product bundles 'Jolt' which is available under an Apache License. For deta distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. + +This product bundles 'AngularUI Codemirror' which is available under an MIT license. + + Copyright (c) 2012 the AngularUI Team, http://angular-ui.github.com + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundles 'AngularUI Router' which is available under an MIT license. + + Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +This product bundles 'js-beautify' which is available under an MIT license. + + Copyright (c) 2007-2013 Einar Lielmanis and contributors. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 17e630a..247eff2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -235,6 +235,16 @@ language governing permissions and limitations under the License. --> <artifactId>json-utils</artifactId> <version>0.0.20</version> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-utils</artifactId> + <version>1.0.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-utils</artifactId> + <version>1.0.0-SNAPSHOT</version> + </dependency> </dependencies> <build> @@ -309,20 +319,20 @@ language governing permissions and limitations under the License. --> <exclude>src/test/resources/TestSplitText/4.txt</exclude> <exclude>src/test/resources/TestSplitText/5.txt</exclude> <exclude>src/test/resources/TestSplitText/6.txt</exclude> - <exclude>src/test/resources/TestTransformJson/input.json</exclude> - <exclude>src/test/resources/TestTransformJson/chainrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/chainrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/cardrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/cardrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/defaultrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/defaultrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/shiftrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/sortrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/shiftrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/removrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/removrOutput.json</exclude> - <exclude>src/test/resources/TestTransformJson/defaultrSpec.json</exclude> - <exclude>src/test/resources/TestTransformJson/defaultrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/input.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/chainrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/chainrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/cardrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/cardrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/defaultrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/defaultrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/shiftrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/sortrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/shiftrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/removrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/removrOutput.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/defaultrSpec.json</exclude> + <exclude>src/test/resources/TestJoltTransformJson/defaultrOutput.json</exclude> <exclude>src/test/resources/TestSplitText/original.txt</exclude> <exclude>src/test/resources/TestTransformXml/math.html</exclude> <exclude>src/test/resources/TestTransformXml/tokens.csv</exclude> http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java new file mode 100644 index 0000000..3406b1f --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.io.InputStreamCallback; +import org.apache.nifi.processor.io.OutputStreamCallback; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.standard.util.TransformFactory; +import org.apache.nifi.stream.io.ByteArrayInputStream; +import org.apache.nifi.stream.io.StreamUtils; +import org.apache.nifi.util.StopWatch; +import org.apache.nifi.util.StringUtils; + +import com.bazaarvoice.jolt.Transform; +import com.bazaarvoice.jolt.JsonUtils; + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"json", "jolt", "transform", "shiftr", "chainr", "defaultr", "removr","cardinality","sort"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@WritesAttribute(attribute = "mime.type",description = "Always set to application/json") +@CapabilityDescription("Applies a list of Jolt specifications to the flowfile JSON payload. A new FlowFile is created " + + "with transformed content and is routed to the 'success' relationship. If the JSON transform " + + "fails, the original FlowFile is routed to the 'failure' relationship.") +public class JoltTransformJSON extends AbstractProcessor { + + public static final AllowableValue SHIFTR = new AllowableValue("jolt-transform-shift", "Shift", "Shift input JSON/data to create the output JSON."); + public static final AllowableValue CHAINR = new AllowableValue("jolt-transform-chain", "Chain", "Execute list of Jolt transformations."); + public static final AllowableValue DEFAULTR = new AllowableValue("jolt-transform-default", "Default", " Apply default values to the output JSON."); + public static final AllowableValue REMOVR = new AllowableValue("jolt-transform-remove", "Remove", " Remove values from input data to create the output JSON."); + public static final AllowableValue CARDINALITY = new AllowableValue("jolt-transform-card", "Cardinality", "Change the cardinality of input elements to create the output JSON."); + public static final AllowableValue SORTR = new AllowableValue("jolt-transform-sort", "Sort", "Sort input json key values alphabetically. Any specification set is ignored."); + + public static final PropertyDescriptor JOLT_TRANSFORM = new PropertyDescriptor.Builder() + .name("jolt-transform") + .displayName("Jolt Transformation DSL") + .description("Specifies the Jolt Transformation that should be used with the provided specification.") + .required(true) + .allowableValues(CARDINALITY, CHAINR, DEFAULTR, REMOVR, SHIFTR, SORTR) + .defaultValue(CHAINR.getValue()) + .build(); + + public static final PropertyDescriptor JOLT_SPEC = new PropertyDescriptor.Builder() + .name("jolt-spec") + .displayName("Jolt Specification") + .description("Jolt Specification for transform of JSON data. This value is ignored if the Jolt Sort Transformation is selected.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("The FlowFile with transformed content will be routed to this relationship") + .build(); + public static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("If a FlowFile fails processing for any reason (for example, the FlowFile is not valid JSON), it will be routed to this relationship") + .build(); + + private final static List<PropertyDescriptor> properties; + private final static Set<Relationship> relationships; + private volatile Transform transform; + private final static String DEFAULT_CHARSET = "UTF-8"; + + static{ + + final List<PropertyDescriptor> _properties = new ArrayList<>(); + _properties.add(JOLT_TRANSFORM); + _properties.add(JOLT_SPEC); + properties = Collections.unmodifiableList(_properties); + + final Set<Relationship> _relationships = new HashSet<>(); + _relationships.add(REL_SUCCESS); + _relationships.add(REL_FAILURE); + relationships = Collections.unmodifiableSet(_relationships); + + } + + @Override + public Set<Relationship> getRelationships() { + return relationships; + } + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return properties; + } + + + @Override + protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { + final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext)); + String transform = validationContext.getProperty(JOLT_TRANSFORM).getValue(); + String specValue = validationContext.getProperty(JOLT_SPEC).isSet() ? validationContext.getProperty(JOLT_SPEC).getValue() : null; + + if(StringUtils.isEmpty(specValue)){ + if(!SORTR.getValue().equals(transform)) { + String message = "A specification is required for this transformation"; + results.add(new ValidationResult.Builder().valid(false) + .explanation(message) + .build()); + } + } else { + try { + Object specJson = SORTR.getValue().equals(transform) ? null : JsonUtils.jsonToObject(specValue, DEFAULT_CHARSET); + TransformFactory.getTransform(transform, specJson); + } catch (final Exception e) { + getLogger().info("Processor is not valid - " + e.toString()); + String message = "Specification not valid for the selected transformation." ; + results.add(new ValidationResult.Builder().valid(false) + .explanation(message) + .build()); + } + } + return results; + } + + @Override + public void onTrigger(final ProcessContext context, ProcessSession session) throws ProcessException { + + final FlowFile original = session.get(); + if (original == null) { + return; + } + + final ComponentLog logger = getLogger(); + final StopWatch stopWatch = new StopWatch(true); + + final byte[] originalContent = new byte[(int) original.getSize()]; + session.read(original, new InputStreamCallback() { + @Override + public void process(final InputStream in) throws IOException { + StreamUtils.fillBuffer(in, originalContent, true); + } + }); + + final String jsonString; + + try { + final ByteArrayInputStream bais = new ByteArrayInputStream(originalContent); + final Object inputJson = JsonUtils.jsonToObject(bais); + final Object transformedJson = transform.transform(inputJson); + jsonString = JsonUtils.toJsonString(transformedJson); + + } catch (RuntimeException re) { + logger.error("Unable to transform {} due to {}", new Object[]{original, re}); + session.transfer(original, REL_FAILURE); + return; + } + + FlowFile transformed = session.write(original, new OutputStreamCallback() { + @Override + public void process(OutputStream out) throws IOException { + out.write(jsonString.getBytes(DEFAULT_CHARSET)); + } + }); + + final String transformType = context.getProperty(JOLT_TRANSFORM).getValue(); + transformed = session.putAttribute(transformed, CoreAttributes.MIME_TYPE.key(),"application/json"); + session.transfer(transformed, REL_SUCCESS); + session.getProvenanceReporter().modifyContent(transformed,"Modified With " + transformType ,stopWatch.getElapsed(TimeUnit.MILLISECONDS)); + logger.info("Transformed {}", new Object[]{original}); + + } + + @OnScheduled + public void setup(final ProcessContext context) { + Object specJson = null; + if(context.getProperty(JOLT_SPEC).isSet() && !SORTR.getValue().equals(context.getProperty(JOLT_TRANSFORM).getValue())){ + specJson = JsonUtils.jsonToObject(context.getProperty(JOLT_SPEC).getValue(), DEFAULT_CHARSET); + } + transform = TransformFactory.getTransform(context.getProperty(JOLT_TRANSFORM).getValue(), specJson); + } + + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformJSON.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformJSON.java deleted file mode 100644 index 675b135..0000000 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformJSON.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.standard; - - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.nifi.annotation.behavior.EventDriven; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SideEffectFree; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.components.AllowableValue; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.flowfile.attributes.CoreAttributes; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.io.OutputStreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.stream.io.ByteArrayInputStream; -import org.apache.nifi.stream.io.StreamUtils; -import org.apache.nifi.util.StopWatch; -import org.apache.nifi.util.StringUtils; - -import com.bazaarvoice.jolt.CardinalityTransform; -import com.bazaarvoice.jolt.Chainr; -import com.bazaarvoice.jolt.Defaultr; -import com.bazaarvoice.jolt.JsonUtils; -import com.bazaarvoice.jolt.Removr; -import com.bazaarvoice.jolt.Shiftr; -import com.bazaarvoice.jolt.Sortr; -import com.bazaarvoice.jolt.Transform; - -@EventDriven -@SideEffectFree -@SupportsBatching -@Tags({"json", "jolt", "transform", "shiftr", "chainr", "defaultr", "removr","cardinality","sort"}) -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@WritesAttribute(attribute = "mime.type",description = "Always set to application/json") -@CapabilityDescription("Applies a list of JOLT specifications to the flowfile JSON payload. A new FlowFile is created " - + "with transformed content and is routed to the 'success' relationship. If the JSON transform " - + "fails, the original FlowFile is routed to the 'failure' relationship.") -public class TransformJSON extends AbstractProcessor { - - public static final AllowableValue SHIFTR = new AllowableValue("jolt-transform-shift", "Shift Transform DSL", "Shift input JSON/data to create the output JSON."); - public static final AllowableValue CHAINR = new AllowableValue("jolt-transform-chain", "Chain Transform DSL", "Execute list of JOLT transformations."); - public static final AllowableValue DEFAULTR = new AllowableValue("jolt-transform-default", "Default Transform DSL", " Apply default values to the output JSON."); - public static final AllowableValue REMOVR = new AllowableValue("jolt-transform-remove", "Remove Transform DSL", " Remove values from input data to create the output JSON."); - public static final AllowableValue CARDINALITY = new AllowableValue("jolt-transform-card", "Cardinality Transform DSL", "Change the cardinality of input elements to create the output JSON."); - public static final AllowableValue SORTR = new AllowableValue("jolt-transform-sort", "Sort Transform DSL", "Sort input json key values alphabetically. Any specification set is ignored."); - - public static final PropertyDescriptor JOLT_TRANSFORM = new PropertyDescriptor.Builder() - .name("jolt-transform") - .displayName("Jolt Transformation") - .description("Specifies the Jolt Transformation that should be used with the provided specification.") - .required(true) - .allowableValues(CARDINALITY, CHAINR, DEFAULTR, REMOVR, SHIFTR, SORTR) - .defaultValue(CHAINR.getValue()) - .build(); - - public static final PropertyDescriptor JOLT_SPEC = new PropertyDescriptor.Builder() - .name("jolt-spec") - .displayName("Jolt Specification") - .description("Jolt Specification for transform of JSON data. This value is ignored if the Jolt Sort Transformation is selected.") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .required(false) - .build(); - - public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("The FlowFile with transformed content will be routed to this relationship") - .build(); - public static final Relationship REL_FAILURE = new Relationship.Builder() - .name("failure") - .description("If a FlowFile fails processing for any reason (for example, the FlowFile is not valid JSON), it will be routed to this relationship") - .build(); - - private final static List<PropertyDescriptor> properties; - private final static Set<Relationship> relationships; - private volatile Transform transform; - private final static String DEFAULT_CHARSET = "UTF-8"; - - static{ - - final List<PropertyDescriptor> _properties = new ArrayList<>(); - _properties.add(JOLT_TRANSFORM); - _properties.add(JOLT_SPEC); - properties = Collections.unmodifiableList(_properties); - - final Set<Relationship> _relationships = new HashSet<>(); - _relationships.add(REL_SUCCESS); - _relationships.add(REL_FAILURE); - relationships = Collections.unmodifiableSet(_relationships); - - } - - @Override - public Set<Relationship> getRelationships() { - return relationships; - } - - @Override - protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { - return properties; - } - - - @Override - protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { - final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext)); - String transform = validationContext.getProperty(JOLT_TRANSFORM).getValue(); - String specValue = validationContext.getProperty(JOLT_SPEC).isSet() ? validationContext.getProperty(JOLT_SPEC).getValue() : null; - - if(StringUtils.isEmpty(specValue)){ - if(!SORTR.getValue().equals(transform)) { - String message = "A specification is required for this transformation"; - results.add(new ValidationResult.Builder().valid(false) - .explanation(message) - .build()); - } - } else { - try { - Object specJson = JsonUtils.jsonToObject(specValue, DEFAULT_CHARSET); - TransformationFactory.getTransform(transform, specJson); - } catch (final Exception e) { - getLogger().info("Processor is not valid - " + e.toString()); - String message = "Specification not valid for the selected transformation." ; - results.add(new ValidationResult.Builder().valid(false) - .explanation(message) - .build()); - } - } - return results; - } - - @Override - public void onTrigger(final ProcessContext context, ProcessSession session) throws ProcessException { - - final FlowFile original = session.get(); - if (original == null) { - return; - } - - final ComponentLog logger = getLogger(); - final StopWatch stopWatch = new StopWatch(true); - - final byte[] originalContent = new byte[(int) original.getSize()]; - session.read(original, new InputStreamCallback() { - @Override - public void process(final InputStream in) throws IOException { - StreamUtils.fillBuffer(in, originalContent, true); - } - }); - - final String jsonString; - - try { - final ByteArrayInputStream bais = new ByteArrayInputStream(originalContent); - final Object inputJson = JsonUtils.jsonToObject(bais); - final Object transformedJson = transform.transform(inputJson); - jsonString = JsonUtils.toJsonString(transformedJson); - - } catch (RuntimeException re) { - logger.error("Unable to transform {} due to {}", new Object[]{original, re}); - session.transfer(original, REL_FAILURE); - return; - } - - FlowFile transformed = session.write(original, new OutputStreamCallback() { - @Override - public void process(OutputStream out) throws IOException { - out.write(jsonString.getBytes(DEFAULT_CHARSET)); - } - }); - - final String transformType = context.getProperty(JOLT_TRANSFORM).getValue(); - transformed = session.putAttribute(transformed, CoreAttributes.MIME_TYPE.key(),"application/json"); - session.transfer(transformed, REL_SUCCESS); - session.getProvenanceReporter().modifyContent(transformed,"Modified With " + transformType ,stopWatch.getElapsed(TimeUnit.MILLISECONDS)); - logger.info("Transformed {}", new Object[]{original}); - - } - - @OnScheduled - public void setup(final ProcessContext context) { - Object specJson = null; - if(context.getProperty(JOLT_SPEC).isSet()){ - specJson = JsonUtils.jsonToObject(context.getProperty(JOLT_SPEC).getValue(), DEFAULT_CHARSET); - } - transform = TransformationFactory.getTransform(context.getProperty(JOLT_TRANSFORM).getValue(), specJson); - } - - private static class TransformationFactory { - - static Transform getTransform(String transform, Object specJson) { - if (transform.equals(DEFAULTR.getValue())) { - return new Defaultr(specJson); - } else if (transform.equals(SHIFTR.getValue())) { - return new Shiftr(specJson); - } else if (transform.equals(REMOVR.getValue())) { - return new Removr(specJson); - } else if (transform.equals(CARDINALITY.getValue())) { - return new CardinalityTransform(specJson); - } else if(transform.equals(SORTR.getValue())){ - return new Sortr(); - } else { - return Chainr.fromSpec(specJson); - } - } - - } - -} http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index bf95078..a4eb235 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -42,6 +42,7 @@ org.apache.nifi.processors.standard.HashAttribute org.apache.nifi.processors.standard.HashContent org.apache.nifi.processors.standard.IdentifyMimeType org.apache.nifi.processors.standard.InvokeHTTP +org.apache.nifi.processors.standard.JoltTransformJSON org.apache.nifi.processors.standard.GetJMSQueue org.apache.nifi.processors.standard.GetJMSTopic org.apache.nifi.processors.standard.ListFile @@ -80,7 +81,6 @@ org.apache.nifi.processors.standard.SplitJson org.apache.nifi.processors.standard.SplitText org.apache.nifi.processors.standard.SplitXml org.apache.nifi.processors.standard.TailFile -org.apache.nifi.processors.standard.TransformJSON org.apache.nifi.processors.standard.TransformXml org.apache.nifi.processors.standard.UnpackContent org.apache.nifi.processors.standard.ValidateXml http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html new file mode 100644 index 0000000..f2f1c05 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.JoltTransformJSON/additionalDetails.html @@ -0,0 +1,36 @@ +<!DOCTYPE html> +<html lang="en"> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<head> + <meta charset="utf-8"/> + <title>JoltTransformJSON</title> + <link rel="stylesheet" href="../../css/component-usage.css" type="text/css"/> +</head> + +<body> +<!-- Processor Documentation ================================================== --> +<h2>Usage Information</h2> + +<p> + The Jolt utilities processing JSON are not not stream based therefore large JSON document + transformation may consume large amounts of memory. Currently UTF-8 FlowFile content and Jolt specifications are supported. + + <Strong>Note:</Strong> When configuring a processor if user selects of the Default transformation yet provides a + Chain specification the system does not alert that the specification is invalid and and will produce failed flow files. + This is a known issue identified within the Jolt library. +</p> +</body> +</html> http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.TransformJSON/additionalDetails.html ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.TransformJSON/additionalDetails.html b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.TransformJSON/additionalDetails.html deleted file mode 100644 index 2141d14..0000000 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/resources/docs/org.apache.nifi.processors.standard.TransformJSON/additionalDetails.html +++ /dev/null @@ -1,36 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<head> - <meta charset="utf-8"/> - <title>TransformJSON</title> - <link rel="stylesheet" href="../../css/component-usage.css" type="text/css"/> -</head> - -<body> -<!-- Processor Documentation ================================================== --> -<h2>Usage Information</h2> - -<p> - The JOLT utilities processing JSON are not not stream based therefore large JSON document - transformation may consume large amounts of memory. Currently UTF-8 FlowFile content and Jolt specifications are supported. - - <Strong>Note:</Strong> When configuring a processor if user selects of the Default transformation yet provides a - Chain specification the system does not alert that the specification is invalid and and will produce failed flow files. - This is a known issue identified within the JOLT library. -</p> -</body> -</html> http://git-wip-us.apache.org/repos/asf/nifi/blob/cb3aa8f5/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java new file mode 100644 index 0000000..79cc95c --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.standard; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; + +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.stream.io.ByteArrayInputStream; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.StringUtils; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Test; + +import com.bazaarvoice.jolt.Diffy; +import com.bazaarvoice.jolt.JsonUtils; + +import static org.junit.Assert.assertTrue; + +public class TestJoltTransformJSON { + + final static Path JSON_INPUT = Paths.get("src/test/resources/TestJoltTransformJson/input.json"); + final static Diffy DIFFY = new Diffy(); + + @Test + public void testRelationshipsCreated() throws IOException{ + Processor processor= new JoltTransformJSON(); + final TestRunner runner = TestRunners.newTestRunner(processor); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.enqueue(JSON_INPUT); + Set<Relationship> relationships = processor.getRelationships(); + assertTrue(relationships.contains(JoltTransformJSON.REL_FAILURE)); + assertTrue(relationships.contains(JoltTransformJSON.REL_SUCCESS)); + assertTrue(relationships.size() == 2); + } + + @Test + public void testInvalidJOLTSpec() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = "[{}]"; + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.assertNotValid(); + } + + @Test + public void testIncorrectJOLTSpec() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String chainrSpec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, chainrSpec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SHIFTR); + runner.assertNotValid(); + } + + @Test + public void testSpecIsNotSet() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SHIFTR); + runner.assertNotValid(); + } + + @Test + public void testSpecIsEmpty() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, StringUtils.EMPTY); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SHIFTR); + runner.assertNotValid(); + } + + @Test + public void testSpecNotRequired() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SORTR); + runner.assertValid(); + } + + @Test + public void testNoFlowFileContent() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.run(); + runner.assertQueueEmpty(); + runner.assertTransferCount(JoltTransformJSON.REL_FAILURE,0); + runner.assertTransferCount(JoltTransformJSON.REL_SUCCESS,0); + } + + @Test + public void testInvalidFlowFileContentJson() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.enqueue("invalid json"); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_FAILURE); + } + + @Test + public void testTransformInputWithChainr() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + transformed.assertAttributeExists(CoreAttributes.MIME_TYPE.key()); + transformed.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(),"application/json"); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/chainrOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithShiftr() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/shiftrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SHIFTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + transformed.assertAttributeExists(CoreAttributes.MIME_TYPE.key()); + transformed.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(),"application/json"); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/shiftrOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithDefaultr() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/defaultrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.DEFAULTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/defaultrOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithRemovr() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/removrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.REMOVR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/removrOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithCardinality() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + final String spec = new String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/cardrSpec.json"))); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.CARDINALITY); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/cardrOutput.json"))); + assertTrue(DIFFY.diff(compareJson, transformedJson).isEmpty()); + } + + @Test + public void testTransformInputWithSortr() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SORTR); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + transformed.assertAttributeExists(CoreAttributes.MIME_TYPE.key()); + transformed.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(),"application/json"); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/sortrOutput.json"))); + String transformedJsonString = JsonUtils.toJsonString(transformedJson); + String compareJsonString = JsonUtils.toJsonString(compareJson); + assertTrue(compareJsonString.equals(transformedJsonString)); + } + + @Test + public void testTransformInputWithSortrPopulatedSpec() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new JoltTransformJSON()); + runner.setProperty(JoltTransformJSON.JOLT_TRANSFORM, JoltTransformJSON.SORTR); + runner.setProperty(JoltTransformJSON.JOLT_SPEC, "abcd"); + runner.enqueue(JSON_INPUT); + runner.run(); + runner.assertAllFlowFilesTransferred(JoltTransformJSON.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(JoltTransformJSON.REL_SUCCESS).get(0); + transformed.assertAttributeExists(CoreAttributes.MIME_TYPE.key()); + transformed.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(),"application/json"); + Object transformedJson = JsonUtils.jsonToObject(new ByteArrayInputStream(transformed.toByteArray())); + Object compareJson = JsonUtils.jsonToObject(Files.newInputStream(Paths.get("src/test/resources/TestJoltTransformJson/sortrOutput.json"))); + String transformedJsonString = JsonUtils.toJsonString(transformedJson); + String compareJsonString = JsonUtils.toJsonString(compareJson); + assertTrue(compareJsonString.equals(transformedJsonString)); + } + + +}
