NIFI-1974 - initial commit variable registry for custom property support
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c600f150 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c600f150 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c600f150 Branch: refs/heads/0.x Commit: c600f150aa78bf51b1bb163c83d90c9e47869b2e Parents: 90c749d Author: Yolanda M. Davis <[email protected]> Authored: Sun Jun 5 21:17:37 2016 -0400 Committer: Mark Payne <[email protected]> Committed: Sat Jun 18 19:49:20 2016 -0400 ---------------------------------------------------------------------- nifi-api/pom.xml | 4 +- .../nifi/registry/FileVariableRegistry.java | 61 ++++++ .../java/org/apache/nifi/registry/MultiMap.java | 145 ++++++++++++++ .../nifi/registry/MultiMapVariableRegistry.java | 70 +++++++ .../registry/PropertiesVariableRegistry.java | 76 ++++++++ .../apache/nifi/registry/VariableRegistry.java | 52 +++++ .../nifi/registry/VariableRegistryFactory.java | 48 +++++ .../nifi/registry/VariableRegistryProvider.java | 23 +++ .../nifi/registry/VariableRegistryUtils.java | 63 ++++++ .../nifi/web/NiFiWebConfigurationContext.java | 5 +- .../nifi/registry/TestVariableRegistry.java | 137 +++++++++++++ .../registry/TestVariableRegistryUtils.java | 123 ++++++++++++ .../TestVariableRegistry/foobar.properties | 16 ++ .../TestVariableRegistry/test.properties | 17 ++ .../bootstrap/NotificationServiceManager.java | 4 +- .../NotificationValidationContext.java | 5 +- .../expression/language/EmptyPreparedQuery.java | 35 +--- .../language/InvalidPreparedQuery.java | 33 +--- .../expression/language/PreparedQuery.java | 16 +- .../attribute/expression/language/Query.java | 190 +------------------ .../language/StandardAttributeExpression.java | 9 +- .../StandardExpressionLanguageCompiler.java | 9 +- .../language/StandardPreparedQuery.java | 37 +--- .../language/StandardPropertyValue.java | 24 ++- .../expression/language/QueryGroovyTest.groovy | 17 +- .../expression/language/TestQuery.java | 112 +++++++---- .../language/TestStandardPreparedQuery.java | 7 +- .../org/apache/nifi/util/NiFiProperties.java | 29 +++ .../nifi/util/MockConfigurationContext.java | 13 +- ...kControllerServiceInitializationContext.java | 2 + .../nifi/util/MockControllerServiceLookup.java | 1 + .../apache/nifi/util/MockProcessContext.java | 23 ++- .../MockProcessorInitializationContext.java | 1 + .../org/apache/nifi/util/MockPropertyValue.java | 18 +- .../apache/nifi/util/MockReportingContext.java | 7 +- .../apache/nifi/util/MockValidationContext.java | 15 +- .../nifi/util/StandardProcessorTestRunner.java | 15 +- .../java/org/apache/nifi/util/TestRunners.java | 3 +- .../CurrentTestStandardProcessorTestRunner.java | 3 +- .../nifi/util/TestMockProcessContext.java | 3 +- .../util/TestStandardProcessorTestRunner.java | 9 +- .../ambari/TestAmbariReportingTask.java | 15 +- .../elasticsearch/TestFetchElasticsearch.java | 6 +- .../manager/impl/ClusteredReportingContext.java | 7 +- .../cluster/manager/impl/WebClusterManager.java | 33 +++- .../reporting/ClusteredReportingTaskNode.java | 11 +- .../apache/nifi/controller/FlowController.java | 46 +++-- .../reporting/AbstractReportingTaskNode.java | 13 +- .../reporting/StandardReportingContext.java | 10 +- .../StandardReportingInitializationContext.java | 1 + .../reporting/StandardReportingTaskNode.java | 8 +- .../scheduling/EventDrivenSchedulingAgent.java | 8 +- .../scheduling/QuartzSchedulingAgent.java | 8 +- .../scheduling/StandardProcessScheduler.java | 9 +- .../scheduling/TimerDrivenSchedulingAgent.java | 8 +- .../service/StandardConfigurationContext.java | 9 +- ...dControllerServiceInitializationContext.java | 1 + .../service/StandardControllerServiceNode.java | 14 +- .../StandardControllerServiceProvider.java | 13 +- .../manager/StandardStateManagerProvider.java | 28 +-- .../nifi/groups/StandardProcessGroup.java | 11 +- .../nifi/processor/StandardProcessContext.java | 11 +- .../processor/StandardValidationContext.java | 18 +- .../StandardValidationContextFactory.java | 9 +- .../TestStandardProcessScheduler.java | 29 +-- .../StandardControllerServiceProviderTest.java | 10 +- .../TestStandardControllerServiceProvider.java | 20 +- .../local/TestWriteAheadLocalStateProvider.java | 8 +- .../zookeeper/TestZooKeeperStateProvider.java | 8 +- .../processor/TestStandardPropertyValue.java | 27 +-- .../src/main/resources/conf/nifi.properties | 6 +- .../StandardNiFiWebConfigurationContext.java | 17 +- .../web/controller/StandardSearchContext.java | 2 +- .../processors/script/TestInvokeGroovy.java | 13 +- .../TestSiteToSiteProvenanceReportingTask.java | 5 +- .../standard/TestRouteOnAttribute.java | 8 +- .../cache/server/TestServerAndClient.java | 13 +- .../update/attributes/TestUpdateAttribute.java | 3 + .../attributes/UpdateAttributeModelFactory.java | 9 +- .../update/attributes/api/RuleResource.java | 20 +- 80 files changed, 1434 insertions(+), 541 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-api/pom.xml b/nifi-api/pom.xml index e8a8222..8bdfc5c 100644 --- a/nifi-api/pom.xml +++ b/nifi-api/pom.xml @@ -21,5 +21,5 @@ <version>0.7.0-SNAPSHOT</version> </parent> <artifactId>nifi-api</artifactId> - <packaging>jar</packaging> -</project> + <packaging>jar</packaging> + </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/FileVariableRegistry.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/FileVariableRegistry.java b/nifi-api/src/main/java/org/apache/nifi/registry/FileVariableRegistry.java new file mode 100644 index 0000000..539f24e --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/FileVariableRegistry.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.registry; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + + +abstract class FileVariableRegistry extends MultiMapVariableRegistry { + + FileVariableRegistry() { + super(); + } + + FileVariableRegistry(File... files) throws IOException{ + super(); + addVariables(files); + } + + FileVariableRegistry(Path... paths) throws IOException{ + super(); + addVariables(paths); + } + + private void addVariables(File ...files) throws IOException{ + if(files != null) { + for (final File file : files) { + registry.addMap(convertFile(file)); + } + } + } + + + private void addVariables(Path ...paths) throws IOException{ + if(paths != null) { + for (final Path path : paths) { + registry.addMap(convertFile(path.toFile())); + } + } + } + + protected abstract Map<String,String> convertFile(File file) throws IOException; + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/MultiMap.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/MultiMap.java b/nifi-api/src/main/java/org/apache/nifi/registry/MultiMap.java new file mode 100644 index 0000000..91e27c0 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/MultiMap.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MultiMap<V> implements Map<String,V> { + + private final List<Map<String,V>> maps; + + MultiMap() { + this.maps = new ArrayList<>(); + } + + @Override + public int size() { + return keySet().size(); + } + + @Override + public boolean isEmpty() { + for (final Map<String,V> map : maps) { + if (!map.isEmpty()) { + return false; + } + } + return true; + } + + @Override + public boolean containsKey(final Object key) { + if (key == null) { + return false; + } + + for (final Map<String,V> map : maps) { + if (map.containsKey(key)) { + return true; + } + } + return false; + } + + @Override + public boolean containsValue(final Object value) { + for (final Map<String,V> map : maps) { + if (map.containsValue(value)) { + return true; + } + } + return false; + } + + @Override + @SuppressWarnings("rawtypes") + public V get(final Object key) { + if (key == null) { + throw new IllegalArgumentException("Null Keys are not allowed"); + } + + for (final Map<String,V> map : maps) { + final V val = map.get(key); + if (val != null) { + return val; + } + } + return null; + } + + @Override + public V put(String key, V value) { + throw new UnsupportedOperationException(); + } + + @Override + public V remove(Object key) { + throw new UnsupportedOperationException(); + } + + @Override + public void putAll(Map<? extends String, ? extends V> m) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Set<String> keySet() { + final Set<String> keySet = new HashSet<>(); + for (final Map map : maps) { + keySet.addAll(map.keySet()); + } + return keySet; + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Collection<V> values() { + final Set<V> values = new HashSet<>(); + for (final Map map : maps) { + values.addAll(map.values()); + } + return values; + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Set<java.util.Map.Entry<String, V>> entrySet() { + final Set<java.util.Map.Entry<String, V>> entrySet = new HashSet<>(); + for (final Map map : maps) { + entrySet.addAll(map.entrySet()); + } + return entrySet; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + void addMap(Map<String,V> map){ + this.maps.add(map); + } + + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/MultiMapVariableRegistry.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/MultiMapVariableRegistry.java b/nifi-api/src/main/java/org/apache/nifi/registry/MultiMapVariableRegistry.java new file mode 100644 index 0000000..f9d04d0 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/MultiMapVariableRegistry.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.util.Map; +import java.util.Set; + +/*** + * This implementation of variable registry uses the MultiMap which stores one or more + * registries that can be searched, accessed and appended. NOTE: Duplicate values within + * or between added registries will be stored however on retrieval the first value encountered will be returned. + * */ +public class MultiMapVariableRegistry implements VariableRegistry { + + protected final MultiMap<String> registry; + + MultiMapVariableRegistry() { + this.registry = new MultiMap<>(); + } + + @SafeVarargs + MultiMapVariableRegistry(Map<String,String>...maps){ + this(); + if(maps != null) { + for (Map<String, String> map : maps) { + addVariables(map); + } + } + } + + void addVariables(Map<String, String> map) { + this.registry.addMap(map); + } + + @Override + public void addRegistry(VariableRegistry variableRegistry) { + if(!variableRegistry.getVariables().isEmpty()) { + this.registry.addMap(variableRegistry.getVariables()); + } + } + + @Override + public Map<String, String> getVariables() { + return registry; + } + + @Override + public String getVariableValue(String variable) { + return registry.get(variable); + } + + @Override + public Set<String> getVariableNames() { + return this.registry.keySet(); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/PropertiesVariableRegistry.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/PropertiesVariableRegistry.java b/nifi-api/src/main/java/org/apache/nifi/registry/PropertiesVariableRegistry.java new file mode 100644 index 0000000..bc5b15b --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/PropertiesVariableRegistry.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class PropertiesVariableRegistry extends FileVariableRegistry { + + private static final Logger LOG = LoggerFactory.getLogger(PropertiesVariableRegistry.class); + + PropertiesVariableRegistry(File... files) throws IOException{ + super(files); + } + + PropertiesVariableRegistry(Path... paths) throws IOException { + super(paths); + } + + PropertiesVariableRegistry(Properties...properties){ + super(); + addVariables(properties); + } + + private void addVariables(Properties... properties){ + if(properties != null) { + for (Properties props : properties) { + addVariables(convertToMap(props)); + } + } + } + + @Override + protected Map<String,String> convertFile(File file) throws IOException{ + + try(final InputStream inStream = new BufferedInputStream(new FileInputStream(file))) { + Properties properties = new Properties(); + properties.load(inStream); + return convertToMap(properties); + } + } + + private Map<String,String> convertToMap(Properties properties){ + HashMap<String,String> propertiesMap = new HashMap<>(properties.keySet().size()); + for(Object key: properties.keySet()){ + propertiesMap.put((String)key,(String) properties.get(key)); + } + return propertiesMap; + } + + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistry.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistry.java b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistry.java new file mode 100644 index 0000000..9f54f23 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistry.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + + +import java.util.Map; +import java.util.Set; + +/** + * Access key/value pairs throughout the application. + */ +public interface VariableRegistry { + + /** + * Returns a map of key/value pairs stored in the registry + * @return variables + **/ + Map<String, String> getVariables(); + + /** + * Return a value for a given variable + * @param variable variable + * @return value + **/ + String getVariableValue(String variable); + + /** + * Concatenate a variable registry + * @param variableRegistry variableRegistry + * */ + void addRegistry(VariableRegistry variableRegistry); + + /** + * Returns a set variable names in the registry + * @return variableNames + **/ + Set<String> getVariableNames(); +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryFactory.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryFactory.java b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryFactory.java new file mode 100644 index 0000000..1852ad4 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Properties; + +public class VariableRegistryFactory { + + public static VariableRegistry getPropertiesInstance(final Properties...properties){ + return new PropertiesVariableRegistry(properties); + } + + public static VariableRegistry getPropertiesInstance(final Path... paths) throws IOException{ + return new PropertiesVariableRegistry(paths); + } + + public static VariableRegistry getPropertiesInstance(final File ...files) throws IOException{ + return new PropertiesVariableRegistry(files); + } + + @SafeVarargs + public static VariableRegistry getInstance(final Map<String,String> ...maps){ + return new MultiMapVariableRegistry(maps); + } + + public static VariableRegistry getInstance(){ + return new MultiMapVariableRegistry(); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryProvider.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryProvider.java b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryProvider.java new file mode 100644 index 0000000..af7ab38 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryProvider.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +public interface VariableRegistryProvider { + + VariableRegistry getVariableRegistry(); + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryUtils.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryUtils.java b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryUtils.java new file mode 100644 index 0000000..c312dc7 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistryUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.nifi.flowfile.FlowFile; + +public class VariableRegistryUtils { + + + public static VariableRegistry createVariableRegistry(){ + VariableRegistry variableRegistry = VariableRegistryFactory.getInstance(); + VariableRegistry envRegistry = VariableRegistryFactory.getInstance(System.getenv()); + VariableRegistry propRegistry = VariableRegistryFactory.getPropertiesInstance(System.getProperties()); + variableRegistry.addRegistry(envRegistry); + variableRegistry.addRegistry(propRegistry); + return variableRegistry; + } + + public static VariableRegistry populateRegistry(VariableRegistry variableRegistry, final FlowFile flowFile, final Map<String, String> additionalAttributes){ + final Map<String, String> flowFileAttributes = flowFile == null ? null : flowFile.getAttributes(); + final Map<String, String> additionalMap = additionalAttributes == null ? null : additionalAttributes; + + Map<String, String> flowFileProps = null; + if (flowFile != null) { + flowFileProps = new HashMap<>(); + flowFileProps.put("flowFileId", String.valueOf(flowFile.getId())); + flowFileProps.put("fileSize", String.valueOf(flowFile.getSize())); + flowFileProps.put("entryDate", String.valueOf(flowFile.getEntryDate())); + flowFileProps.put("lineageStartDate", String.valueOf(flowFile.getLineageStartDate())); + } + VariableRegistry newRegistry = VariableRegistryFactory.getInstance(); + newRegistry.addRegistry(variableRegistry); + + if(flowFileAttributes != null) { + newRegistry.addRegistry(VariableRegistryFactory.getInstance(flowFileAttributes)); + } + if(additionalMap != null) { + newRegistry.addRegistry(VariableRegistryFactory.getInstance(additionalMap)); + } + if(flowFileProps != null) { + newRegistry.addRegistry(VariableRegistryFactory.getInstance(flowFileProps)); + } + return newRegistry; + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java index 021d243..4420a56 100644 --- a/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java +++ b/nifi-api/src/main/java/org/apache/nifi/web/NiFiWebConfigurationContext.java @@ -20,12 +20,13 @@ import java.util.Collection; import java.util.Map; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.registry.VariableRegistryProvider; /** * NiFi web context providing limited access to dataflow configuration for * component custom UIs. */ -public interface NiFiWebConfigurationContext { +public interface NiFiWebConfigurationContext extends VariableRegistryProvider { /** * @param serviceIdentifier of the controller service @@ -117,4 +118,6 @@ public interface NiFiWebConfigurationContext { * populated or isn't appropriate for the given request */ ComponentDetails getComponentDetails(NiFiWebRequestContext requestContext) throws ResourceNotFoundException, ClusterRequestException; + + } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistry.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistry.java b/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistry.java new file mode 100644 index 0000000..0167e6d --- /dev/null +++ b/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistry.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.registry; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestVariableRegistry { + + @Test + public void testReadMap(){ + Map<String,String> variables1 = new HashMap<>(); + variables1.put("fake.property.1","fake test value"); + + Map<String,String> variables2 = new HashMap<>(); + variables1.put("fake.property.2","fake test value"); + + VariableRegistry registry = VariableRegistryFactory.getInstance(variables1,variables2); + + Map<String,String> variables = registry.getVariables(); + assertTrue(variables.size() == 2); + assertTrue(variables.get("fake.property.1").equals("fake test value")); + assertTrue(registry.getVariableValue("fake.property.2").equals("fake test value")); + } + + @Test + public void testReadProperties(){ + Properties properties = new Properties(); + properties.setProperty("fake.property.1","fake test value"); + VariableRegistry registry = VariableRegistryFactory.getPropertiesInstance(properties); + Map<String,String> variables = registry.getVariables(); + assertTrue(variables.get("fake.property.1").equals("fake test value")); + } + + @Test + public void testReadFiles() throws IOException{ + final Path fooPath = Paths.get("src/test/resources/TestVariableRegistry/foobar.properties"); + final Path testPath = Paths.get("src/test/resources/TestVariableRegistry/test.properties"); + VariableRegistry registry = VariableRegistryFactory.getPropertiesInstance(fooPath.toFile(),testPath.toFile()); + Map<String,String> variables = registry.getVariables(); + assertTrue(variables.size() == 3); + assertTrue(variables.get("fake.property.1").equals("test me out 1")); + assertTrue(variables.get("fake.property.3").equals("test me out 3, test me out 4")); + } + + @Test + public void testReadPaths() throws IOException{ + final Path fooPath = Paths.get("src/test/resources/TestVariableRegistry/foobar.properties"); + final Path testPath = Paths.get("src/test/resources/TestVariableRegistry/test.properties"); + VariableRegistry registry = VariableRegistryFactory.getPropertiesInstance(fooPath,testPath); + Map<String,String> variables = registry.getVariables(); + assertTrue(variables.size() == 3); + assertTrue(variables.get("fake.property.1").equals("test me out 1")); + assertTrue(variables.get("fake.property.3").equals("test me out 3, test me out 4")); + } + + @Test + public void testAddRegistry() throws IOException{ + + final Path fooPath = Paths.get("src/test/resources/TestVariableRegistry/foobar.properties"); + VariableRegistry pathRegistry = VariableRegistryFactory.getPropertiesInstance(fooPath); + + final Path testPath = Paths.get("src/test/resources/TestVariableRegistry/test.properties"); + VariableRegistry fileRegistry = VariableRegistryFactory.getPropertiesInstance(testPath.toFile()); + + Properties properties = new Properties(); + properties.setProperty("fake.property.5","test me out 5"); + VariableRegistry propRegistry = VariableRegistryFactory.getPropertiesInstance(properties); + + propRegistry.addRegistry(pathRegistry); + propRegistry.addRegistry(fileRegistry); + + Map<String,String> variables = propRegistry.getVariables(); + assertTrue(variables.size() == 4); + } + + @Test + public void testNoOverwriteRegistry()throws IOException{ + final Path fooPath = Paths.get("src/test/resources/TestVariableRegistry/foobar.properties"); + VariableRegistry pathRegistry = VariableRegistryFactory.getPropertiesInstance(fooPath); + + final Path testPath = Paths.get("src/test/resources/TestVariableRegistry/test.properties"); + VariableRegistry fileRegistry = VariableRegistryFactory.getPropertiesInstance(testPath.toFile()); + + Properties properties = new Properties(); + properties.setProperty("fake.property.3","test me out 5"); + VariableRegistry propRegistry = VariableRegistryFactory.getPropertiesInstance(properties); + + propRegistry.addRegistry(pathRegistry); + propRegistry.addRegistry(fileRegistry); + + Map<String,String> variables = propRegistry.getVariables(); + String testDuplicate = propRegistry.getVariableValue("fake.property.3"); + assertTrue(variables.size() == 3); + assertTrue(testDuplicate.equals("test me out 5")); + } + + @Test + public void testGetVariableNames() throws IOException{ + final Path fooPath = Paths.get("src/test/resources/TestVariableRegistry/foobar.properties"); + final Path testPath = Paths.get("src/test/resources/TestVariableRegistry/test.properties"); + VariableRegistry registry = VariableRegistryFactory.getPropertiesInstance(fooPath,testPath); + Set<String> variableNames= registry.getVariableNames(); + assertTrue(variableNames.size() == 3); + assertTrue(variableNames.contains("fake.property.1")); + assertTrue(variableNames.contains("fake.property.2")); + assertTrue(variableNames.contains("fake.property.3")); + assertTrue(!variableNames.contains("fake.property.4")); + } + + + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistryUtils.java ---------------------------------------------------------------------- diff --git a/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistryUtils.java b/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistryUtils.java new file mode 100644 index 0000000..a1d5ea0 --- /dev/null +++ b/nifi-api/src/test/java/org/apache/nifi/registry/TestVariableRegistryUtils.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.nifi.flowfile.FlowFile; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestVariableRegistryUtils { + + @Test + public void testCreateVariableRegistry(){ + System.setProperty("fake","test"); + VariableRegistry variableRegistry = VariableRegistryUtils.createVariableRegistry(); + Map<String,String> variables = variableRegistry.getVariables(); + assertTrue(variables.containsKey("PATH")); + assertTrue(variables.get("fake").equals("test")); + } + + @Test + public void testPopulateRegistry(){ + System.setProperty("fake","test"); + FlowFile flowFile = createFlowFile(); + + VariableRegistry variableRegistry = VariableRegistryUtils.createVariableRegistry(); + VariableRegistry populatedRegistry = VariableRegistryUtils.populateRegistry(variableRegistry,flowFile,null); + Map<String,String> variables = populatedRegistry.getVariables(); + assertTrue(variables.containsKey("PATH")); + assertTrue(variables.get("fake").equals("test")); + assertTrue(variables.get("flowFileId").equals("1")); + assertTrue(variables.get("fileSize").equals("50")); + assertTrue(variables.get("entryDate").equals("1000")); + assertTrue(variables.get("lineageStartDate").equals("10000")); + assertTrue(variables.get("filename").equals("fakefile.txt")); + } + + @Test + public void testPopulateRegistryWithEmptyFlowFileAndAttributes(){ + System.setProperty("fake","test"); + VariableRegistry variableRegistry = VariableRegistryUtils.createVariableRegistry(); + VariableRegistry populatedRegistry = VariableRegistryUtils.populateRegistry(variableRegistry,null,null); + Map<String,String> variables = populatedRegistry.getVariables(); + assertTrue( variables.containsKey("PATH")); + assertTrue( variables.get("fake").equals("test")); + } + + + private FlowFile createFlowFile(){ + return new FlowFile() { + @Override + public long getId() { + return 1; + } + + @Override + public long getEntryDate() { + return 1000; + } + + @Override + public long getLineageStartDate() { + return 10000; + } + + @Override + public Long getLastQueueDate() { + return null; + } + + @Override + public Set<String> getLineageIdentifiers() { + return null; + } + + @Override + public boolean isPenalized() { + return false; + } + + @Override + public String getAttribute(String key) { + return null; + } + + @Override + public long getSize() { + return 50; + } + + @Override + public Map<String, String> getAttributes() { + Map<String,String> attributes = new HashMap<>(); + attributes.put("filename","fakefile.txt"); + return attributes; + } + + @Override + public int compareTo(FlowFile o) { + return 0; + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/test/resources/TestVariableRegistry/foobar.properties ---------------------------------------------------------------------- diff --git a/nifi-api/src/test/resources/TestVariableRegistry/foobar.properties b/nifi-api/src/test/resources/TestVariableRegistry/foobar.properties new file mode 100644 index 0000000..1094e1b --- /dev/null +++ b/nifi-api/src/test/resources/TestVariableRegistry/foobar.properties @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +fake.property.3=test me out 3, test me out 4 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-api/src/test/resources/TestVariableRegistry/test.properties ---------------------------------------------------------------------- diff --git a/nifi-api/src/test/resources/TestVariableRegistry/test.properties b/nifi-api/src/test/resources/TestVariableRegistry/test.properties new file mode 100644 index 0000000..6191449 --- /dev/null +++ b/nifi-api/src/test/resources/TestVariableRegistry/test.properties @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +fake.property.1=test me out 1 +fake.property.2=test me out 2 http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java ---------------------------------------------------------------------- diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java index 21d8e82..e80f2b1 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NotificationServiceManager.java @@ -247,7 +247,7 @@ public class NotificationServiceManager { configuredValue = fullPropDescriptor.getDefaultValue(); } - return new StandardPropertyValue(configuredValue, null); + return new StandardPropertyValue(configuredValue, null, null); } @Override @@ -364,7 +364,7 @@ public class NotificationServiceManager { value = descriptor.getDefaultValue(); } - return new StandardPropertyValue(value, null); + return new StandardPropertyValue(value, null, null); } @Override http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/NotificationValidationContext.java ---------------------------------------------------------------------- diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/NotificationValidationContext.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/NotificationValidationContext.java index 49afc16..3ca1003 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/NotificationValidationContext.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/notification/NotificationValidationContext.java @@ -48,12 +48,13 @@ public class NotificationValidationContext implements ValidationContext { @Override public PropertyValue newPropertyValue(final String rawValue) { - return new StandardPropertyValue(rawValue, null); + return new StandardPropertyValue(rawValue, null, null); } @Override public ExpressionLanguageCompiler newExpressionLanguageCompiler() { - return new StandardExpressionLanguageCompiler(); + + return new StandardExpressionLanguageCompiler(null); } @Override http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java index d85c9ef..5dec2fa 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/EmptyPreparedQuery.java @@ -16,11 +16,10 @@ */ package org.apache.nifi.attribute.expression.language; -import java.util.Map; import org.apache.nifi.expression.AttributeValueDecorator; -import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.registry.VariableRegistry; public class EmptyPreparedQuery implements PreparedQuery { @@ -31,37 +30,7 @@ public class EmptyPreparedQuery implements PreparedQuery { } @Override - public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions() throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions(final FlowFile flowFile) throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions(Map<String, String> attributes) throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions(Map<String, String> attributes, AttributeValueDecorator decorator) throws ProcessException { - return value; - } - - @Override - public String evaluateExpressions(FlowFile flowFile, Map<String, String> additionalAttributes, AttributeValueDecorator decorator) throws ProcessException { + public String evaluateExpressions(VariableRegistry variableRegistry, AttributeValueDecorator decorator) throws ProcessException { return value; } } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java index aa2428d..0ca9a8f 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/InvalidPreparedQuery.java @@ -16,12 +16,11 @@ */ package org.apache.nifi.attribute.expression.language; -import java.util.Map; import org.apache.nifi.attribute.expression.language.exception.AttributeExpressionLanguageException; import org.apache.nifi.expression.AttributeValueDecorator; -import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.registry.VariableRegistry; /** * An implementation of PreparedQuery that throws an @@ -40,37 +39,9 @@ public class InvalidPreparedQuery implements PreparedQuery { } @Override - public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException { + public String evaluateExpressions(final VariableRegistry variableRegistry, final AttributeValueDecorator decorator) throws ProcessException { throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); } - @Override - public String evaluateExpressions() throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } - - @Override - public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } - - @Override - public String evaluateExpressions(final FlowFile flowFile) throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } - @Override - public String evaluateExpressions(final Map<String, String> attributes) throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } - - @Override - public String evaluateExpressions(final Map<String, String> attributes, final AttributeValueDecorator decorator) throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } - - @Override - public String evaluateExpressions(FlowFile flowFile, Map<String, String> additionalAttributes, AttributeValueDecorator decorator) throws ProcessException { - throw new AttributeExpressionLanguageException("Invalid Expression: " + query + " due to " + explanation); - } } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java index ad9225d..37d8b86 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/PreparedQuery.java @@ -16,25 +16,13 @@ */ package org.apache.nifi.attribute.expression.language; -import java.util.Map; import org.apache.nifi.expression.AttributeValueDecorator; -import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.registry.VariableRegistry; public interface PreparedQuery { - String evaluateExpressions(FlowFile flowFile, AttributeValueDecorator decorator) throws ProcessException; + String evaluateExpressions(VariableRegistry registry, AttributeValueDecorator decorator) throws ProcessException; - String evaluateExpressions() throws ProcessException; - - String evaluateExpressions(AttributeValueDecorator decorator) throws ProcessException; - - String evaluateExpressions(FlowFile flowFile) throws ProcessException; - - String evaluateExpressions(Map<String, String> attributes) throws ProcessException; - - String evaluateExpressions(Map<String, String> attributes, AttributeValueDecorator decorator) throws ProcessException; - - String evaluateExpressions(FlowFile flowFile, Map<String, String> additionalAttributes, AttributeValueDecorator decorator) throws ProcessException; } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java index d39f0f8..f2d3915 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java @@ -18,13 +18,9 @@ package org.apache.nifi.attribute.expression.language; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionLexer; @@ -198,6 +194,7 @@ import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpre import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID; import org.apache.nifi.attribute.expression.language.evaluation.selection.MappingEvaluator; +import org.apache.nifi.registry.VariableRegistry; /** * Class used for creating and evaluating NiFi Expression Language. Once a Query @@ -367,8 +364,8 @@ public class Query { return -1; } - static String evaluateExpression(final Tree tree, final String queryText, final Map<String, String> expressionMap, final AttributeValueDecorator decorator) throws ProcessException { - final Object evaluated = Query.fromTree(tree, queryText).evaluate(expressionMap).getValue(); + static String evaluateExpression(final Tree tree, final String queryText, final VariableRegistry registry, final AttributeValueDecorator decorator) throws ProcessException { + final Object evaluated = Query.fromTree(tree, queryText).evaluate(registry).getValue(); if (evaluated == null) { return null; } @@ -378,29 +375,12 @@ public class Query { return decorator == null ? escaped : decorator.decorate(escaped); } - static String evaluateExpressions(final String rawValue, Map<String, String> expressionMap) throws ProcessException { - return evaluateExpressions(rawValue, expressionMap, null); + static String evaluateExpressions(final String rawValue, VariableRegistry registry) throws ProcessException { + return evaluateExpressions(rawValue, registry, null); } - static String evaluateExpressions(final String rawValue) throws ProcessException { - return evaluateExpressions(rawValue, createExpressionMap(null), null); - } - - static String evaluateExpressions(final String rawValue, final FlowFile flowFile) throws ProcessException { - return evaluateExpressions(rawValue, createExpressionMap(flowFile), null); - } - - static String evaluateExpressions(final String rawValue, Map<String, String> expressionMap, final AttributeValueDecorator decorator) throws ProcessException { - return Query.prepare(rawValue).evaluateExpressions(expressionMap, decorator); - } - - public static String evaluateExpressions(final String rawValue, final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException { - if (rawValue == null) { - return null; - } - - final Map<String, String> expressionMap = createExpressionMap(flowFile); - return evaluateExpressions(rawValue, expressionMap, decorator); + static String evaluateExpressions(final String rawValue, VariableRegistry registry, final AttributeValueDecorator decorator) throws ProcessException { + return Query.prepare(rawValue).evaluateExpressions(registry, decorator); } private static Evaluator<?> getRootSubjectEvaluator(final Evaluator<?> evaluator) { @@ -426,150 +406,6 @@ public class Query { return value.replaceAll("\\$\\$(?=\\$*\\{.*?\\})", "\\$"); } - static Map<String, String> createExpressionMap(final FlowFile flowFile) { - return createExpressionMap(flowFile, null); - } - - static Map<String, String> createExpressionMap(final FlowFile flowFile, final Map<String, String> additionalAttributes) { - final Map<String, String> attributeMap = flowFile == null ? Collections.<String, String> emptyMap() : flowFile.getAttributes(); - final Map<String, String> additionalOrEmpty = additionalAttributes == null ? Collections.<String, String> emptyMap() : additionalAttributes; - final Map<String, String> envMap = System.getenv(); - final Map<?, ?> sysProps = System.getProperties(); - - final Map<String, String> flowFileProps = new HashMap<>(); - if (flowFile != null) { - flowFileProps.put("flowFileId", String.valueOf(flowFile.getId())); - flowFileProps.put("fileSize", String.valueOf(flowFile.getSize())); - flowFileProps.put("entryDate", String.valueOf(flowFile.getEntryDate())); - flowFileProps.put("lineageStartDate", String.valueOf(flowFile.getLineageStartDate())); - } - - return wrap(additionalOrEmpty, attributeMap, flowFileProps, envMap, sysProps); - } - - private static Map<String, String> wrap(final Map<String, String> additional, final Map<String, String> attributes, final Map<String, String> flowFileProps, - final Map<String, String> env, final Map<?, ?> sysProps) { - @SuppressWarnings("rawtypes") - final Map[] maps = new Map[] {additional, attributes, flowFileProps, env, sysProps}; - - return new Map<String, String>() { - @Override - public int size() { - int size = 0; - for (final Map<?, ?> map : maps) { - size += map.size(); - } - return size; - } - - @Override - public boolean isEmpty() { - for (final Map<?, ?> map : maps) { - if (!map.isEmpty()) { - return false; - } - } - return true; - } - - @Override - public boolean containsKey(final Object key) { - if (key == null) { - return false; - } - if (!(key instanceof String)) { - return false; - } - - for (final Map<?, ?> map : maps) { - if (map.containsKey(key)) { - return true; - } - } - return false; - } - - @Override - public boolean containsValue(final Object value) { - for (final Map<?, ?> map : maps) { - if (map.containsValue(value)) { - return true; - } - } - return false; - } - - @Override - @SuppressWarnings("rawtypes") - public String get(final Object key) { - if (key == null) { - throw new IllegalArgumentException("Null Keys are not allowed"); - } - if (!(key instanceof String)) { - return null; - } - - for (final Map map : maps) { - final Object val = map.get(key); - if (val != null) { - return String.valueOf(val); - } - } - return null; - } - - @Override - public String put(String key, String value) { - throw new UnsupportedOperationException(); - } - - @Override - public String remove(final Object key) { - throw new UnsupportedOperationException(); - } - - @Override - public void putAll(final Map<? extends String, ? extends String> m) { - throw new UnsupportedOperationException(); - } - - @Override - public void clear() { - throw new UnsupportedOperationException(); - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Set<String> keySet() { - final Set<String> keySet = new HashSet<>(); - for (final Map map : maps) { - keySet.addAll(map.keySet()); - } - return keySet; - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Collection<String> values() { - final Set<String> values = new HashSet<>(); - for (final Map map : maps) { - values.addAll(map.values()); - } - return values; - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Set<java.util.Map.Entry<String, String>> entrySet() { - final Set<java.util.Map.Entry<String, String>> entrySet = new HashSet<>(); - for (final Map map : maps) { - entrySet.addAll(map.entrySet()); - } - return entrySet; - } - - }; - } public static Query fromTree(final Tree tree, final String text) { return new Query(text, tree, buildEvaluator(tree)); @@ -706,20 +542,12 @@ public class Query { return evaluator.getResultType(); } - QueryResult<?> evaluate() { - return evaluate(createExpressionMap(null)); - } - - QueryResult<?> evaluate(final FlowFile flowFile) { - return evaluate(createExpressionMap(flowFile)); - } - - QueryResult<?> evaluate(final Map<String, String> attributes) { + QueryResult<?> evaluate(final VariableRegistry registry) { if (evaluated.getAndSet(true)) { throw new IllegalStateException("A Query cannot be evaluated more than once"); } - return evaluator.evaluate(attributes); + return evaluator.evaluate(registry.getVariables()); } Tree getTree() { http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java index 49ef6ef..cbfb511 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardAttributeExpression.java @@ -20,13 +20,17 @@ import org.apache.nifi.expression.AttributeExpression; import org.apache.nifi.expression.AttributeValueDecorator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.registry.VariableRegistryUtils; public class StandardAttributeExpression implements AttributeExpression { private final Query query; + private final VariableRegistry variableRegistry; - public StandardAttributeExpression(final Query query) { + public StandardAttributeExpression(final Query query, final VariableRegistry variableRegistry) { this.query = query; + this.variableRegistry = variableRegistry; } @Override @@ -51,7 +55,8 @@ public class StandardAttributeExpression implements AttributeExpression { @Override public String evaluate(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException { - final Object evaluationResult = query.evaluate(flowFile).getValue(); + VariableRegistry flowFileRegistry = VariableRegistryUtils.populateRegistry(variableRegistry,flowFile,null); + final Object evaluationResult = query.evaluate(flowFileRegistry).getValue(); if (evaluationResult == null) { return ""; } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java index cec73d1..e85853f 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardExpressionLanguageCompiler.java @@ -20,13 +20,20 @@ import org.apache.nifi.attribute.expression.language.exception.AttributeExpressi import org.apache.nifi.expression.AttributeExpression; import org.apache.nifi.expression.ExpressionLanguageCompiler; import org.apache.nifi.expression.AttributeExpression.ResultType; +import org.apache.nifi.registry.VariableRegistry; public class StandardExpressionLanguageCompiler implements ExpressionLanguageCompiler { + private final VariableRegistry variableRegistry; + + public StandardExpressionLanguageCompiler(final VariableRegistry variableRegistry) { + this.variableRegistry = variableRegistry; + } + @Override public AttributeExpression compile(final String expression) throws IllegalArgumentException { try { - return new StandardAttributeExpression(Query.compile(expression)); + return new StandardAttributeExpression(Query.compile(expression),variableRegistry); } catch (final AttributeExpressionLanguageParsingException e) { throw new IllegalArgumentException(e.getMessage()); } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java index b81a583..4ee3e02 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPreparedQuery.java @@ -22,10 +22,10 @@ import java.util.List; import java.util.Map; import org.apache.nifi.expression.AttributeValueDecorator; -import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.exception.ProcessException; import org.antlr.runtime.tree.Tree; +import org.apache.nifi.registry.VariableRegistry; public class StandardPreparedQuery implements PreparedQuery { @@ -37,20 +37,16 @@ public class StandardPreparedQuery implements PreparedQuery { this.trees = new HashMap<>(trees); } - @Override - public String evaluateExpressions(Map<String, String> attributes) throws ProcessException { - return evaluateExpressions(attributes, null); - } @Override - public String evaluateExpressions(final Map<String, String> attributes, final AttributeValueDecorator decorator) throws ProcessException { + public String evaluateExpressions(final VariableRegistry registry, final AttributeValueDecorator decorator) throws ProcessException { final StringBuilder sb = new StringBuilder(); for (final String val : queryStrings) { final Tree tree = trees.get(val); if (tree == null) { sb.append(val); } else { - final String evaluated = Query.evaluateExpression(tree, val, attributes, decorator); + final String evaluated = Query.evaluateExpression(tree, val, registry, decorator); if (evaluated != null) { sb.append(evaluated); } @@ -59,31 +55,4 @@ public class StandardPreparedQuery implements PreparedQuery { return sb.toString(); } - @Override - public String evaluateExpressions(final FlowFile flowFile, final Map<String, String> additionalAttributes, final AttributeValueDecorator decorator) throws ProcessException { - final Map<String, String> expressionMap = Query.createExpressionMap(flowFile, additionalAttributes); - return evaluateExpressions(expressionMap, decorator); - } - - @Override - public String evaluateExpressions(final FlowFile flowFile, final AttributeValueDecorator decorator) throws ProcessException { - final Map<String, String> expressionMap = Query.createExpressionMap(flowFile); - return evaluateExpressions(expressionMap, decorator); - } - - @Override - public String evaluateExpressions() throws ProcessException { - return evaluateExpressions((FlowFile) null, null); - } - - @Override - public String evaluateExpressions(final AttributeValueDecorator decorator) throws ProcessException { - return evaluateExpressions((FlowFile) null, decorator); - } - - @Override - public String evaluateExpressions(final FlowFile flowFile) throws ProcessException { - return evaluateExpressions(flowFile, null); - } - } http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPropertyValue.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPropertyValue.java b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPropertyValue.java index a5c336a..c3cff49 100644 --- a/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPropertyValue.java +++ b/nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/StandardPropertyValue.java @@ -26,6 +26,8 @@ import org.apache.nifi.expression.AttributeValueDecorator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.DataUnit; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.registry.VariableRegistryUtils; import org.apache.nifi.util.FormatUtils; public class StandardPropertyValue implements PropertyValue { @@ -33,24 +35,27 @@ public class StandardPropertyValue implements PropertyValue { private final String rawValue; private final ControllerServiceLookup serviceLookup; private final PreparedQuery preparedQuery; + private final VariableRegistry variableRegistry; - public StandardPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup) { - this(rawValue, serviceLookup, Query.prepare(rawValue)); + public StandardPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup, final VariableRegistry variableRegistry) { + this(rawValue, serviceLookup, Query.prepare(rawValue), variableRegistry); } /** * Constructs a new StandardPropertyValue with the given value & service lookup and indicates whether or not the rawValue contains any NiFi Expressions. If it is unknown whether or not the value - * contains any NiFi Expressions, the {@link #StandardPropertyValue(String, ControllerServiceLookup)} constructor should be used or <code>true</code> should be passed. However, if it is known that - * the value contains no NiFi Expression, that information should be provided so that calls to {@link #evaluateAttributeExpressions()} are much more efficient - * + * contains any NiFi Expressions, the {@link #StandardPropertyValue(String, ControllerServiceLookup, VariableRegistry)} constructor should be used or <code>true</code> should be passed. + * However, if it is known that the value contains no NiFi Expression, that information should be provided so that calls to {@link #evaluateAttributeExpressions()} are much more efficient * @param rawValue value * @param serviceLookup lookup * @param preparedQuery query + * @param variableRegistry variableRegistry */ - public StandardPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup, final PreparedQuery preparedQuery) { + public StandardPropertyValue(final String rawValue, final ControllerServiceLookup serviceLookup, final PreparedQuery preparedQuery, + final VariableRegistry variableRegistry) { this.rawValue = rawValue; this.serviceLookup = serviceLookup; this.preparedQuery = preparedQuery; + this.variableRegistry = variableRegistry; } @Override @@ -134,9 +139,14 @@ public class StandardPropertyValue implements PropertyValue { return this; } - return new StandardPropertyValue(preparedQuery.evaluateExpressions(flowFile, additionalAttributes, decorator), serviceLookup, null); + VariableRegistry sysRegistry = variableRegistry != null ? variableRegistry : VariableRegistryUtils.createVariableRegistry(); + VariableRegistry registry = VariableRegistryUtils.populateRegistry(sysRegistry,flowFile,additionalAttributes); + + return new StandardPropertyValue(preparedQuery.evaluateExpressions(registry,decorator), serviceLookup,null,registry); } + + @Override public String toString() { return rawValue; http://git-wip-us.apache.org/repos/asf/nifi/blob/c600f150/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy b/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy index bdd704d..3a5f587 100644 --- a/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy +++ b/nifi-commons/nifi-expression-language/src/test/groovy/org/apache/nifi/attribute/expression/language/QueryGroovyTest.groovy @@ -18,6 +18,8 @@ package org.apache.nifi.attribute.expression.language import org.apache.nifi.attribute.expression.language.evaluation.QueryResult import org.apache.nifi.expression.AttributeExpression +import org.apache.nifi.registry.VariableRegistry +import org.apache.nifi.registry.VariableRegistryFactory import org.junit.After import org.junit.Before import org.junit.BeforeClass @@ -57,6 +59,7 @@ public class QueryGroovyTest extends GroovyTestCase { final Map<String, String> attributes = [ single : ORIGINAL_VALUE, repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + final VariableRegistry variableRegistry = VariableRegistryFactory.getInstance(attributes) logger.info("Attributes: ${attributes}") final String REPLACEMENT_VALUE = "Goodbye Planet" @@ -75,10 +78,10 @@ public class QueryGroovyTest extends GroovyTestCase { Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) // Act - QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes) + QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(variableRegistry) logger.info("Replace single result: ${replaceSingleResult.value}") - QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(variableRegistry) logger.info("Replace repeating result: ${replaceRepeatingResult.value}") // Assert @@ -97,6 +100,7 @@ public class QueryGroovyTest extends GroovyTestCase { final Map<String, String> attributes = [ single : ORIGINAL_VALUE, repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + final VariableRegistry variableRegistry = VariableRegistryFactory.getInstance(attributes) logger.info("Attributes: ${attributes}") final String REPLACEMENT_VALUE = "Goodbye Planet" @@ -115,10 +119,10 @@ public class QueryGroovyTest extends GroovyTestCase { Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) // Act - QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes) + QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(variableRegistry) logger.info("Replace single result: ${replaceSingleResult.value}") - QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(variableRegistry) logger.info("Replace repeating result: ${replaceRepeatingResult.value}") // Assert @@ -137,6 +141,7 @@ public class QueryGroovyTest extends GroovyTestCase { final Map<String, String> attributes = [ single : ORIGINAL_VALUE, repeating: [ORIGINAL_VALUE].multiply(n).join(" ")] + final VariableRegistry variableRegistry = VariableRegistryFactory.getInstance(attributes) logger.info("Attributes: ${attributes}") final String REPLACEMENT_VALUE = "Goodbye Planet" @@ -155,10 +160,10 @@ public class QueryGroovyTest extends GroovyTestCase { Query replaceRepeatingQuery = Query.compile(REPLACE_REPEATING_EXPRESSION) // Act - QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(attributes) + QueryResult<?> replaceSingleResult = replaceSingleQuery.evaluate(variableRegistry) logger.info("Replace single result: ${replaceSingleResult.value}") - QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(attributes) + QueryResult<?> replaceRepeatingResult = replaceRepeatingQuery.evaluate(variableRegistry) logger.info("Replace repeating result: ${replaceRepeatingResult.value}") // Assert
