http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java new file mode 100644 index 0000000..2d1a407 --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/FlowFileFilters.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processor.util; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.FlowFileFilter; + +public class FlowFileFilters { + + /** + * Returns a new {@link FlowFileFilter} that will pull FlowFiles until the + * maximum file size has been reached, or the maximum FlowFile Count was + * been reached (this is important because FlowFiles may be 0 bytes!). If + * the first FlowFile exceeds the max size, the FlowFile will be selected + * and no other FlowFile will be. + * + * @param maxSize the maximum size of the group of FlowFiles + * @param unit the unit of the <code>maxSize</code> argument + * @param maxCount the maximum number of FlowFiles to pull + * @return filter + */ + public static FlowFileFilter newSizeBasedFilter(final double maxSize, final DataUnit unit, final int maxCount) { + final double maxBytes = DataUnit.B.convert(maxSize, unit); + + return new FlowFileFilter() { + int count = 0; + long size = 0L; + + @Override + public FlowFileFilterResult filter(final FlowFile flowFile) { + if (count == 0) { + count++; + size += flowFile.getSize(); + + return FlowFileFilterResult.ACCEPT_AND_CONTINUE; + } + + if ((size + flowFile.getSize() > maxBytes) || (count + 1 > maxCount)) { + return FlowFileFilterResult.REJECT_AND_TERMINATE; + } + + count++; + size += flowFile.getSize(); + return FlowFileFilterResult.ACCEPT_AND_CONTINUE; + } + + }; + } + +}
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java new file mode 100644 index 0000000..dca15b0 --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/SSLProperties.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processor.util; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.net.ssl.SSLContext; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.security.util.CertificateUtils; +import org.apache.nifi.security.util.KeystoreType; +import org.apache.nifi.security.util.SslContextFactory; +import org.apache.nifi.security.util.SslContextFactory.ClientAuth; + +public class SSLProperties { + + public static final PropertyDescriptor TRUSTSTORE = new PropertyDescriptor.Builder() + .name("Truststore Filename") + .description("The fully-qualified filename of the Truststore") + .defaultValue(null) + .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) + .sensitive(false) + .build(); + + public static final PropertyDescriptor TRUSTSTORE_TYPE = new PropertyDescriptor.Builder() + .name("Truststore Type") + .description("The Type of the Truststore. Either JKS or PKCS12") + .allowableValues("JKS", "PKCS12") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue(null) + .sensitive(false) + .build(); + + public static final PropertyDescriptor TRUSTSTORE_PASSWORD = new PropertyDescriptor.Builder() + .name("Truststore Password") + .description("The password for the Truststore") + .defaultValue(null) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + + public static final PropertyDescriptor KEYSTORE = new PropertyDescriptor.Builder() + .name("Keystore Filename") + .description("The fully-qualified filename of the Keystore") + .defaultValue(null) + .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) + .sensitive(false) + .build(); + + public static final PropertyDescriptor KEYSTORE_TYPE = new PropertyDescriptor.Builder() + .name("Keystore Type") + .description("The Type of the Keystore") + .allowableValues("JKS", "PKCS12") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(false) + .build(); + + public static final PropertyDescriptor KEYSTORE_PASSWORD = new PropertyDescriptor.Builder() + .name("Keystore Password") + .defaultValue(null) + .description("The password for the Keystore") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + + public static Collection<ValidationResult> validateStore(final Map<PropertyDescriptor, String> properties) { + final Collection<ValidationResult> results = new ArrayList<>(); + results.addAll(validateStore(properties, KeystoreValidationGroup.KEYSTORE)); + results.addAll(validateStore(properties, KeystoreValidationGroup.TRUSTSTORE)); + return results; + } + + public static Collection<ValidationResult> validateStore(final Map<PropertyDescriptor, String> properties, final KeystoreValidationGroup keyStoreOrTrustStore) { + final Collection<ValidationResult> results = new ArrayList<>(); + + final String filename; + final String password; + final String type; + + if (keyStoreOrTrustStore == KeystoreValidationGroup.KEYSTORE) { + filename = properties.get(KEYSTORE); + password = properties.get(KEYSTORE_PASSWORD); + type = properties.get(KEYSTORE_TYPE); + } else { + filename = properties.get(TRUSTSTORE); + password = properties.get(TRUSTSTORE_PASSWORD); + type = properties.get(TRUSTSTORE_TYPE); + } + + final String keystoreDesc = (keyStoreOrTrustStore == KeystoreValidationGroup.KEYSTORE) ? "Keystore" : "Truststore"; + + final int nulls = countNulls(filename, password, type); + if (nulls != 3 && nulls != 0) { + results.add(new ValidationResult.Builder().valid(false).explanation("Must set either 0 or 3 properties for " + keystoreDesc).subject(keystoreDesc + " Properties").build()); + } else if (nulls == 0) { + // all properties were filled in. + final File file = new File(filename); + if (!file.exists() || !file.canRead()) { + results.add(new ValidationResult.Builder().valid(false).subject(keystoreDesc + " Properties").explanation("Cannot access file " + file.getAbsolutePath()).build()); + } else { + try { + final boolean storeValid = CertificateUtils.isStoreValid(file.toURI().toURL(), KeystoreType.valueOf(type), password.toCharArray()); + if (!storeValid) { + results.add( + new ValidationResult.Builder() + .subject(keystoreDesc + " Properties") + .valid(false) + .explanation("Invalid KeyStore Password or Type specified for file " + filename) + .build() + ); + } + } catch (MalformedURLException e) { + results.add(new ValidationResult.Builder().subject(keystoreDesc + " Properties").valid(false).explanation("Malformed URL from file: " + e).build()); + } + } + } + + return results; + } + + private static int countNulls(Object... objects) { + int count = 0; + for (final Object x : objects) { + if (x == null) { + count++; + } + } + + return count; + } + + public static enum KeystoreValidationGroup { + + KEYSTORE, TRUSTSTORE + } + + private static final String DEFAULT_SSL_PROTOCOL_ALGORITHM = "TLS"; + + public static List<PropertyDescriptor> getKeystoreDescriptors(final boolean required) { + final List<PropertyDescriptor> descriptors = new ArrayList<>(); + for (final PropertyDescriptor descriptor : KEYSTORE_DESCRIPTORS) { + final PropertyDescriptor.Builder builder = new PropertyDescriptor.Builder().fromPropertyDescriptor(descriptor).required(required); + if (required && descriptor.getName().equals(KEYSTORE_TYPE.getName())) { + builder.defaultValue("JKS"); + } + descriptors.add(builder.build()); + } + + return descriptors; + } + + public static List<PropertyDescriptor> getTruststoreDescriptors(final boolean required) { + final List<PropertyDescriptor> descriptors = new ArrayList<>(); + for (final PropertyDescriptor descriptor : TRUSTSTORE_DESCRIPTORS) { + final PropertyDescriptor.Builder builder = new PropertyDescriptor.Builder().fromPropertyDescriptor(descriptor).required(required); + if (required && descriptor.getName().equals(TRUSTSTORE_TYPE.getName())) { + builder.defaultValue("JKS"); + } + descriptors.add(builder.build()); + } + + return descriptors; + } + + public static SSLContext createSSLContext(final ProcessContext context, final ClientAuth clientAuth) + throws UnrecoverableKeyException, KeyManagementException, KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { + final String keystoreFile = context.getProperty(KEYSTORE).getValue(); + if (keystoreFile == null) { + return SslContextFactory.createTrustSslContext( + context.getProperty(TRUSTSTORE).getValue(), + context.getProperty(TRUSTSTORE_PASSWORD).getValue().toCharArray(), + context.getProperty(TRUSTSTORE_TYPE).getValue(), + DEFAULT_SSL_PROTOCOL_ALGORITHM); + } else { + final String truststoreFile = context.getProperty(TRUSTSTORE).getValue(); + if (truststoreFile == null) { + return SslContextFactory.createSslContext( + context.getProperty(KEYSTORE).getValue(), + context.getProperty(KEYSTORE_PASSWORD).getValue().toCharArray(), + context.getProperty(KEYSTORE_TYPE).getValue(), DEFAULT_SSL_PROTOCOL_ALGORITHM); + } else { + return SslContextFactory.createSslContext( + context.getProperty(KEYSTORE).getValue(), + context.getProperty(KEYSTORE_PASSWORD).getValue().toCharArray(), + context.getProperty(KEYSTORE_TYPE).getValue(), + context.getProperty(TRUSTSTORE).getValue(), + context.getProperty(TRUSTSTORE_PASSWORD).getValue().toCharArray(), + context.getProperty(TRUSTSTORE_TYPE).getValue(), + clientAuth, + DEFAULT_SSL_PROTOCOL_ALGORITHM); + } + } + } + + private static final Set<PropertyDescriptor> KEYSTORE_DESCRIPTORS = new HashSet<>(); + private static final Set<PropertyDescriptor> TRUSTSTORE_DESCRIPTORS = new HashSet<>(); + + static { + KEYSTORE_DESCRIPTORS.add(KEYSTORE); + KEYSTORE_DESCRIPTORS.add(KEYSTORE_TYPE); + KEYSTORE_DESCRIPTORS.add(KEYSTORE_PASSWORD); + + TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE); + TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE_TYPE); + TRUSTSTORE_DESCRIPTORS.add(TRUSTSTORE_PASSWORD); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java new file mode 100644 index 0000000..37ba7d8 --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java @@ -0,0 +1,708 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processor.util; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.expression.AttributeExpression.ResultType; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.util.FormatUtils; + +public class StandardValidators { + + // + // + // STATICALLY DEFINED VALIDATORS + // + // + public static final Validator ATTRIBUTE_KEY_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + final ValidationResult.Builder builder = new ValidationResult.Builder(); + builder.subject(subject).input(input); + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return builder.valid(true).explanation("Contains Expression Language").build(); + } + + try { + FlowFile.KeyValidator.validateKey(input); + builder.valid(true); + } catch (final IllegalArgumentException e) { + builder.valid(false).explanation(e.getMessage()); + } + + return builder.build(); + } + }; + + public static final Validator ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + final ValidationResult.Builder builder = new ValidationResult.Builder(); + builder.subject("Property Name").input(subject); + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return builder.valid(true).explanation("Contains Expression Language").build(); + } + + try { + FlowFile.KeyValidator.validateKey(subject); + builder.valid(true); + } catch (final IllegalArgumentException e) { + builder.valid(false).explanation(e.getMessage()); + } + + return builder.build(); + } + }; + + public static final Validator POSITIVE_INTEGER_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + final int intVal = Integer.parseInt(value); + + if (intVal <= 0) { + reason = "not a positive value"; + } + } catch (final NumberFormatException e) { + reason = "not a valid integer"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + public static final Validator POSITIVE_LONG_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + final long longVal = Long.parseLong(value); + + if (longVal <= 0) { + reason = "not a positive value"; + } + } catch (final NumberFormatException e) { + reason = "not a valid 64-bit integer"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + public static final Validator PORT_VALIDATOR = createLongValidator(1, 65535, true); + + public static final Validator NON_EMPTY_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + return new ValidationResult.Builder().subject(subject).input(value).valid(value != null && !value.isEmpty()).explanation(subject + " cannot be empty").build(); + } + }; + + public static final Validator BOOLEAN_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + final boolean valid = "true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value); + final String explanation = valid ? null : "Value must be 'true' or 'false'"; + return new ValidationResult.Builder().subject(subject).input(value).valid(valid).explanation(explanation).build(); + } + }; + + public static final Validator INTEGER_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + Integer.parseInt(value); + } catch (final NumberFormatException e) { + reason = "not a valid integer"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + public static final Validator LONG_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + Long.parseLong(value); + } catch (final NumberFormatException e) { + reason = "not a valid Long"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + public static final Validator NON_NEGATIVE_INTEGER_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + final int intVal = Integer.parseInt(value); + + if (intVal < 0) { + reason = "value is negative"; + } + } catch (final NumberFormatException e) { + reason = "value is not a valid integer"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + public static final Validator CHARACTER_SET_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + if (!Charset.isSupported(value)) { + reason = "Character Set is not supported by this JVM."; + } + } catch (final UnsupportedCharsetException uce) { + reason = "Character Set is not supported by this JVM."; + } catch (final IllegalArgumentException iae) { + reason = "Character Set value cannot be null."; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + }; + + /** + * URL Validator that does not allow the Expression Language to be used + */ + public static final Validator URL_VALIDATOR = createURLValidator(); + + public static final Validator URI_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + try { + new URI(input); + return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid URI").valid(true).build(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URI").valid(false).build(); + } + } + }; + + public static final Validator REGULAR_EXPRESSION_VALIDATOR = createRegexValidator(0, Integer.MAX_VALUE, false); + + public static final Validator ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + try { + context.newExpressionLanguageCompiler().compile(input); + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(e.getMessage()).build(); + } + } + + }; + + public static final Validator TIME_PERIOD_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + if (input == null) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Time Period cannot be null").build(); + } + if (Pattern.compile(FormatUtils.TIME_DURATION_REGEX).matcher(input.toLowerCase()).matches()) { + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } else { + return new ValidationResult.Builder() + .subject(subject) + .input(input) + .valid(false) + .explanation("Must be of format <duration> <TimeUnit> where <duration> is a " + + "non-negative integer and TimeUnit is a supported Time Unit, such " + + "as: nanos, millis, secs, mins, hrs, days") + .build(); + } + } + }; + + public static final Validator DATA_SIZE_VALIDATOR = new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + if (input == null) { + return new ValidationResult.Builder() + .subject(subject) + .input(input) + .valid(false) + .explanation("Data Size cannot be null") + .build(); + } + if (Pattern.compile(DataUnit.DATA_SIZE_REGEX).matcher(input.toUpperCase()).matches()) { + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } else { + return new ValidationResult.Builder() + .subject(subject).input(input) + .valid(false) + .explanation("Must be of format <Data Size> <Data Unit> where <Data Size>" + + " is a non-negative integer and <Data Unit> is a supported Data" + + " Unit, such as: B, KB, MB, GB, TB") + .build(); + } + } + }; + + public static final Validator FILE_EXISTS_VALIDATOR = new FileExistsValidator(true); + + // + // + // FACTORY METHODS FOR VALIDATORS + // + // + public static Validator createDirectoryExistsValidator(final boolean allowExpressionLanguage, final boolean createDirectoryIfMissing) { + return new DirectoryExistsValidator(allowExpressionLanguage, createDirectoryIfMissing); + } + + private static Validator createURLValidator() { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + try { + final String evaluatedInput = context.newPropertyValue(input).evaluateAttributeExpressions().getValue(); + new URL(evaluatedInput); + return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid URL").valid(true).build(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URL").valid(false).build(); + } + } + }; + } + + public static Validator createTimePeriodValidator(final long minTime, final TimeUnit minTimeUnit, final long maxTime, final TimeUnit maxTimeUnit) { + return new TimePeriodValidator(minTime, minTimeUnit, maxTime, maxTimeUnit); + } + + public static Validator createAttributeExpressionLanguageValidator(final ResultType expectedResultType) { + return createAttributeExpressionLanguageValidator(expectedResultType, true); + } + + public static Validator createDataSizeBoundsValidator(final long minBytesInclusive, final long maxBytesInclusive) { + return new Validator() { + + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + final ValidationResult vr = DATA_SIZE_VALIDATOR.validate(subject, input, context); + if (!vr.isValid()) { + return vr; + } + final long dataSizeBytes = DataUnit.parseDataSize(input, DataUnit.B).longValue(); + if (dataSizeBytes < minBytesInclusive) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Cannot be smaller than " + minBytesInclusive + " bytes").build(); + } + if (dataSizeBytes > maxBytesInclusive) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Cannot be larger than " + maxBytesInclusive + " bytes").build(); + } + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } + }; + + } + + public static Validator createRegexMatchingValidator(final Pattern pattern) { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + final boolean matches = pattern.matcher(input).matches(); + return new ValidationResult.Builder() + .input(input) + .subject(subject) + .valid(matches) + .explanation(matches ? null : "Value does not match regular expression: " + pattern.pattern()) + .build(); + } + }; + } + + /** + * Creates a @{link Validator} that ensure that a value is a valid Java + * Regular Expression with at least <code>minCapturingGroups</code> + * capturing groups and at most <code>maxCapturingGroups</code> capturing + * groups. If <code>supportAttributeExpressionLanguage</code> is set to + * <code>true</code>, the value may also include the Expression Language, + * but the result of evaluating the Expression Language will be applied + * before the Regular Expression is performed. In this case, the Expression + * Language will not support FlowFile Attributes but only System/JVM + * Properties + * + * @param minCapturingGroups minimum capturing groups allowed + * @param maxCapturingGroups maximum capturing groups allowed + * @param supportAttributeExpressionLanguage whether or not to support + * expression language + * @return validator + */ + public static Validator createRegexValidator(final int minCapturingGroups, final int maxCapturingGroups, final boolean supportAttributeExpressionLanguage) { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + try { + final String substituted; + if (supportAttributeExpressionLanguage) { + try { + substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue(); + } catch (final Exception e) { + return new ValidationResult.Builder() + .subject(subject) + .input(value) + .valid(false) + .explanation("Failed to evaluate the Attribute Expression Language due to " + e.toString()) + .build(); + } + } else { + substituted = value; + } + + final Pattern pattern = Pattern.compile(substituted); + final int numGroups = pattern.matcher("").groupCount(); + if (numGroups < minCapturingGroups || numGroups > maxCapturingGroups) { + return new ValidationResult.Builder() + .subject(subject) + .input(value) + .valid(false) + .explanation("RegEx is required to have between " + minCapturingGroups + " and " + maxCapturingGroups + " Capturing Groups but has " + numGroups) + .build(); + } + + return new ValidationResult.Builder().subject(subject).input(value).valid(true).build(); + } catch (final Exception e) { + return new ValidationResult.Builder() + .subject(subject) + .input(value) + .valid(false) + .explanation("Not a valid Java Regular Expression") + .build(); + } + + } + }; + } + + public static Validator createAttributeExpressionLanguageValidator(final ResultType expectedResultType, final boolean allowExtraCharacters) { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + final String syntaxError = context.newExpressionLanguageCompiler().validateExpression(input, allowExtraCharacters); + if (syntaxError != null) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation(syntaxError).build(); + } + + final ResultType resultType = allowExtraCharacters ? ResultType.STRING : context.newExpressionLanguageCompiler().getResultType(input); + if (!resultType.equals(expectedResultType)) { + return new ValidationResult.Builder() + .subject(subject) + .input(input) + .valid(false) + .explanation("Expected Attribute Query to return type " + expectedResultType + " but query returns type " + resultType) + .build(); + } + + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } + }; + } + + public static Validator createLongValidator(final long minimum, final long maximum, final boolean inclusive) { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + String reason = null; + try { + final long longVal = Long.parseLong(input); + if (longVal < minimum || (!inclusive && longVal == minimum) | longVal > maximum || (!inclusive && longVal == maximum)) { + reason = "Value must be between " + minimum + " and " + maximum + " (" + (inclusive ? "inclusive" : "exclusive") + ")"; + } + } catch (final NumberFormatException e) { + reason = "not a valid integer"; + } + + return new ValidationResult.Builder().subject(subject).input(input).explanation(reason).valid(reason == null).build(); + } + + }; + } + + // + // + // SPECIFIC VALIDATOR IMPLEMENTATIONS THAT CANNOT BE ANONYMOUS CLASSES + // + // + static class TimePeriodValidator implements Validator { + + private final Pattern pattern; + + private final long minNanos; + private final long maxNanos; + + private final String minValueEnglish; + private final String maxValueEnglish; + + public TimePeriodValidator(final long minValue, final TimeUnit minTimeUnit, final long maxValue, final TimeUnit maxTimeUnit) { + pattern = Pattern.compile(FormatUtils.TIME_DURATION_REGEX); + + this.minNanos = TimeUnit.NANOSECONDS.convert(minValue, minTimeUnit); + this.maxNanos = TimeUnit.NANOSECONDS.convert(maxValue, maxTimeUnit); + this.minValueEnglish = minValue + " " + minTimeUnit.toString(); + this.maxValueEnglish = maxValue + " " + maxTimeUnit.toString(); + } + + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + if (input == null) { + return new ValidationResult.Builder().subject(subject).input(input).valid(false).explanation("Time Period cannot be null").build(); + } + final String lowerCase = input.toLowerCase(); + final boolean validSyntax = pattern.matcher(lowerCase).matches(); + final ValidationResult.Builder builder = new ValidationResult.Builder(); + if (validSyntax) { + final long nanos = FormatUtils.getTimeDuration(lowerCase, TimeUnit.NANOSECONDS); + + if (nanos < minNanos || nanos > maxNanos) { + builder.subject(subject).input(input).valid(false) + .explanation("Must be in the range of " + minValueEnglish + " to " + maxValueEnglish); + } else { + builder.subject(subject).input(input).valid(true); + } + } else { + builder.subject(subject).input(input).valid(false) + .explanation("Must be of format <duration> <TimeUnit> where <duration> is a non-negative " + + "integer and TimeUnit is a supported Time Unit, such as: nanos, millis, secs, mins, hrs, days"); + } + return builder.build(); + } + } + + public static class FileExistsValidator implements Validator { + + private final boolean allowEL; + + public FileExistsValidator(final boolean allowExpressionLanguage) { + this.allowEL = allowExpressionLanguage; + } + + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + final String substituted; + if (allowEL) { + try { + substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(value).valid(false) + .explanation("Not a valid Expression Language value: " + e.getMessage()).build(); + } + } else { + substituted = value; + } + + final File file = new File(substituted); + final boolean valid = file.exists(); + final String explanation = valid ? null : "File " + file + " does not exist"; + return new ValidationResult.Builder().subject(subject).input(value).valid(valid).explanation(explanation).build(); + } + } + + public static class DirectoryExistsValidator implements Validator { + + private final boolean allowEL; + private final boolean create; + + public DirectoryExistsValidator(final boolean allowExpressionLanguage, final boolean create) { + this.allowEL = allowExpressionLanguage; + this.create = create; + } + + @Override + public ValidationResult validate(final String subject, final String value, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { + return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); + } + + final String substituted; + if (allowEL) { + try { + substituted = context.newPropertyValue(value).evaluateAttributeExpressions().getValue(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(value).valid(false) + .explanation("Not a valid Expression Language value: " + e.getMessage()).build(); + } + + if (substituted.trim().isEmpty() && !value.trim().isEmpty()) { + // User specified an Expression and nothing more... assume valid. + return new ValidationResult.Builder().subject(subject).input(value).valid(true).build(); + } + } else { + substituted = value; + } + + String reason = null; + try { + final File file = new File(substituted); + if (!file.exists()) { + if (!create) { + reason = "Directory does not exist"; + } else if (!file.mkdirs()) { + reason = "Directory does not exist and could not be created"; + } + } else if (!file.isDirectory()) { + reason = "Path does not point to a directory"; + } + } catch (final Exception e) { + reason = "Value is not a valid directory name"; + } + + return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); + } + } + + /** + * Creates a validator based on existence of a {@link ControllerService}. + * + * @param serviceClass the controller service API your + * {@link ConfigurableComponent} depends on + * @return a Validator + * @deprecated As of release 0.1.0-incubating, replaced by + * {@link org.apache.nifi.components.PropertyDescriptor.Builder#identifiesControllerService(Class)} + */ + @Deprecated + public static Validator createControllerServiceExistsValidator(final Class<? extends ControllerService> serviceClass) { + return new Validator() { + @Override + public ValidationResult validate(final String subject, final String input, final ValidationContext context) { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + final ControllerService svc = context.getControllerServiceLookup().getControllerService(input); + + if (svc == null) { + return new ValidationResult.Builder().valid(false).input(input).subject(subject).explanation("No Controller Service exists with this ID").build(); + } + + if (!serviceClass.isAssignableFrom(svc.getClass())) { + return new ValidationResult.Builder() + .valid(false) + .input(input) + .subject(subject) + .explanation("Controller Service with this ID is of type " + svc.getClass().getName() + " but is expected to be of type " + serviceClass.getName()) + .build(); + } + + final ValidationContext serviceValidationContext = context.getControllerServiceValidationContext(svc); + final Collection<ValidationResult> serviceValidationResults = svc.validate(serviceValidationContext); + for (final ValidationResult result : serviceValidationResults) { + if (!result.isValid()) { + return new ValidationResult.Builder() + .valid(false) + .input(input) + .subject(subject) + .explanation("Controller Service " + input + " is not valid: " + result.getExplanation()) + .build(); + } + } + + return new ValidationResult.Builder().input(input).subject(subject).valid(true).build(); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java new file mode 100644 index 0000000..359def2 --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/TestFormatUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processor; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.util.FormatUtils; + +import org.junit.Test; + +public class TestFormatUtils { + + @Test + public void testParse() { + assertEquals(3, FormatUtils.getTimeDuration("3000 ms", TimeUnit.SECONDS)); + assertEquals(3000, FormatUtils.getTimeDuration("3000 s", TimeUnit.SECONDS)); + assertEquals(0, FormatUtils.getTimeDuration("999 millis", TimeUnit.SECONDS)); + assertEquals(4L * 24L * 60L * 60L * 1000000000L, FormatUtils.getTimeDuration("4 days", TimeUnit.NANOSECONDS)); + assertEquals(24, FormatUtils.getTimeDuration("1 DAY", TimeUnit.HOURS)); + assertEquals(60, FormatUtils.getTimeDuration("1 hr", TimeUnit.MINUTES)); + assertEquals(60, FormatUtils.getTimeDuration("1 Hrs", TimeUnit.MINUTES)); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java new file mode 100644 index 0000000..bcd402d --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processor.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestStandardValidators { + + @Test + public void testTimePeriodValidator() { + Validator val = StandardValidators.createTimePeriodValidator(1L, TimeUnit.SECONDS, Long.MAX_VALUE, TimeUnit.NANOSECONDS); + ValidationResult vr; + + final ValidationContext validationContext = Mockito.mock(ValidationContext.class); + + vr = val.validate("TimePeriodTest", "0 sense made", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("TimePeriodTest", null, validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("TimePeriodTest", "0 secs", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("TimePeriodTest", "999 millis", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("TimePeriodTest", "999999999 nanos", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("TimePeriodTest", "1 sec", validationContext); + assertTrue(vr.isValid()); + } + + @Test + public void testDataSizeBoundsValidator() { + Validator val = StandardValidators.createDataSizeBoundsValidator(100, 1000); + ValidationResult vr; + + final ValidationContext validationContext = Mockito.mock(ValidationContext.class); + vr = val.validate("DataSizeBounds", "5 GB", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("DataSizeBounds", "0 B", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("DataSizeBounds", "99 B", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("DataSizeBounds", "100 B", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("DataSizeBounds", "999 B", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("DataSizeBounds", "1000 B", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("DataSizeBounds", "1001 B", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("DataSizeBounds", "water", validationContext); + assertFalse(vr.isValid()); + + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/.gitignore ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-properties/.gitignore b/nifi-commons/nifi-properties/.gitignore new file mode 100755 index 0000000..073c9fa --- /dev/null +++ b/nifi-commons/nifi-properties/.gitignore @@ -0,0 +1,3 @@ +/target +/target +/target http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-commons/nifi-properties/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-properties/pom.xml b/nifi-commons/nifi-properties/pom.xml new file mode 100644 index 0000000..32dbf40 --- /dev/null +++ b/nifi-commons/nifi-properties/pom.xml @@ -0,0 +1,24 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-commons</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>nifi-properties</artifactId> +</project>
