Repository: nifi Updated Branches: refs/heads/master 11b935a27 -> 60d88b5a6
http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SchemaRegistryService.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SchemaRegistryService.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SchemaRegistryService.java index 0988935..18e3830 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SchemaRegistryService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SchemaRegistryService.java @@ -17,17 +17,7 @@ package org.apache.nifi.serialization; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; - import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.avro.AvroSchemaValidator; import org.apache.nifi.components.AllowableValue; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; @@ -35,65 +25,32 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.schema.access.AvroSchemaTextStrategy; -import org.apache.nifi.schema.access.HortonworksAttributeSchemaReferenceStrategy; -import org.apache.nifi.schema.access.HortonworksEncodedSchemaReferenceStrategy; import org.apache.nifi.schema.access.SchemaAccessStrategy; +import org.apache.nifi.schema.access.SchemaAccessUtils; import org.apache.nifi.schema.access.SchemaField; -import org.apache.nifi.schema.access.SchemaNamePropertyStrategy; import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.schemaregistry.services.SchemaRegistry; import org.apache.nifi.serialization.record.RecordSchema; -public abstract class SchemaRegistryService extends AbstractControllerService { +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; - static final AllowableValue SCHEMA_NAME_PROPERTY = new AllowableValue("schema-name", "Use 'Schema Name' Property", - "The name of the Schema to use is specified by the 'Schema Name' Property. The value of this property is used to lookup the Schema in the configured Schema Registry service."); - static final AllowableValue SCHEMA_TEXT_PROPERTY = new AllowableValue("schema-text-property", "Use 'Schema Text' Property", - "The text of the Schema itself is specified by the 'Schema Text' Property. The value of this property must be a valid Avro Schema. " - + "If Expression Language is used, the value of the 'Schema Text' property must be valid after substituting the expressions."); - static final AllowableValue HWX_CONTENT_ENCODED_SCHEMA = new AllowableValue("hwx-content-encoded-schema", "HWX Content-Encoded Schema Reference", - "The content of the FlowFile contains a reference to a schema in the Schema Registry service. The reference is encoded as a single byte indicating the 'protocol version', " - + "followed by 8 bytes indicating the schema identifier, and finally 4 bytes indicating the schema version, as per the Hortonworks Schema Registry serializers and deserializers, " - + "found at https://github.com/hortonworks/registry";); - static final AllowableValue HWX_SCHEMA_REF_ATTRIBUTES = new AllowableValue("hwx-schema-ref-attributes", "HWX Schema Reference Attributes", - "The FlowFile contains 3 Attributes that will be used to lookup a Schema from the configured Schema Registry: 'schema.identifier', 'schema.version', and 'schema.protocol.version'"); - - protected static final PropertyDescriptor SCHEMA_REGISTRY = new PropertyDescriptor.Builder() - .name("Schema Registry") - .description("Specifies the Controller Service to use for the Schema Registry") - .identifiesControllerService(SchemaRegistry.class) - .required(false) - .build(); - - protected static final PropertyDescriptor SCHEMA_ACCESS_STRATEGY = new PropertyDescriptor.Builder() - .name("Schema Access Strategy") - .description("Specifies how to obtain the schema that is to be used for interpreting the data.") - .allowableValues(SCHEMA_NAME_PROPERTY, SCHEMA_TEXT_PROPERTY, HWX_SCHEMA_REF_ATTRIBUTES, HWX_CONTENT_ENCODED_SCHEMA) - .defaultValue(SCHEMA_NAME_PROPERTY.getValue()) - .required(true) - .build(); - - static final PropertyDescriptor SCHEMA_NAME = new PropertyDescriptor.Builder() - .name("Schema Name") - .description("Specifies the name of the schema to lookup in the Schema Registry property") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(true) - .defaultValue("${schema.name}") - .required(false) - .build(); - - static final PropertyDescriptor SCHEMA_TEXT = new PropertyDescriptor.Builder() - .name("schema-text") - .displayName("Schema Text") - .description("The text of an Avro-formatted Schema") - .addValidator(new AvroSchemaValidator()) - .expressionLanguageSupported(true) - .defaultValue("${avro.schema}") - .required(false) - .build(); +import static org.apache.nifi.schema.access.SchemaAccessUtils.HWX_CONTENT_ENCODED_SCHEMA; +import static org.apache.nifi.schema.access.SchemaAccessUtils.HWX_SCHEMA_REF_ATTRIBUTES; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_NAME; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_NAME_PROPERTY; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_REGISTRY; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_TEXT; +import static org.apache.nifi.schema.access.SchemaAccessUtils.SCHEMA_TEXT_PROPERTY; +public abstract class SchemaRegistryService extends AbstractControllerService { private volatile ConfigurationContext configurationContext; private volatile SchemaAccessStrategy schemaAccessStrategy; @@ -134,7 +91,7 @@ public abstract class SchemaRegistryService extends AbstractControllerService { final PropertyDescriptor descriptor = getSchemaAcessStrategyDescriptor(); final String schemaAccess = context.getProperty(descriptor).getValue(); - this.schemaAccessStrategy = getSchemaAccessStrategy(schemaAccess, schemaRegistry); + this.schemaAccessStrategy = getSchemaAccessStrategy(schemaAccess, schemaRegistry, context); } protected ConfigurationContext getConfigurationContext() { @@ -146,41 +103,13 @@ public abstract class SchemaRegistryService extends AbstractControllerService { } public RecordSchema getSchema(final FlowFile flowFile, final InputStream contentStream) throws SchemaNotFoundException, IOException { - return getSchemaAccessStrategy().getSchema(flowFile, contentStream, configurationContext); - } - - private String getSchemaAccessStrategyName(final String schemaAccessValue) { - for (final AllowableValue allowableValue : getSchemaAcessStrategyDescriptor().getAllowableValues()) { - if (allowableValue.getValue().equalsIgnoreCase(schemaAccessValue)) { - return allowableValue.getDisplayName(); - } - } - - return null; - } - - private boolean isSchemaRegistryRequired(final String schemaAccessValue) { - return HWX_CONTENT_ENCODED_SCHEMA.getValue().equalsIgnoreCase(schemaAccessValue) || SCHEMA_NAME_PROPERTY.getValue().equalsIgnoreCase(schemaAccessValue) - || HWX_SCHEMA_REF_ATTRIBUTES.getValue().equalsIgnoreCase(schemaAccessValue); + return getSchemaAccessStrategy().getSchema(flowFile, contentStream); } @Override protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) { final String schemaAccessStrategy = validationContext.getProperty(getSchemaAcessStrategyDescriptor()).getValue(); - if (isSchemaRegistryRequired(schemaAccessStrategy)) { - final boolean registrySet = validationContext.getProperty(SCHEMA_REGISTRY).isSet(); - if (!registrySet) { - final String schemaAccessStrategyName = getSchemaAccessStrategyName(schemaAccessStrategy); - - return Collections.singleton(new ValidationResult.Builder() - .subject("Schema Registry") - .explanation("The '" + schemaAccessStrategyName + "' Schema Access Strategy requires that the Schema Registry property be set.") - .valid(false) - .build()); - } - } - - return Collections.emptyList(); + return SchemaAccessUtils.validateSchemaAccessStrategy(validationContext, schemaAccessStrategy, getSchemaAccessStrategyValues()); } protected List<AllowableValue> getSchemaAccessStrategyValues() { @@ -196,32 +125,12 @@ public abstract class SchemaRegistryService extends AbstractControllerService { return suppliedFields; } - protected SchemaAccessStrategy getSchemaAccessStrategy(final String allowableValue, final SchemaRegistry schemaRegistry) { - if (allowableValue.equalsIgnoreCase(SCHEMA_NAME_PROPERTY.getValue())) { - return new SchemaNamePropertyStrategy(schemaRegistry, getConfigurationContext().getProperty(SCHEMA_NAME)); - } else if (allowableValue.equalsIgnoreCase(SCHEMA_TEXT_PROPERTY.getValue())) { - return new AvroSchemaTextStrategy(getConfigurationContext().getProperty(SCHEMA_TEXT)); - } else if (allowableValue.equalsIgnoreCase(HWX_CONTENT_ENCODED_SCHEMA.getValue())) { - return new HortonworksEncodedSchemaReferenceStrategy(schemaRegistry); - } else if (allowableValue.equalsIgnoreCase(HWX_SCHEMA_REF_ATTRIBUTES.getValue())) { - return new HortonworksAttributeSchemaReferenceStrategy(schemaRegistry); - } - - return null; + protected SchemaAccessStrategy getSchemaAccessStrategy(final String allowableValue, final SchemaRegistry schemaRegistry, final ConfigurationContext context) { + return SchemaAccessUtils.getSchemaAccessStrategy(allowableValue, schemaRegistry, context); } protected SchemaAccessStrategy getSchemaAccessStrategy(final String allowableValue, final SchemaRegistry schemaRegistry, final ValidationContext context) { - if (allowableValue.equalsIgnoreCase(SCHEMA_NAME_PROPERTY.getValue())) { - return new SchemaNamePropertyStrategy(schemaRegistry, context.getProperty(SCHEMA_NAME)); - } else if (allowableValue.equalsIgnoreCase(SCHEMA_TEXT_PROPERTY.getValue())) { - return new AvroSchemaTextStrategy(context.getProperty(SCHEMA_TEXT)); - } else if (allowableValue.equalsIgnoreCase(HWX_CONTENT_ENCODED_SCHEMA.getValue())) { - return new HortonworksEncodedSchemaReferenceStrategy(schemaRegistry); - } else if (allowableValue.equalsIgnoreCase(HWX_SCHEMA_REF_ATTRIBUTES.getValue())) { - return new HortonworksAttributeSchemaReferenceStrategy(schemaRegistry); - } - - return null; + return SchemaAccessUtils.getSchemaAccessStrategy(allowableValue, schemaRegistry, context); } } http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SimpleDateFormatValidator.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SimpleDateFormatValidator.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SimpleDateFormatValidator.java deleted file mode 100644 index f25749b..0000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/serialization/SimpleDateFormatValidator.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.serialization; - -import java.text.SimpleDateFormat; - -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; - -public class SimpleDateFormatValidator implements Validator { - - @Override - public ValidationResult validate(final String subject, final String input, final ValidationContext context) { - try { - new SimpleDateFormat(input); - } catch (final Exception e) { - return new ValidationResult.Builder() - .input(input) - .subject(subject) - .valid(false) - .explanation("Invalid Date format: " + e.getMessage()) - .build(); - } - - return new ValidationResult.Builder() - .input(input) - .subject(subject) - .valid(true) - .build(); - } - -} http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextWriter.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextWriter.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextWriter.java index 7fdc7a4..39416c8 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextWriter.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/text/FreeFormTextWriter.java @@ -50,11 +50,11 @@ public class FreeFormTextWriter implements RecordSetWriter { int count = 0; try { - final RecordSchema schema = recordSet.getSchema(); - final List<String> colNames = getColumnNames(schema); - Record record; while ((record = recordSet.next()) != null) { + final RecordSchema schema = record.getSchema(); + final List<String> colNames = getColumnNames(schema); + count++; write(record, out, colNames); } http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVHeaderSchemaStrategy.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVHeaderSchemaStrategy.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVHeaderSchemaStrategy.java index 3eed784..20548db 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVHeaderSchemaStrategy.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVHeaderSchemaStrategy.java @@ -40,7 +40,6 @@ public class TestCSVHeaderSchemaStrategy { @Test public void testSimple() throws SchemaNotFoundException, IOException { - final CSVHeaderSchemaStrategy strategy = new CSVHeaderSchemaStrategy(); final String headerLine = "a, b, c, d, e\\,z, f"; final byte[] headerBytes = headerLine.getBytes(); @@ -53,10 +52,11 @@ public class TestCSVHeaderSchemaStrategy { properties.put(CSVUtils.ESCAPE_CHAR, "\\"); final ConfigurationContext context = new MockConfigurationContext(properties, null); + final CSVHeaderSchemaStrategy strategy = new CSVHeaderSchemaStrategy(context); final RecordSchema schema; try (final InputStream bais = new ByteArrayInputStream(headerBytes)) { - schema = strategy.getSchema(null, bais, context); + schema = strategy.getSchema(null, bais); } final List<String> expectedFieldNames = Arrays.asList("a", "b", "c", "d", "e,z", "f"); http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-schema-registry-service-api/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-schema-registry-service-api/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-schema-registry-service-api/pom.xml index 265eb71..4f6cac3 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-schema-registry-service-api/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-schema-registry-service-api/pom.xml @@ -28,5 +28,9 @@ <groupId>org.apache.nifi</groupId> <artifactId>nifi-record-serialization-service-api</artifactId> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml index 5cee52e..9cf71d6 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml @@ -66,5 +66,10 @@ <artifactId>nifi-record-serialization-service-api</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + <scope>compile</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/nifi-nar-bundles/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index 1a4a49a..d7bcf30 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -81,6 +81,8 @@ <module>nifi-poi-bundle</module> <module>nifi-cdc</module> <module>nifi-cybersecurity-bundle</module> + <module>nifi-parquet-bundle</module> + <module>nifi-extension-utils</module> </modules> <build> @@ -103,7 +105,7 @@ <doUpdate>false</doUpdate> <shortRevisionLength>7</shortRevisionLength> <getRevisionOnlyOnce>true</getRevisionOnlyOnce> - <revisionOnScmFailure /> + <revisionOnScmFailure/> <buildNumberPropertyName>buildRevision</buildNumberPropertyName> <scmBranchPropertyName>buildBranch</scmBranchPropertyName> </configuration> @@ -126,6 +128,24 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-service-api</artifactId> + <version>1.2.0-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + <version>1.2.0-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-schema-registry-service-api</artifactId> + <version>1.2.0-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-ssl-context-service-api</artifactId> <version>1.2.0-SNAPSHOT</version> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/nifi/blob/60d88b5a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7c4f041..06d155a 100644 --- a/pom.xml +++ b/pom.xml @@ -1398,6 +1398,12 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-parquet-nar</artifactId> + <version>1.2.0-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-properties</artifactId> <version>1.2.0-SNAPSHOT</version> </dependency> @@ -1438,6 +1444,31 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-record-utils</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-record-utils</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hadoop-record-utils</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock-record-utils</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-mock</artifactId> <version>1.2.0-SNAPSHOT</version> <scope>test</scope>
