exceptionfactory commented on code in PR #6265: URL: https://github.com/apache/nifi/pull/6265#discussion_r946753075
########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/src/main/java/org/apache/nifi/processors/PutIoTDB.java: ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors; + +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.nifi.processors.model.IoTDBSchema; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.model.ValidationResult; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +@Tags({"iotdb", "insert", "tablet"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription( + "This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to Apache IoTDB using native interface.") +public class PutIoTDB extends AbstractIoTDB { + + static final PropertyDescriptor RECORD_READER_FACTORY = + new PropertyDescriptor.Builder() + .name("Record Reader") + .description( + "Specifies the type of Record Reader controller service to use for parsing the incoming data " + + "and determining the schema") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor SCHEMA = + new PropertyDescriptor.Builder() + .name("Schema") + .description( + "The schema that IoTDB needs doesn't support good by NiFi.\n" + + "Therefore, you can define the schema here.\n" + + "Besides, you can set encoding type and compression type by this method.\n" + + "If you don't set this property, the inferred schema will be used.\n") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor ALIGNED = + new PropertyDescriptor.Builder() + .name("Aligned") + .description("Whether to use the Apache IoTDB Aligned Timeseries interface") + .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .defaultValue("false") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor MAX_ROW_NUMBER = + new PropertyDescriptor.Builder() + .name("Max Row Number") + .description( + "Specifies the max row number of each Apache IoTDB Tablet") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static { + descriptors.add(RECORD_READER_FACTORY); + descriptors.add(SCHEMA); + descriptors.add(ALIGNED); + descriptors.add(MAX_ROW_NUMBER); + } + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return descriptors; + } + + @Override + public Set<Relationship> getRelationships() { + return relationships; + } + + @Override + public void onTrigger(ProcessContext processContext, ProcessSession processSession) + throws ProcessException { + final RecordReaderFactory recordParserFactory = + processContext + .getProperty(RECORD_READER_FACTORY) + .asControllerService(RecordReaderFactory.class); + + FlowFile flowFile = processSession.get(); + + if (flowFile == null) { + processSession.transfer(flowFile, REL_SUCCESS); + return; + } + + String schemaProperty = processContext.getProperty(SCHEMA).evaluateAttributeExpressions(flowFile).getValue(); + String alignedProperty = processContext.getProperty(ALIGNED).evaluateAttributeExpressions(flowFile).getValue(); + String maxRowNumberProperty = processContext.getProperty(MAX_ROW_NUMBER).evaluateAttributeExpressions(flowFile).getValue(); + + final boolean aligned = alignedProperty != null ? Boolean.valueOf(alignedProperty) : false; + int maxRowNumber = maxRowNumberProperty != null ? Integer.valueOf(maxRowNumberProperty) : 1024; + + try (final InputStream inputStream = processSession.read(flowFile); + final RecordReader recordReader = + recordParserFactory.createRecordReader(flowFile, inputStream, getLogger())) { + boolean needInitFormatter; + IoTDBSchema schema; + ValidationResult result; + + result = + schemaProperty != null + ? validateSchemaAttribute(schemaProperty) + : validateSchema(recordReader.getSchema()); + + if (!result.getKey()) { + getLogger().error(String.format("The property `schema` has an error: %s", result.getValue())); + inputStream.close(); + recordReader.close(); + processSession.transfer(flowFile, REL_FAILURE); + return; + } else { + if (result.getValue() != null) { + getLogger().warn(String.format("The property `schema` has a warn: %s", result.getValue())); + } + } + + schema = + schemaProperty != null + ? mapper.readValue(schemaProperty, IoTDBSchema.class) + : convertSchema(recordReader.getSchema()); + + List<String> fieldNames = schema.getFieldNames(); + + needInitFormatter = schema.getTimeType() != IoTDBSchema.TimeType.LONG; + + HashMap<String, Tablet> tablets = generateTablets(schema, maxRowNumber); + DateTimeFormatter format = null; + + Record record; + + while ((record = recordReader.nextRecord()) != null) { + Object[] values = record.getValues(); + if (format == null && needInitFormatter) { + format = initFormatter((String) values[0]); + if (format == null) { + getLogger().error("{} Record [{}] time format not supported\", flowFile, recordNumber"); Review Comment: The trailing slash character should be removed: ```suggestion getLogger().error("{} Record [{}] time format not supported", flowFile, recordNumber"); ``` ########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/src/main/java/org/apache/nifi/processors/PutIoTDB.java: ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors; + +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.nifi.processors.model.IoTDBSchema; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.model.ValidationResult; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +@Tags({"iotdb", "insert", "tablet"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription( + "This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to Apache IoTDB using native interface.") +public class PutIoTDB extends AbstractIoTDB { + + static final PropertyDescriptor RECORD_READER_FACTORY = + new PropertyDescriptor.Builder() + .name("Record Reader") + .description( + "Specifies the type of Record Reader controller service to use for parsing the incoming data " + + "and determining the schema") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor SCHEMA = + new PropertyDescriptor.Builder() + .name("Schema") + .description( + "The schema that IoTDB needs doesn't support good by NiFi.\n" + + "Therefore, you can define the schema here.\n" + + "Besides, you can set encoding type and compression type by this method.\n" + + "If you don't set this property, the inferred schema will be used.\n") Review Comment: The wording of this description is not quite clear, and does not need to state what NiFi does not support. Recommend adjusting the wording to include more details about the expected format, along the following lines: The Apache IoTDB Schema Template defined using JSON. The Processor will infer the IoTDB Schema when this property is not configured. The [IoTDB Schema Template](https://iotdb.apache.org/UserGuide/Master/Data-Concept/Schema-Template.html) documentation provides some details, but is their an official format specification? It would be helpful to link to that documentation if available, otherwise it will be unclear how to define this property. ########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/pom.xml: ########## @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>nifi-iotdb-bundle</artifactId> + <groupId>org.apache.nifi</groupId> + <version>1.18.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-iotdb-processors</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <version>${iotdb.sdk.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-server</artifactId> + <version>${iotdb.sdk.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-server</artifactId> + <version>${iotdb.sdk.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-service-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock-record-utils</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-dbcp-service-api</artifactId> + <version>${project.version}</version> + </dependency> Review Comment: This dependency is not necessary and should be removed ########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/src/main/java/org/apache/nifi/processors/PutIoTDB.java: ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors; + +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.nifi.processors.model.IoTDBSchema; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.model.ValidationResult; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +@Tags({"iotdb", "insert", "tablet"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription( + "This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to Apache IoTDB using native interface.") +public class PutIoTDB extends AbstractIoTDB { + + static final PropertyDescriptor RECORD_READER_FACTORY = + new PropertyDescriptor.Builder() + .name("Record Reader") + .description( + "Specifies the type of Record Reader controller service to use for parsing the incoming data " + + "and determining the schema") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor SCHEMA = + new PropertyDescriptor.Builder() + .name("Schema") Review Comment: Recommend naming this property `Schema Template` to align with the IoTDB terminology. ########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/src/main/java/org/apache/nifi/processors/PutIoTDB.java: ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors; + +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.nifi.processors.model.IoTDBSchema; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.model.ValidationResult; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +@Tags({"iotdb", "insert", "tablet"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription( + "This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to Apache IoTDB using native interface.") +public class PutIoTDB extends AbstractIoTDB { + + static final PropertyDescriptor RECORD_READER_FACTORY = + new PropertyDescriptor.Builder() + .name("Record Reader") + .description( + "Specifies the type of Record Reader controller service to use for parsing the incoming data " + + "and determining the schema") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor SCHEMA = + new PropertyDescriptor.Builder() + .name("Schema") + .description( + "The schema that IoTDB needs doesn't support good by NiFi.\n" + + "Therefore, you can define the schema here.\n" + + "Besides, you can set encoding type and compression type by this method.\n" + + "If you don't set this property, the inferred schema will be used.\n") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor ALIGNED = + new PropertyDescriptor.Builder() + .name("Aligned") + .description("Whether to use the Apache IoTDB Aligned Timeseries interface") + .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .defaultValue("false") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor MAX_ROW_NUMBER = + new PropertyDescriptor.Builder() + .name("Max Row Number") + .description( + "Specifies the max row number of each Apache IoTDB Tablet") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static { + descriptors.add(RECORD_READER_FACTORY); + descriptors.add(SCHEMA); + descriptors.add(ALIGNED); + descriptors.add(MAX_ROW_NUMBER); + } + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return descriptors; + } + + @Override + public Set<Relationship> getRelationships() { + return relationships; + } + + @Override + public void onTrigger(ProcessContext processContext, ProcessSession processSession) + throws ProcessException { + final RecordReaderFactory recordParserFactory = + processContext + .getProperty(RECORD_READER_FACTORY) + .asControllerService(RecordReaderFactory.class); + + FlowFile flowFile = processSession.get(); + + if (flowFile == null) { + processSession.transfer(flowFile, REL_SUCCESS); Review Comment: This transfer should be removed because `flowFile` is null. ########## nifi-nar-bundles/nifi-iotdb-bundle/nifi-iotdb-processors/src/main/java/org/apache/nifi/processors/PutIoTDB.java: ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors; + +import java.io.InputStream; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.nifi.processors.model.IoTDBSchema; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.model.ValidationResult; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; + +@Tags({"iotdb", "insert", "tablet"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription( + "This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to Apache IoTDB using native interface.") +public class PutIoTDB extends AbstractIoTDB { + + static final PropertyDescriptor RECORD_READER_FACTORY = + new PropertyDescriptor.Builder() + .name("Record Reader") + .description( + "Specifies the type of Record Reader controller service to use for parsing the incoming data " + + "and determining the schema") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor SCHEMA = + new PropertyDescriptor.Builder() + .name("Schema") + .description( + "The schema that IoTDB needs doesn't support good by NiFi.\n" + + "Therefore, you can define the schema here.\n" + + "Besides, you can set encoding type and compression type by this method.\n" + + "If you don't set this property, the inferred schema will be used.\n") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor ALIGNED = + new PropertyDescriptor.Builder() + .name("Aligned") + .description("Whether to use the Apache IoTDB Aligned Timeseries interface") + .allowableValues("true", "false") + .addValidator(StandardValidators.BOOLEAN_VALIDATOR) + .defaultValue("false") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static final PropertyDescriptor MAX_ROW_NUMBER = + new PropertyDescriptor.Builder() + .name("Max Row Number") + .description( + "Specifies the max row number of each Apache IoTDB Tablet") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + static { + descriptors.add(RECORD_READER_FACTORY); + descriptors.add(SCHEMA); + descriptors.add(ALIGNED); + descriptors.add(MAX_ROW_NUMBER); + } + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return descriptors; + } + + @Override + public Set<Relationship> getRelationships() { + return relationships; + } + + @Override + public void onTrigger(ProcessContext processContext, ProcessSession processSession) + throws ProcessException { + final RecordReaderFactory recordParserFactory = + processContext + .getProperty(RECORD_READER_FACTORY) + .asControllerService(RecordReaderFactory.class); + + FlowFile flowFile = processSession.get(); + + if (flowFile == null) { + processSession.transfer(flowFile, REL_SUCCESS); + return; + } + + String schemaProperty = processContext.getProperty(SCHEMA).evaluateAttributeExpressions(flowFile).getValue(); + String alignedProperty = processContext.getProperty(ALIGNED).evaluateAttributeExpressions(flowFile).getValue(); + String maxRowNumberProperty = processContext.getProperty(MAX_ROW_NUMBER).evaluateAttributeExpressions(flowFile).getValue(); + + final boolean aligned = alignedProperty != null ? Boolean.valueOf(alignedProperty) : false; + int maxRowNumber = maxRowNumberProperty != null ? Integer.valueOf(maxRowNumberProperty) : 1024; + + try (final InputStream inputStream = processSession.read(flowFile); + final RecordReader recordReader = + recordParserFactory.createRecordReader(flowFile, inputStream, getLogger())) { + boolean needInitFormatter; + IoTDBSchema schema; + ValidationResult result; + + result = + schemaProperty != null + ? validateSchemaAttribute(schemaProperty) + : validateSchema(recordReader.getSchema()); + + if (!result.getKey()) { + getLogger().error(String.format("The property `schema` has an error: %s", result.getValue())); + inputStream.close(); + recordReader.close(); + processSession.transfer(flowFile, REL_FAILURE); + return; + } else { + if (result.getValue() != null) { + getLogger().warn(String.format("The property `schema` has a warn: %s", result.getValue())); + } + } + + schema = + schemaProperty != null + ? mapper.readValue(schemaProperty, IoTDBSchema.class) + : convertSchema(recordReader.getSchema()); + + List<String> fieldNames = schema.getFieldNames(); + + needInitFormatter = schema.getTimeType() != IoTDBSchema.TimeType.LONG; + + HashMap<String, Tablet> tablets = generateTablets(schema, maxRowNumber); + DateTimeFormatter format = null; + + Record record; + + while ((record = recordReader.nextRecord()) != null) { + Object[] values = record.getValues(); + if (format == null && needInitFormatter) { + format = initFormatter((String) values[0]); + if (format == null) { + getLogger().error("{} Record [{}] time format not supported\", flowFile, recordNumber"); + inputStream.close(); + recordReader.close(); + processSession.transfer(flowFile, REL_FAILURE); + return; + } + } + + long timestamp; + if (needInitFormatter) { + timestamp = Timestamp.valueOf(LocalDateTime.parse((String) values[0], format)).getTime(); Review Comment: Recommend breaking this into multiple lines for better readability. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
