date:20170217

[nifi] Git Push Summary

2017-02-17 Thread alopresto

Repository: nifi
Updated Tags:  refs/tags/nifi-1.1.2-RC1 [created] 4d70c4462

nifi git commit: [NIFI-3181] add operate palette actions to context menu. This closes #1516

2017-02-17 Thread mcgilman

Repository: nifi
Updated Branches:
  refs/heads/master 6a1854c97 -> 19302263c


[NIFI-3181] add operate palette actions to context menu. This closes #1516


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/19302263
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/19302263
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/19302263

Branch: refs/heads/master
Commit: 19302263c35c8f648feb378a0eaab4af43aa1366
Parents: 6a1854c
Author: Scott Aslan 
Authored: Fri Feb 17 12:02:56 2017 -0500
Committer: Matt Gilman 
Committed: Fri Feb 17 15:40:23 2017 -0500

--
 .../WEB-INF/partials/canvas/navigation.jsp  |  4 +-
 .../nf-ng-canvas-graph-controls-controller.js   | 20 ---
 .../main/webapp/js/nf/canvas/nf-canvas-utils.js | 30 +-
 .../main/webapp/js/nf/canvas/nf-context-menu.js | 62 +++-
 4 files changed, 90 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/nifi/blob/19302263/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/navigation.jsp
--
diff --git 
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/navigation.jsp
 
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/navigation.jsp
index 1670dcf..6d97b0e 100644
--- 
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/navigation.jsp
+++ 
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/WEB-INF/partials/canvas/navigation.jsp
@@ -98,7 +98,7 @@
 
 
 
+
ng-disabled="!(appCtrl.nf.CanvasUtils.canManagePolicies())">
 
 
 
@@ -154,7 +154,7 @@
 
 
 
+
ng-disabled="!(appCtrl.nf.CanvasUtils.getComponentByType('Connection').isDisconnected(appCtrl.nf.CanvasUtils.getSelection())
 && appCtrl.nf.CanvasUtils.canModify(appCtrl.nf.CanvasUtils.getSelection()));">
 
 
 

http://git-wip-us.apache.org/repos/asf/nifi/blob/19302263/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-graph-controls-controller.js
--
diff --git 
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-graph-controls-controller.js
 
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-graph-controls-controller.js
index e7dedea..3274a48 100644
--- 
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-graph-controls-controller.js
+++ 
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-graph-controls-controller.js
@@ -348,26 +348,6 @@
 } else if (canvasUtils.hasDetails(selection)) {
 actions.showDetails(selection);
 }
-},
-
-/**
- * Determines whether the user can configure or open the policy 
management page.
- */
-canManagePolicies: function () {
-var selection = canvasUtils.getSelection();
-
-// ensure 0 or 1 components selected
-if (selection.size() <= 1) {
-// if something is selected, ensure it's not a connection
-if (!selection.empty() && 
canvasUtils.isConnection(selection)) {
-return false;
-}
-
-// ensure access to read tenants
-return common.canAccessTenants();
-}
-
-return false;
 }
 }
 

http://git-wip-us.apache.org/repos/asf/nifi/blob/19302263/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js
--
diff --git 
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-utils.js

[2/2] nifi git commit: NIFI-3354 Added support for simple AVRO/CSV/JSON transformers that utilize external Schema Added support for simple Key/Value Schema Registry as Controller Service Added support

2017-02-17 Thread ozhurakousky

NIFI-3354 Added support for simple AVRO/CSV/JSON transformers that utilize 
external Schema
Added support for simple Key/Value Schema Registry as Controller Service
Added support for registering multiple schemas as dynamic properties of Schema 
Registry  Controller Service
Added the following 8 processors
- ExtractAvroFieldsViaSchemaRegistry
- TransformAvroToCSVViaSchemaRegistry
- TransformAvroToJsonViaSchemaRegistry
- TransformCSVToAvroViaSchemaRegistry
- TransformCSVToJsonViaSchemaRegistry
- TransformJsonToAvroViaSchemaRegistry
- TransformJsonToCSVViaSchemaRegistry
- UpdateAttributeWithSchemaViaSchemaRegistry

polishing

NIFI-3354 Adding support for HDFS Schema Registry, unions and default values in 
the Avro Schema and NULL columns in the source CSV

NIFI-3354 Adding support for logicalTypes per the Avro 1.7.7 spec

NIFI-3354 polishing and restructuring CSVUtils

NIFI-3354 renamed processors to address PR comment

NIFI-3354 addressed latest PR comments
- removed HDFS-based ControllerService. It will be migrated into a separate 
bundle as a true extension.
- removed UpdateAttribute. . . processor
- added mime.type attribute to all Transform* processors

NIFI-3354 added missing L entries

This closes pr/1436


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6a1854c9
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6a1854c9
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6a1854c9

Branch: refs/heads/master
Commit: 6a1854c9758005a67d5315f31533fdb88ec55b81
Parents: ded18b9
Author: Oleg Zhurakousky 
Authored: Fri Jan 20 10:04:48 2017 -0500
Committer: Oleg Zhurakousky 
Committed: Fri Feb 17 14:32:06 2017 -0500

--
 nifi-assembly/pom.xml   |   5 +
 .../nifi-registry-nar/pom.xml   |  31 ++
 .../src/main/resources/META-INF/LICENSE | 240 +++
 .../src/main/resources/META-INF/NOTICE  |  66 
 .../nifi-registry-processors/pom.xml|  74 +
 .../processors/AbstractCSVTransformer.java  |  57 
 .../processors/AbstractContentTransformer.java  | 101 +++
 .../processors/AbstractTransformer.java |  93 ++
 .../schemaregistry/processors/AvroUtils.java|  67 +
 .../processors/BaseContentTransformer.java  |  51 
 .../processors/BaseTransformer.java | 189 
 .../schemaregistry/processors/CSVUtils.java | 299 +++
 .../processors/ExtractAvroFields.java   | 100 +++
 .../schemaregistry/processors/JsonUtils.java|  74 +
 .../processors/RegistryCommon.java  |  84 ++
 .../processors/TransformAvroToCSV.java  |  57 
 .../processors/TransformAvroToJson.java |  46 +++
 .../processors/TransformCSVToAvro.java  |  80 +
 .../processors/TransformCSVToJson.java  |  80 +
 .../processors/TransformJsonToAvro.java |  45 +++
 .../processors/TransformJsonToCSV.java  |  45 +++
 .../org.apache.nifi.processor.Processor |  21 ++
 .../processors/TransformersTest.java| 188 
 .../expected_ouput_csv/decimal_logicalType.txt  |   1 +
 .../decimal_logicalType_invalid_scale.txt   |   1 +
 ...mal_logicalType_valid_scale_with_default.txt |   1 +
 .../decimal_logicalType_with_default.txt|   1 +
 .../expected_ouput_csv/primitive_types.txt  |   1 +
 .../primitive_types_with_matching_default.txt   |   1 +
 .../union_null_last_field_with_default.txt  |   1 +
 .../union_null_middle_field_with_default.txt|   1 +
 .../expected_ouput_csv/union_with_default.txt   |   1 +
 ...l_logicalType_invalid_scale_with_default.txt |  16 +
 ...mal_logicalType_valid_scale_with_default.txt |  16 +
 ..._logicalType_valid_scale_with_no_default.txt |  15 +
 .../input_avro/primitive_types_no_defaults.txt  |  11 +
 .../primitive_types_union_with_defaults.txt |  11 +
 .../primitive_types_with_matching_default.txt   |  11 +
 .../primitive_types_with_mismatch_default.txt   |  11 +
 .../input_avro/union_and_matching_defaults.txt  |  18 ++
 .../input_avro/union_and_mismatch_defaults.txt  |  18 ++
 .../resources/input_csv/decimal_logicalType.txt |   1 +
 .../decimal_logicalType_missing_value.txt   |   1 +
 .../resources/input_csv/primitive_types.txt |   1 +
 .../primitive_types_with_matching_default.txt   |   1 +
 .../union_null_last_field_with_default.txt  |   1 +
 .../union_null_middle_field_with_default.txt|   1 +
 .../resources/input_csv/union_with_default.txt  |   1 +
 .../input_csv/union_with_missing_value.txt  |   1 +
 .../nifi-registry-service/pom.xml   |  48 +++
 .../schemaregistry/services/SchemaRegistry.java |  46 +++
 .../services/SimpleKeyValueSchemaRegistry.java  |  96 ++
 ...org.apache.nifi.controller.ControllerService |  15 +

[1/2] nifi git commit: NIFI-3354 Added support for simple AVRO/CSV/JSON transformers that utilize external Schema Added support for simple Key/Value Schema Registry as Controller Service Added support

2017-02-17 Thread ozhurakousky

Repository: nifi
Updated Branches:
  refs/heads/master ded18b94d -> 6a1854c97


http://git-wip-us.apache.org/repos/asf/nifi/blob/6a1854c9/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformAvroToJson.java
--
diff --git 
a/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformAvroToJson.java
 
b/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformAvroToJson.java
new file mode 100644
index 000..ba45563
--- /dev/null
+++ 
b/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformAvroToJson.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.schemaregistry.processors;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+
+
+@Tags({ "registry", "schema", "avro", "json", "transform" })
+@CapabilityDescription("Transforms AVRO content of the Flow File to JSON using 
the schema provided by the Schema Registry Service.")
+@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
+public final class TransformAvroToJson extends AbstractContentTransformer {
+
+/**
+ *
+ */
+@Override
+protected Map transform(InputStream in, OutputStream out, 
InvocationContextProperties contextProperties, Schema schema) {
+GenericRecord avroRecord = AvroUtils.read(in, schema);
+JsonUtils.write(avroRecord, out);
+return Collections.singletonMap(CoreAttributes.MIME_TYPE.key(), 
"application/json");
+}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/nifi/blob/6a1854c9/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformCSVToAvro.java
--
diff --git 
a/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformCSVToAvro.java
 
b/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformCSVToAvro.java
new file mode 100644
index 000..f44e440
--- /dev/null
+++ 
b/nifi-nar-bundles/nifi-registry-bundle/nifi-registry-processors/src/main/java/org/apache/nifi/schemaregistry/processors/TransformCSVToAvro.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.schemaregistry.processors;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import

[jira] [Assigned] (MINIFI-216) Update/Override Repository directory paths in minifi configs

2017-02-17 Thread Bryan Rosander (JIRA)


 [ 
https://issues.apache.org/jira/browse/MINIFI-216?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Bryan Rosander reassigned MINIFI-216:
-

Assignee: Bryan Rosander

> Update/Override Repository directory paths in minifi configs
> 
>
> Key: MINIFI-216
> URL: https://issues.apache.org/jira/browse/MINIFI-216
> Project: Apache NiFi MiNiFi
>  Issue Type: Improvement
>Reporter: Arti Wadhwani
>Assignee: Bryan Rosander
>Priority: Minor
>
> Currently as per this 
> https://github.com/apache/nifi-minifi/blob/master/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java#L203
>   currently flowfile repository directory (and others too) is hardcoded and 
> generated when NiFi is started/restarted. 
> {code}
> nifi.flowfile.repository.directory=./flowfile_repository
> nifi.content.repository.directory.default=./content_repository
> nifi.database.directory=./database_repository
> {code}
> We need the ability to change/update such properties and basically all other 
> directory paths in minifi configs to point to a different location as per 
> user's requirement.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

nifi git commit: NIFI-3255 removed dependency on session.merge from SplitText

2017-02-17 Thread ozhurakousky

Repository: nifi
Updated Branches:
  refs/heads/master ec868362f -> ded18b94d


NIFI-3255 removed dependency on session.merge from SplitText

NIFI-3255 addressed PR comments

NIFI-3255 fixed linkage for Split creation

This closes #1394


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ded18b94
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ded18b94
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ded18b94

Branch: refs/heads/master
Commit: ded18b94dbcaa91dcad2d6ff995e0a9c1282323b
Parents: ec86836
Author: Oleg Zhurakousky 
Authored: Wed Jan 4 15:37:30 2017 -0500
Committer: Oleg Zhurakousky 
Committed: Fri Feb 17 12:31:21 2017 -0500

--
 .../nifi/processors/standard/SplitText.java | 38 
 1 file changed, 32 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/nifi/blob/ded18b94/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
--
diff --git 
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
 
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
index e57841f..4e62993 100644
--- 
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
+++ 
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/SplitText.java
@@ -18,6 +18,7 @@ package org.apache.nifi.processors.standard;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -32,6 +33,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.nifi.annotation.behavior.EventDriven;
 import org.apache.nifi.annotation.behavior.InputRequirement;
 import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -56,6 +58,7 @@ import org.apache.nifi.processor.ProcessSession;
 import org.apache.nifi.processor.Relationship;
 import org.apache.nifi.processor.exception.ProcessException;
 import org.apache.nifi.processor.io.InputStreamCallback;
+import org.apache.nifi.processor.io.OutputStreamCallback;
 import org.apache.nifi.processor.util.StandardValidators;
 import org.apache.nifi.stream.io.util.TextLineDemarcator;
 import org.apache.nifi.stream.io.util.TextLineDemarcator.OffsetInfo;
@@ -282,8 +285,7 @@ public class SplitText extends AbstractProcessor {
  * it signifies the header information and its contents will be included in
  * each and every computed split.
  */
-private List generateSplitFlowFiles(final String fragmentId, 
FlowFile sourceFlowFile, SplitInfo splitInfo,
-  List 
computedSplitsInfo, ProcessSession processSession){
+private List generateSplitFlowFiles(String fragmentId, FlowFile 
sourceFlowFile, SplitInfo splitInfo, List computedSplitsInfo, 
ProcessSession processSession){
 List splitFlowFiles = new ArrayList<>();
 FlowFile headerFlowFile = null;
 long headerCrlfLength = 0;
@@ -295,19 +297,19 @@ public class SplitText extends AbstractProcessor {
 
 if ((computedSplitsInfo.size() == 0) && (headerFlowFile != null)) {
 FlowFile splitFlowFile = processSession.clone(sourceFlowFile, 0, 
headerFlowFile.getSize() - headerCrlfLength);
-splitFlowFile = SplitText.this.updateAttributes(processSession, 
splitFlowFile, 0, splitFlowFile.getSize(),
+splitFlowFile = this.updateAttributes(processSession, 
splitFlowFile, 0, splitFlowFile.getSize(),
 fragmentId, fragmentIndex++, 0, 
sourceFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
 splitFlowFiles.add(splitFlowFile);
 } else {
 for (SplitInfo computedSplitInfo : computedSplitsInfo) {
-long length = SplitText.this.removeTrailingNewLines ? 
computedSplitInfo.trimmedLength : computedSplitInfo.length;
+long length = this.removeTrailingNewLines ? 
computedSplitInfo.trimmedLength : computedSplitInfo.length;
 boolean proceedWithClone = headerFlowFile != null || length > 
0;
 if (proceedWithClone) {
 FlowFile splitFlowFile = null;
 if (headerFlowFile != null) {
 if (length > 0) {

nifi git commit: NIFI-3495 fixed the index issue with TextLineDemarcator

2017-02-17 Thread markap14

Repository: nifi
Updated Branches:
  refs/heads/master 095c04eda -> ec868362f


NIFI-3495 fixed the index issue with TextLineDemarcator

This closes #1518.


Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/ec868362
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/ec868362
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/ec868362

Branch: refs/heads/master
Commit: ec868362f3317a79b6518c780af1b9debb843f32
Parents: 095c04e
Author: Oleg Zhurakousky 
Authored: Thu Feb 16 21:05:59 2017 -0500
Committer: Mark Payne 
Committed: Fri Feb 17 10:27:40 2017 -0500

--
 .../nifi/stream/io/util/TextLineDemarcator.java | 32 +++-
 .../stream/io/util/TextLineDemarcatorTest.java  | 32 
 2 files changed, 57 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/nifi/blob/ec868362/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/TextLineDemarcator.java
--
diff --git 
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/TextLineDemarcator.java
 
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/TextLineDemarcator.java
index 7c918b4..2b5c7fe 100644
--- 
a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/TextLineDemarcator.java
+++ 
b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/util/TextLineDemarcator.java
@@ -110,7 +110,7 @@ public class TextLineDemarcator {
 for (i = this.index; i < this.bufferLength; i++) {
 byteVal = this.buffer[i];
 lineLength++;
-int crlfLength = isEol(byteVal, i);
+int crlfLength = computeEol(byteVal, i + 1);
 if (crlfLength > 0) {
 i += crlfLength;
 if (crlfLength == 2) {
@@ -120,7 +120,6 @@ public class TextLineDemarcator {
 if (startsWith != null) {
 token = this.extractDataToken(lineLength);
 }
-this.index = i;
 this.mark = this.index;
 break lineLoop;
 }
@@ -150,21 +149,40 @@ public class TextLineDemarcator {
 return offsetInfo;
 }
 
-private int isEol(byte currentByte, int currentIndex) {
+/**
+ * Determines if the line terminates. Returns int specifying the length of
+ * the CRLF (i.e., only CR or LF or CR and LF) and therefore can only have
+ * values of:
+ *   0 - not the end of the line
+ *   1 - the end of the line either via CR or LF
+ *   2 - the end of the line with both CR and LF
+ *
+ * It performs the read ahead on the buffer if need to.
+ */
+private int computeEol(byte currentByte, int providedIndex) {
+int actualIndex = providedIndex - 1;
+boolean readAhead = false;
 int crlfLength = 0;
 if (currentByte == '\n') {
 crlfLength = 1;
 } else if (currentByte == '\r') {
-if ((currentIndex + 1) >= this.bufferLength) {
-this.index = currentIndex + 1;
+if (providedIndex >= this.bufferLength) {
+this.index = this.bufferLength;
 this.fill();
+providedIndex = this.index;
+readAhead = true;
 }
 crlfLength = 1;
-if (currentIndex < this.buffer.length - 1) {
-currentByte = this.buffer[currentIndex + 1];
+if (providedIndex < this.buffer.length - 1) {
+currentByte = this.buffer[providedIndex];
 crlfLength = currentByte == '\n' ? 2 : 1;
 }
 }
+
+if (crlfLength > 0) {
+this.index = readAhead ? this.index + (crlfLength - 1) : 
(actualIndex + crlfLength);
+}
+
 return crlfLength;
 }
 

http://git-wip-us.apache.org/repos/asf/nifi/blob/ec868362/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/util/TextLineDemarcatorTest.java
--
diff --git 
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/util/TextLineDemarcatorTest.java
 
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/util/TextLineDemarcatorTest.java
index cd8b7c5..321c51b 100644
--- 
a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/util/TextLineDemarcatorTest.java
+++ 
b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/stream/io/util/TextLineDemarcatorTest.java
@@ -121,6 +121,38 @@ public class TextLineDemarcatorTest {
 }
 
 @Test
+public void

[nifi] Git Push Summary

nifi git commit: [NIFI-3181] add operate palette actions to context menu. This closes #1516

[2/2] nifi git commit: NIFI-3354 Added support for simple AVRO/CSV/JSON transformers that utilize external Schema Added support for simple Key/Value Schema Registry as Controller Service Added support

[1/2] nifi git commit: NIFI-3354 Added support for simple AVRO/CSV/JSON transformers that utilize external Schema Added support for simple Key/Value Schema Registry as Controller Service Added support

[jira] [Assigned] (MINIFI-216) Update/Override Repository directory paths in minifi configs

nifi git commit: NIFI-3255 removed dependency on session.merge from SplitText

nifi git commit: NIFI-3495 fixed the index issue with TextLineDemarcator

7 matches

Site Navigation

Mail list logo

Footer information