tpalfy commented on code in PR #6794:
URL: https://github.com/apache/nifi/pull/6794#discussion_r1115841881
##########
nifi-nar-bundles/nifi-salesforce-bundle/nifi-salesforce-processors/src/main/java/org/apache/nifi/processors/salesforce/QuerySalesforceObject.java:
##########
@@ -387,11 +461,78 @@ public void onTrigger(final ProcessContext context, final
ProcessSession session
} while (nextRecordsUrl.get() != null);
}
- private InputStream getResultInputStream(AtomicReference<String>
nextRecordsUrl, String querySObject) {
- if (nextRecordsUrl.get() == null) {
+ private void processCustomQuery(ProcessContext context, ProcessSession
session, FlowFile originalFlowFile) {
+ String customQuery =
context.getProperty(CUSTOM_SOQL_QUERY).evaluateAttributeExpressions(originalFlowFile).getValue();
+ AtomicReference<String> nextRecordsUrl = new AtomicReference<>();
+ AtomicReference<String> totalSize = new AtomicReference<>();
+ boolean isOriginalTransferred = false;
+ List<FlowFile> flowFiles = new ArrayList<>();
+ do {
+ FlowFile outgoingFlowFile;
+ if (originalFlowFile != null) {
+ outgoingFlowFile = session.create(originalFlowFile);
+ } else {
+ outgoingFlowFile = session.create();
+ }
+ Map<String, String> attributes = new HashMap<>();
+ try (InputStream response =
getResultInputStream(nextRecordsUrl.get(), customQuery)) {
+ outgoingFlowFile = session.write(outgoingFlowFile,
parseHttpResponse(response, nextRecordsUrl, totalSize));
+ int recordCount = nextRecordsUrl.get() != null ? 2000 :
Integer.parseInt(totalSize.get()) % 2000;
+ attributes.put(CoreAttributes.MIME_TYPE.key(),
"application/json");
+ attributes.put(TOTAL_RECORD_COUNT,
String.valueOf(recordCount));
+ session.adjustCounter("Salesforce records processed",
recordCount, false);
+ session.putAllAttributes(outgoingFlowFile, attributes);
+ flowFiles.add(outgoingFlowFile);
+ } catch (IOException e) {
+ throw new ProcessException("Couldn't get Salesforce records",
e);
+ } catch (Exception e) {
+ if (originalFlowFile != null) {
+ session.transfer(originalFlowFile, REL_FAILURE);
+ isOriginalTransferred = true;
+ }
+ getLogger().error("Couldn't get Salesforce records", e);
+ session.remove(flowFiles);
+ flowFiles.clear();
+ break;
+ }
+ } while (nextRecordsUrl.get() != null);
+
+ if (!flowFiles.isEmpty()) {
+ session.transfer(flowFiles, REL_SUCCESS);
+ }
+ if (originalFlowFile != null && !isOriginalTransferred) {
+ session.transfer(originalFlowFile, REL_ORIGINAL);
+ }
Review Comment:
An exception can occur between the creation of the `outgoingFlowFile` and
it's addition to the `flowFiles` collection. (For example when OAuth fails.)
When this happens we don't remove it from the session.
Here's a suggested fix (contains some other minor changes):
```suggestion
String customQuery =
context.getProperty(CUSTOM_SOQL_QUERY).evaluateAttributeExpressions(originalFlowFile).getValue();
AtomicReference<String> nextRecordsUrl = new AtomicReference<>();
AtomicReference<String> totalSize = new AtomicReference<>();
boolean isOriginalTransferred = false;
List<FlowFile> outgoingFlowFiles = new ArrayList<>();
do {
FlowFile outgoingFlowFile;
try (InputStream response =
getResultInputStream(nextRecordsUrl.get(), customQuery)) {
if (originalFlowFile != null) {
outgoingFlowFile = session.create(originalFlowFile);
} else {
outgoingFlowFile = session.create();
}
outgoingFlowFiles.add(outgoingFlowFile);
outgoingFlowFile = session.write(outgoingFlowFile,
parseHttpResponse(response, nextRecordsUrl, totalSize));
int recordCount = nextRecordsUrl.get() != null ? 2000 :
Integer.parseInt(totalSize.get()) % 2000;
Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(),
"application/json");
attributes.put(TOTAL_RECORD_COUNT,
String.valueOf(recordCount));
session.adjustCounter("Salesforce records processed",
recordCount, false);
session.putAllAttributes(outgoingFlowFile, attributes);
} catch (IOException e) {
throw new ProcessException("Couldn't get Salesforce
records", e);
} catch (Exception e) {
if (originalFlowFile != null) {
session.transfer(originalFlowFile, REL_FAILURE);
isOriginalTransferred = true;
}
getLogger().error("Couldn't get Salesforce records", e);
session.remove(outgoingFlowFiles);
outgoingFlowFiles.clear();
break;
}
} while (nextRecordsUrl.get() != null);
if (!outgoingFlowFiles.isEmpty()) {
session.transfer(outgoingFlowFiles, REL_SUCCESS);
}
if (originalFlowFile != null && !isOriginalTransferred) {
session.transfer(originalFlowFile, REL_ORIGINAL);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
