[incubator-gobblin] branch master updated: [GOBBLIN-1188] fix log message for SFDC iterators[]

[hutran]

This is an automated email from the ASF dual-hosted git repository.

hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 4045e68  [GOBBLIN-1188] fix log message for SFDC iterators[]
4045e68 is described below

commit 4045e6853943d1d57ea4813df7a744e7a38d1ae9
Author: Alex Li <al...@linkedin.com>
AuthorDate: Wed Jun 10 11:38:53 2020 -0700

    [GOBBLIN-1188] fix log message for SFDC iterators[]
    
    Closes #3036 from arekusuri/GOBBLIN-1188
---
 .../apache/gobblin/salesforce/QueryResultIterator.java | 11 ++++-------
 .../gobblin/salesforce/ResultChainingIterator.java     | 18 +++++++++---------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git 
a/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/QueryResultIterator.java
 
b/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/QueryResultIterator.java
index 56c76ea..8ddd934 100644
--- 
a/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/QueryResultIterator.java
+++ 
b/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/QueryResultIterator.java
@@ -65,10 +65,6 @@ public class QueryResultIterator implements 
Iterator<JsonElement> {
     if (queryResultIter == null) {
       initQueryResultIter();
     }
-    if (!queryResultIter.hasNext()) {
-      // no more data, print out total
-      log.info("Rest API query records total:{}", recordCount);
-    }
     return queryResultIter.hasNext();
   }
 
@@ -85,11 +81,12 @@ public class QueryResultIterator implements 
Iterator<JsonElement> {
     if (queryResultIter == null) {
       initQueryResultIter();
     }
+    JsonElement jsonElement = queryResultIter.next();
     recordCount ++;
     if (!queryResultIter.hasNext()) {
-      // no more data, print out total
-      log.info("Rest API query records total:{}", recordCount);
+      // variable `jsonElement` has last record. no more data, print out total
+      log.info("----Rest API query records total:{}----", recordCount);
     }
-    return queryResultIter.next();
+    return jsonElement;
   }
 }
diff --git 
a/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/ResultChainingIterator.java
 
b/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/ResultChainingIterator.java
index bab77e5..cb77a2a 100644
--- 
a/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/ResultChainingIterator.java
+++ 
b/gobblin-salesforce/src/main/java/org/apache/gobblin/salesforce/ResultChainingIterator.java
@@ -19,6 +19,7 @@ package org.apache.gobblin.salesforce;
 
 import com.google.common.collect.Iterators;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
 import com.sforce.async.BulkConnection;
 import java.util.Iterator;
 import java.util.List;
@@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class ResultChainingIterator implements Iterator<JsonElement> {
   private Iterator<JsonElement> iter;
   private int recordCount = 0;
+  private int isDeletedRecordCount = 0;
 
   public ResultChainingIterator(BulkConnection conn, List<FileIdVO> 
fileIdList, int retryLimit,
       long retryInterval, long retryExceedQuotaInterval) {
@@ -54,11 +56,6 @@ public class ResultChainingIterator implements 
Iterator<JsonElement> {
 
   @Override
   public boolean hasNext() {
-    if (!iter.hasNext()) {
-      // hasNext is false, means all data in this iterator was fetched
-      // we can print out record total.
-      log.info("====Total records: [{}] ====", recordCount);
-    }
     return iter.hasNext();
   }
 
@@ -66,11 +63,14 @@ public class ResultChainingIterator implements 
Iterator<JsonElement> {
   public JsonElement next() {
     JsonElement jsonElement = iter.next();
     recordCount ++;
+    JsonElement isDeletedElement = 
jsonElement.getAsJsonObject().get("IsDeleted");
+    if (isDeletedElement != null && isDeletedElement.getAsBoolean()) {
+      isDeletedRecordCount ++;
+    }
     if (!iter.hasNext()) {
-      // see hasNext.
-      //In case caller may not check hasNext and use next() == null as end of 
the interator
-      // we can print out total here.
-      log.info("====Total records: [{}] ====", recordCount);
+      // `jsonElement` has last record, print out total and isDeleted=true 
records(soft deleted) total
+      log.info("====Total records: [{}] isDeleted=true records: [{}]====", 
recordCount, isDeletedRecordCount);
+      return null;
     }
     return jsonElement;
   }