[hop] branch master updated: allow limit size 0 to fetch all rows. fixes #3205

[hansva]

This is an automated email from the ASF dual-hosted git repository.

hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/master by this push:
     new 9ac0caa20a allow limit size 0 to fetch all rows. fixes #3205
     new 1501701e2d Merge pull request #3206 from bamaer/3034
9ac0caa20a is described below

commit 9ac0caa20a0af14791e211f84c8a78ca4228c7fb
Author: Bart Maertens <bart.maert...@know.bi>
AuthorDate: Tue Sep 5 16:12:41 2023 +0200

    allow limit size 0 to fetch all rows. fixes #3205
---
 .../pages/pipeline/transforms/google-analytics.adoc     |  2 +-
 .../transforms/googleanalytics/GoogleAnalytics.java     | 17 ++++++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc
 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc
index 9d0191aa9b..2f8871ab73 100644
--- 
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc
+++ 
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc
@@ -81,4 +81,4 @@ Click **Preview** to preview data based on the defined query.
 
 === Limit Size
 
-Limit the number of rows to retrieve for the specified GA4 property to a 
number of rows.   
\ No newline at end of file
+Limit the number of rows to retrieve for the specified GA4 property to a 
number of rows. Set the limit size to 0 to fetch all available rows.
\ No newline at end of file
diff --git 
a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java
 
b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java
index d819fd4140..2093ea0f31 100644
--- 
a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java
+++ 
b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java
@@ -64,6 +64,8 @@ public class GoogleAnalytics extends 
BaseTransform<GoogleAnalyticsMeta, GoogleAn
     private int REQUEST_ROW_SIZE = 100000;
     private int rowsProcessed = 0;
 
+    private int rowLimit;
+
     public GoogleAnalytics(TransformMeta transformMeta, GoogleAnalyticsMeta 
meta, GoogleAnalyticsData data, int copyNr, PipelineMeta pipelineMeta, Pipeline 
pipeline) {
         super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
     }
@@ -73,6 +75,11 @@ public class GoogleAnalytics extends 
BaseTransform<GoogleAnalyticsMeta, GoogleAn
 
         if(super.init()){
             try {
+                if(meta.getRowLimit() == 0) {
+                    rowLimit = Integer.MAX_VALUE;
+                }else{
+                    rowLimit = meta.getRowLimit();
+                }
                 inputStream = new FileInputStream(meta.getOAuthKeyFile());
                 Credentials credentials = 
ServiceAccountCredentials.fromStream(inputStream);
 
@@ -90,8 +97,8 @@ public class GoogleAnalytics extends 
BaseTransform<GoogleAnalyticsMeta, GoogleAn
                     
metricList.add(Metric.newBuilder().setName(metric).build());
                 }
 
-                if(meta.getRowLimit() < REQUEST_ROW_SIZE){
-                    REQUEST_ROW_SIZE = meta.getRowLimit();
+                if(rowLimit < REQUEST_ROW_SIZE){
+                    REQUEST_ROW_SIZE = rowLimit;
                 }
             } catch (FileNotFoundException e) {
                 e.printStackTrace();
@@ -195,11 +202,11 @@ public class GoogleAnalytics extends 
BaseTransform<GoogleAnalyticsMeta, GoogleAn
             }
             // check if we're still below the row limit, adjust 
REQUEST_ROW_SIZE if we're getting close.
             rowsProcessed = (int)getLinesWritten();
-            if(getLinesWritten() + REQUEST_ROW_SIZE > meta.getRowLimit()){
-                REQUEST_ROW_SIZE = meta.getRowLimit() - (int)getLinesWritten();
+            if(getLinesWritten() + REQUEST_ROW_SIZE > rowLimit){
+                REQUEST_ROW_SIZE = rowLimit - (int)getLinesWritten();
             }
             requestOffset = (int)getLinesWritten();
-            if(rowsProcessed < meta.getRowLimit()){
+            if(rowsProcessed < rowLimit){
                 readResponse();
             }
         }