This is an automated email from the ASF dual-hosted git repository.

bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git


The following commit(s) were added to refs/heads/main by this push:
     new ed39e5e1cf REST Metadata Pagination Continues After Pipeline Is 
Stopped, fixes #9273 (#9452)
ed39e5e1cf is described below

commit ed39e5e1cfb08ef357dfb3d16335911dc56099e7
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 12 20:37:48 2026 +0100

    REST Metadata Pagination Continues After Pipeline Is Stopped, fixes #9273 
(#9452)
---
 .../apache/hop/pipeline/transforms/rest/Rest.java  |  11 +-
 .../transforms/rest/RestPagingStopTest.java        | 184 +++++++++++++++++++++
 2 files changed, 194 insertions(+), 1 deletion(-)

diff --git 
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
 
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
index 2f2f827b8f..180f82de9a 100644
--- 
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
+++ 
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
@@ -940,7 +940,9 @@ public class Rest extends BaseTransform<RestMeta, RestData> 
{
     int loopIdx = 0;
     int totalEmitted = 0;
     RestExchangeResult firstExchange = null;
-    while (hasMore && loopIdx < maxLoops) {
+    // A stop only sets a flag, and putRow() quietly drops rows once it is 
set, so nothing else
+    // would end this loop before the last page or maxPagesLoops (#9273).
+    while (hasMore && loopIdx < maxLoops && !isStopped()) {
       String uriOv =
           usesLinkStylePaging(pagingType) ? (loopIdx == 0 ? null : 
state.linkNextUrl) : null;
 
@@ -994,6 +996,10 @@ public class Rest extends BaseTransform<RestMeta, 
RestData> {
       }
     }
 
+    if (isStopped()) {
+      // Zero rows after a stop is the stop, not a broken split path or an 
error response.
+      return;
+    }
     enforcePagingSplitEmittedSomething(resolvedSplitPath, firstExchange, 
totalEmitted);
   }
 
@@ -1323,6 +1329,9 @@ public class Rest extends BaseTransform<RestMeta, 
RestData> {
 
     int emitted = 0;
     for (String sliceBody : chunks) {
+      if (isStopped()) {
+        break;
+      }
       RestExchangeResult slice =
           new RestExchangeResult(
               sliceBody,
diff --git 
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingStopTest.java
 
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingStopTest.java
new file mode 100644
index 0000000000..94938f9cf7
--- /dev/null
+++ 
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingStopTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.rest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockingDetails;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.rest.RestConnection;
+import org.apache.hop.metadata.rest.RestPaginationType;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.rest.fields.ResultField;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Issue #9273: stopping a pipeline has to end REST pagination, instead of 
fetching (and logging)
+ * every remaining page up to the page limit.
+ */
+class RestPagingStopTest {
+
+  @BeforeEach
+  void setUp() {
+    if (!HopLogStore.isInitialized()) {
+      HopLogStore.init();
+    }
+  }
+
+  @Test
+  void stoppingEndsPaginationAfterTheCurrentPage() throws Exception {
+    // PAGE_NUMBER keeps going while pages have content, and this server never 
runs dry: only
+    // the stop (or maxPagesLoops) can end it.
+    PagingRest rest = build("{\"id\":1}", null);
+    rest.stopAfter = 2;
+
+    rest.runPaginationLoop(new Object[] {"row"});
+
+    assertEquals(2, requestCount(rest), "no page may be requested after the 
stop");
+    assertEquals(2, rest.emitted.size());
+  }
+
+  @Test
+  void stoppingEndsTheSplitOfTheCurrentPage() throws Exception {
+    PagingRest rest = build("[1,2,3,4,5]", "$[*]");
+    rest.stopAfter = 2;
+
+    rest.runPaginationLoop(new Object[] {"row"});
+
+    assertEquals(1, requestCount(rest));
+    assertEquals(2, rest.emitted.size(), "the rest of the page is not turned 
into rows");
+  }
+
+  @Test
+  void stoppingBeforeTheFirstPageIsNotAnError() throws Exception {
+    // With a split path, zero rows normally fails the transform. After a stop 
it must not.
+    PagingRest rest = build("[1,2,3]", "$[*]");
+    rest.stopAfter = 0;
+
+    rest.runPaginationLoop(new Object[] {"row"});
+
+    assertEquals(0, requestCount(rest));
+    assertEquals(0, rest.emitted.size());
+  }
+
+  @Test
+  void withoutAStopPaginationRunsToMaxPagesLoops() throws Exception {
+    PagingRest rest = build("{\"id\":1}", null);
+
+    rest.runPaginationLoop(new Object[] {"row"});
+
+    assertEquals(5, requestCount(rest));
+  }
+
+  private static long requestCount(PagingRest rest) {
+    return mockingDetails(rest.data().client).getInvocations().stream()
+        .filter(i -> i.getMethod().getName().equals("execute"))
+        .count();
+  }
+
+  private PagingRest build(String body, String splitPath) throws Exception {
+    TransformMeta transformMeta = new TransformMeta();
+    transformMeta.setName("TestRest");
+    PipelineMeta pipelineMeta = new PipelineMeta();
+    pipelineMeta.setName("TestRest");
+    pipelineMeta.addTransform(transformMeta);
+
+    RestMeta meta = new RestMeta();
+    meta.setDefault();
+    meta.setMethod(RestMeta.HTTP_METHOD_GET);
+    meta.setUrl("http://example.com/items";);
+    meta.setPaginationEnabled(true);
+    meta.setMaxPagesLoops(5);
+    meta.setResultSplitPath(splitPath);
+    meta.setResultField(new ResultField());
+    meta.getResultField().setFieldName("result");
+    meta.getResultField().setCode("status");
+
+    RestData data = new RestData();
+    data.mediaType = ContentType.APPLICATION_JSON;
+    data.method = RestMeta.HTTP_METHOD_GET;
+    data.realUrl = "http://example.com/items";;
+    data.resultFieldName = "result";
+    data.resultCodeFieldName = "status";
+    CloseableHttpClient client =
+        FakeHttpClient.returning(200, body, Map.of("Content-Type", 
"application/json"));
+    data.client = client;
+
+    IRowMeta rowMeta = new RowMeta();
+    rowMeta.addValueMeta(new ValueMetaString("field1"));
+    data.inputRowMeta = rowMeta;
+    data.outputRowMeta = rowMeta.clone();
+
+    PagingRest rest =
+        new PagingRest(transformMeta, meta, data, 0, pipelineMeta, new 
LocalPipelineEngine());
+    rest.setMetadataProvider(mock(IHopMetadataProvider.class));
+
+    // The connection is normally loaded from metadata in init().
+    RestConnection connection = new RestConnection();
+    connection.setPaginationType(RestPaginationType.PAGE_NUMBER);
+    Field field = Rest.class.getDeclaredField("connection");
+    field.setAccessible(true);
+    field.set(rest, connection);
+    return rest;
+  }
+
+  /** Collects the emitted rows, and pretends the pipeline was stopped after a 
number of them. */
+  static class PagingRest extends Rest {
+    final List<Object[]> emitted = new ArrayList<>();
+    int stopAfter = -1;
+
+    PagingRest(
+        TransformMeta transformMeta,
+        RestMeta meta,
+        RestData data,
+        int copyNr,
+        PipelineMeta pipelineMeta,
+        Pipeline pipeline) {
+      super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+    }
+
+    RestData data() {
+      return (RestData) getData();
+    }
+
+    @Override
+    public void putRow(IRowMeta rowMeta, Object[] row) {
+      emitted.add(row);
+    }
+
+    @Override
+    public boolean isStopped() {
+      return stopAfter >= 0 && emitted.size() >= stopAfter;
+    }
+  }
+}

Reply via email to