This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7c9571735b REST connection: first page number, page size and
empty-page detection for pagination, fixes #9548, fixes #9549 (#9573)
7c9571735b is described below
commit 7c9571735b22cf96967a211a4cf5e8d49dab77f8
Author: Bart Maertens <[email protected]>
AuthorDate: Thu Nov 19 10:43:49 2026 +0100
REST connection: first page number, page size and empty-page detection for
pagination, fixes #9548, fixes #9549 (#9573)
* REST connection: first page number, page size and empty-page detection
for pagination, fixes #9548, fixes #9549
* REST connection: own page size parameter for page-number paging, so
leftover limit fields change nothing, #9548
* REST pagination: review fixes for #9573 (first page only for PAGE_NUMBER,
empty XML items, logged no-op settings, parse each page once), #9548
---
.../resource/http/paging/paging.conf | 30 +
.../ROOT/pages/metadata-types/rest-connection.adoc | 40 +-
.../ROOT/pages/pipeline/transforms/rest.adoc | 3 +-
...0054-rest-pagination-page-number-zero-based.hpl | 329 +++++++++++
.../http/0055-rest-pagination-items-path.hpl | 261 +++++++++
.../http/0055-rest-pagination-total-count.hpl | 261 +++++++++
...0054-rest-pagination-page-number-zero-based.hwf | 92 ++++
.../http/main-0055-rest-pagination-items-path.hwf | 127 +++++
.../restconnection/paging-envelope-items.json | 13 +
.../paging-envelope-total-count.json | 14 +
.../restconnection/paging-envelope-zero-based.json | 12 +
.../apache/hop/metadata/rest/RestConnection.java | 43 ++
.../hop/metadata/rest/RestConnectionEditor.java | 117 +++-
.../rest/messages/messages_en_US.properties | 14 +-
.../rest/RestConnectionPaginationTest.java | 109 ++++
.../apache/hop/pipeline/transforms/rest/Rest.java | 430 +++++++++++++--
.../rest/messages/messages_en_US.properties | 10 +
.../transforms/rest/RestPagingEnvelopeTest.java | 607 +++++++++++++++++++++
18 files changed, 2444 insertions(+), 68 deletions(-)
diff --git a/docker/integration-tests/resource/http/paging/paging.conf
b/docker/integration-tests/resource/http/paging/paging.conf
index 90838ca49a..031a75b092 100644
--- a/docker/integration-tests/resource/http/paging/paging.conf
+++ b/docker/integration-tests/resource/http/paging/paging.conf
@@ -25,6 +25,10 @@
# Deliberately dumb: nginx maps request state to a canned response. The paging
token a mode sends
# is what selects the page, so a mode that fails to send its token gets page
one again and the
# test fails on the duplicate ids.
+#
+# /envelope is the exception to "one endpoint per mode": a second PAGE_NUMBER
endpoint shaped like
+# the Ironclad public API (issues #9548, #9549). Pages are zero-based, the
page size has to be
+# sent, and the items come wrapped in an envelope that is never empty, even
past the last page.
events {
worker_connections 1024;
@@ -70,6 +74,24 @@ http {
3 '';
}
+ # ----------------------------------------------------------------
PAGE_NUMBER, envelope
+ # ?page=0&pageSize=2 is the first page. Anything else, including page=1
without a page size
+ # (what page-number paging sent before #9548), is past the end: an
envelope with "list":[].
+ map "$arg_page:$arg_pageSize" $envelope_page {
+ default 4;
+ '0:2' 1;
+ '1:2' 2;
+ '2:2' 3;
+ }
+ map $arg_page $envelope_page_number {
+ default 0;
+ '~^[0-9]+$' $arg_page;
+ }
+ map $arg_pageSize $envelope_page_size {
+ default 20;
+ '~^[0-9]+$' $arg_pageSize;
+ }
+
# ----------------------------------------------------------------
OFFSET_LIMIT
# offset advances by the configured limit; an offset past the end returns
an empty array,
# which is the stop signal.
@@ -106,6 +128,14 @@ http {
return 200 $items;
}
+ # ------------------------------------------------------------
PAGE_NUMBER, envelope
+ # {"page":N,"pageSize":M,"count":6,"list":[...]}. The body is never
empty, so paging can
+ # only see the end through the items path, or stop early through the
total count.
+ location /envelope {
+ set $paging_page $envelope_page;
+ return 200
'{"page":$envelope_page_number,"pageSize":$envelope_page_size,"count":6,"list":$items}';
+ }
+
# ------------------------------------------------------------
OFFSET_LIMIT
location /offset {
set $paging_page $offset_page;
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
index 54b78653f1..c611548e49 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
@@ -162,12 +162,46 @@ Pagination is configured on the connection; the REST
client reads these settings
|Pagination strategy|How Hop obtains the next page. See the strategies below.
Default is `NONE`.
|Page / cursor parameter name|Query parameter name for `PAGE_NUMBER` or
`CURSOR` (for example `page`, `cursor`), body field name for `BODY_CURSOR`, or
request header name for `HEADER_CURSOR`. Ignored by `OFFSET_LIMIT`,
`LINK_HEADER` and `BODY_NEXT_URL`.
|Offset query parameter name|Query parameter for the starting index in
`OFFSET_LIMIT` paging. Defaults to `offset` when blank.
-|Limit / batch size parameter name|Parameter for the batch size, used by
`OFFSET_LIMIT`, `BODY_CURSOR` and `HEADER_CURSOR`. Defaults to `limit` when
blank.
-|Default batch size|Page size sent with `OFFSET_LIMIT`, `BODY_CURSOR` and
`HEADER_CURSOR`. When `0`, Hop falls back to an internal default (100).
+|First page number|The page number `PAGE_NUMBER` starts at. Leave empty for
`1`. Use `0` for an API that counts pages from zero. Supports variables.
+|Page size parameter name|Query parameter that carries the page size with
`PAGE_NUMBER` (for example `pageSize`). The value is the default batch size.
Leave empty to send no page size, which is what page-number paging always did.
+|Limit / batch size parameter name|Parameter for the batch size, used by
`OFFSET_LIMIT`, `BODY_CURSOR` and `HEADER_CURSOR`. Defaults to `limit` when
blank. Not used by `PAGE_NUMBER`, which has its own page size parameter name.
+|Default batch size|Page size sent with `OFFSET_LIMIT`, `BODY_CURSOR` and
`HEADER_CURSOR`. When `0`, Hop falls back to an internal default (100).
`PAGE_NUMBER` sends it as the page size, only when the page size parameter name
is filled in as well.
|Cursor JsonPath|JsonPath expression against a JSON response body to read the
next cursor token, for every cursor strategy.
|Cursor XPath|XPath expression against an XML response body to read the next
cursor token, for every cursor strategy.
|Next page URL JsonPath|JsonPath expression that reads the next page URL out
of a JSON response body (`BODY_NEXT_URL`).
|Next page URL XPath|XPath expression that reads the next page URL out of an
XML response body (`BODY_NEXT_URL`).
+|Items JsonPath / XPath|Where the items are in a page, for example `$.list` or
`/envelope/list/item`. Used by `PAGE_NUMBER`, `OFFSET_LIMIT` and `CURSOR`: a
page where this finds nothing ends paging. Leave empty to keep the default
rule. See **When paging stops** below. When both paths are set, an XML response
uses the XPath and any other response the JsonPath; the same goes for the total
count paths. An XPath is only used with an XML application type; otherwise the
transform logs that it [...]
+|Total count JsonPath / XPath|Where the API reports the total number of items,
for example `$.count` or `/envelope/count`. Used by `PAGE_NUMBER`,
`OFFSET_LIMIT` and `CURSOR`: paging stops once that many items were read, so
the last, empty, page is never requested. The items are counted with the items
path, so the total count is only used together with an items path; without one,
the transform logs that the total count is not used. It does not count the rows
of the result split path, beca [...]
+|===
+
+=== When paging stops
+
+`PAGE_NUMBER`, `OFFSET_LIMIT` and `CURSOR` request pages until one comes back
empty. What counts as empty:
+
+. When **Items JsonPath** (or **Items XPath**) is set: a page where it finds
no items.
+. Otherwise, when the REST client transform has a result split path: a page
where the split path finds nothing.
+. Otherwise: a page with an empty response body.
+
+Many APIs wrap their items in an envelope that is never empty, for example
`{"page":3,"pageSize":20,"count":46,"list":[]}`. Without an items path or a
split path, Hop keeps requesting pages until the transform's maximum number of
page requests is reached. Set the items path to `$.list` for such an API, also
when the transform has no split path because it wants one row per page. The
items path decides when paging stops; the split path still decides how a page
is split into rows.
+
+A JsonPath to an array, such as `$.list`, counts the elements of that array.
An XPath counts the nodes it selects, so select the items themselves, for
example `/envelope/list/item`. An XPath to the element around the items, such
as `/envelope/list`, still ends paging on the empty `<list/>` of the last page,
but counts every page as one item, so a total count cannot stop paging early
with it.
+
+When the transform also has a split path, the two should point at the same
items. If the items path finds items in a page where the split path gives no
rows, or finds nothing in a first page where the split path does give rows, the
transform logs both paths once.
+
+A non-2xx response always ends the loop, and the transform's maximum number of
page requests (256 by default) is a safeguard for every strategy.
+
+Example for an API that counts pages from zero and takes a page size (`GET
/workflows?page=0&pageSize=100`):
+
+[options="header"]
+|===
+|Option |Value
+|Pagination strategy|`PAGE_NUMBER`
+|Page / cursor parameter name|`page`
+|First page number|`0`
+|Page size parameter name|`pageSize`
+|Default batch size|`100`
+|Items JsonPath|`$.list`
+|Total count JsonPath|`$.count`
|===
=== Pagination strategies
@@ -178,7 +212,7 @@ Pagination is configured on the connection; the REST client
reads these settings
|`NONE`|No pagination metadata is applied. Use this for single-request APIs.
|`LINK_HEADER`|Follows the RFC 5988 `Link` response header: Hop requests the
bracketed URI whose `rel` relation includes `next`. GitHub, Shopify, and
similar APIs use this style. Hop stops when no `next` link is present or when
the same URL would be requested again (cycle guard).
|`OFFSET_LIMIT`|Adds offset/limit (or custom-named) query parameters. After
each successful non-empty page, the offset increases by the batch size. An
empty page ends the loop.
-|`PAGE_NUMBER`|Adds a page query parameter starting at `1` and increments it
after each successful non-empty page. An empty page ends the loop.
+|`PAGE_NUMBER`|Adds a page query parameter starting at the first page number
(`1` by default) and increments it after each successful non-empty page. When
the page size parameter name and the default batch size are both filled in, the
page size is sent as well. An empty page ends the loop.
|`CURSOR`|Adds a cursor query parameter on follow-up requests. The token is
extracted from the previous response with JsonPath (JSON) or XPath (XML). An
empty page or missing next token ends the loop.
|`BODY_CURSOR`|Merges the cursor and batch size into the request body of a
`POST`, `PUT` or `PATCH` (JSON or form url-encoded, following the application
type). On a `GET` they are sent as query parameters instead, because HTTP
clients do not send a `GET` body. The loop ends when the next cursor is empty —
not when a page is empty, since APIs of this shape (Slack, for example) can
return sparse pages while still advertising a next cursor.
|`HEADER_CURSOR`|Sends the cursor as a request header rather than a parameter.
The next token is read from the body as with `CURSOR`, and an empty token ends
the loop.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
index a65f09e14c..b835c4b10a 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
@@ -206,7 +206,8 @@ Use this tab to loop over paged REST APIs. Paging behaviour
(Link header, offset
Paging stops when:
-* the connection strategy reports no further page (no `rel="next"` link, empty
page, or missing cursor),
+* the connection strategy reports no further page (no `rel="next"` link, empty
page, or missing cursor). For `PAGE_NUMBER`, `OFFSET_LIMIT` and `CURSOR`, a
page is empty when the connection's items path finds nothing in it, or else
when this transform's split path finds nothing, or else when the body is empty
(see xref:metadata-types/rest-connection.adoc[REST connection], **When paging
stops**),
+* the connection's total count path says every item was read,
* HTTP status is outside the 2xx range,
* the maximum page request limit is reached, or
* for `LINK_HEADER`, the next URL was already fetched (cycle guard).
diff --git
a/integration-tests/http/0054-rest-pagination-page-number-zero-based.hpl
b/integration-tests/http/0054-rest-pagination-page-number-zero-based.hpl
new file mode 100644
index 0000000000..2408c069c7
--- /dev/null
+++ b/integration-tests/http/0054-rest-pagination-page-number-zero-based.hpl
@@ -0,0 +1,329 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0054-rest-pagination-page-number-zero-based</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>PAGE_NUMBER pagination against an API that counts pages from
zero and needs a page size (#9548): first page number 0, page size parameter
pageSize=2.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 10:00:00.000</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>REST client</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>REST client</from>
+ <to>read items</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>read items</from>
+ <to>count items</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count items</from>
+ <to>check pagination</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>check pagination</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>800</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>REST client</name>
+ <type>Rest</type>
+ <description>Paging is driven entirely by the REST connection; the split
path turns each
+ page into one row per item. Starting at page 1, or without the page size,
the mock answers
+ with an empty list and no item is read.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <applicationType>JSON</applicationType>
+ <bodyField></bodyField>
+ <connectionTimeout/>
+ <connection_name>paging-envelope-zero-based</connection_name>
+ <dynamicMethod>N</dynamicMethod>
+ <headers>
+</headers>
+ <httpLogin/>
+ <httpPassword>Encrypted </httpPassword>
+ <ignoreSsl>N</ignoreSsl>
+ <matrixParameters>
+</matrixParameters>
+ <maxPagesLoops>8</maxPagesLoops>
+ <method>GET</method>
+ <methodFieldName/>
+ <paginationEnabled>Y</paginationEnabled>
+ <parameters>
+</parameters>
+ <proxyHost/>
+ <proxyPort/>
+ <readTimeout/>
+ <result>
+ <code>result_status</code>
+ <name>result</name>
+ <response_header/>
+ <response_time/>
+ </result>
+ <resultSplitPath>$.list[*]</resultSplitPath>
+ <trustStoreFile/>
+ <trustStorePassword>Encrypted </trustStorePassword>
+ <url>/envelope</url>
+ <urlField/>
+ <urlInField>N</urlInField>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>read items</name>
+ <type>JsonInput</type>
+ <description>Pulls the id out of each item row the split path
produced.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <include>N</include>
+ <include_field/>
+ <rownum>N</rownum>
+ <addresultfile>N</addresultfile>
+ <readurl>N</readurl>
+ <removeSourceField>N</removeSourceField>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <ignoreMissingPath>Y</ignoreMissingPath>
+ <defaultPathLeafToNull>Y</defaultPathLeafToNull>
+ <rownum_field/>
+ <file>
+ <name/>
+ <filemask/>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
+ </file>
+ <fields>
+ <field>
+ <name>item_id</name>
+ <path>$.id</path>
+ <type>Integer</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <trim_type>none</trim_type>
+ <repeat>N</repeat>
+ </field>
+ </fields>
+ <limit>0</limit>
+ <IsInFields>Y</IsInFields>
+ <IsAFile>N</IsAFile>
+ <valueField>result</valueField>
+ <shortFileFieldName/>
+ <pathFieldName/>
+ <hiddenFieldName/>
+ <lastModificationTimeFieldName/>
+ <uriNameFieldName/>
+ <rootUriNameFieldName/>
+ <extensionFieldName/>
+ <sizeFieldName/>
+ <attributes/>
+ <GUI>
+ <xloc>400</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count items</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <all_rows>N</all_rows>
+ <ignore_aggregate>N</ignore_aggregate>
+ <field_ignore/>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>grp</prefix>
+ <add_linenr>N</add_linenr>
+ <linenr_fieldname/>
+ <give_back_row>Y</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>row_count</aggregate>
+ <subject>item_id</subject>
+ <type>COUNT_ANY</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>distinct_ids</aggregate>
+ <subject>item_id</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>check pagination</name>
+ <type>FilterRows</type>
+ <description>Fails unless all three pages were fetched exactly once: 6
rows, 6 distinct
+ ids. The group by always gives back a row, so reading nothing fails
too.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <conditions>
+ <condition>
+ <negated>Y</negated>
+ <leftvalue>row_count</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>6</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ <condition>
+ <negated>Y</negated>
+ <operator>OR</operator>
+ <leftvalue>distinct_ids</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>6</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ </conditions>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>656</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/http/0055-rest-pagination-items-path.hpl
b/integration-tests/http/0055-rest-pagination-items-path.hpl
new file mode 100644
index 0000000000..f44138ceae
--- /dev/null
+++ b/integration-tests/http/0055-rest-pagination-items-path.hpl
@@ -0,0 +1,261 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0055-rest-pagination-items-path</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>PAGE_NUMBER pagination over an envelope without a result
split path (#9549): the items path $.list ends paging on the first page whose
list is empty.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 10:00:00.000</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>REST client</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>REST client</from>
+ <to>count pages</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count pages</from>
+ <to>check pagination</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>check pagination</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>800</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>REST client</name>
+ <type>Rest</type>
+ <description>No result split path: one row per page. The envelope is never
empty, so only
+ the connection's items path (or total count) can end paging before the
10-page safeguard.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <applicationType>JSON</applicationType>
+ <bodyField></bodyField>
+ <connectionTimeout/>
+ <connection_name>paging-envelope-items</connection_name>
+ <dynamicMethod>N</dynamicMethod>
+ <headers>
+</headers>
+ <httpLogin/>
+ <httpPassword>Encrypted </httpPassword>
+ <ignoreSsl>N</ignoreSsl>
+ <matrixParameters>
+</matrixParameters>
+ <maxPagesLoops>10</maxPagesLoops>
+ <method>GET</method>
+ <methodFieldName/>
+ <paginationEnabled>Y</paginationEnabled>
+ <parameters>
+</parameters>
+ <proxyHost/>
+ <proxyPort/>
+ <readTimeout/>
+ <result>
+ <code>result_status</code>
+ <name>result</name>
+ <response_header/>
+ <response_time/>
+ </result>
+ <resultSplitPath/>
+ <trustStoreFile/>
+ <trustStorePassword>Encrypted </trustStorePassword>
+ <url>/envelope</url>
+ <urlField/>
+ <urlInField>N</urlInField>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count pages</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <all_rows>N</all_rows>
+ <ignore_aggregate>N</ignore_aggregate>
+ <field_ignore/>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>grp</prefix>
+ <add_linenr>N</add_linenr>
+ <linenr_fieldname/>
+ <give_back_row>Y</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>page_count</aggregate>
+ <subject>result</subject>
+ <type>COUNT_ANY</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>distinct_pages</aggregate>
+ <subject>result</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>check pagination</name>
+ <type>FilterRows</type>
+ <description>Fails unless exactly 4 different pages were fetched: the
three pages with items and the
+ first empty one. 10 means paging ran into the safeguard.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <conditions>
+ <condition>
+ <negated>Y</negated>
+ <leftvalue>page_count</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>4</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ <condition>
+ <negated>Y</negated>
+ <operator>OR</operator>
+ <leftvalue>distinct_pages</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>4</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ </conditions>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>656</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/http/0055-rest-pagination-total-count.hpl
b/integration-tests/http/0055-rest-pagination-total-count.hpl
new file mode 100644
index 0000000000..cc90b73879
--- /dev/null
+++ b/integration-tests/http/0055-rest-pagination-total-count.hpl
@@ -0,0 +1,261 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0055-rest-pagination-total-count</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>PAGE_NUMBER pagination over an envelope without a result
split path (#9549): the total count path $.count stops paging after the last
full page.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 10:00:00.000</modified_date>
+ <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>REST client</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>REST client</from>
+ <to>count pages</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count pages</from>
+ <to>check pagination</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>check pagination</from>
+ <to>Abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>800</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <interval_in_ms>5000</interval_in_ms>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <never_ending>N</never_ending>
+ <limit>1</limit>
+ <row_time_field>now</row_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>REST client</name>
+ <type>Rest</type>
+ <description>No result split path: one row per page. The envelope is never
empty, so only
+ the connection's items path (or total count) can end paging before the
10-page safeguard.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <applicationType>JSON</applicationType>
+ <bodyField></bodyField>
+ <connectionTimeout/>
+ <connection_name>paging-envelope-total-count</connection_name>
+ <dynamicMethod>N</dynamicMethod>
+ <headers>
+</headers>
+ <httpLogin/>
+ <httpPassword>Encrypted </httpPassword>
+ <ignoreSsl>N</ignoreSsl>
+ <matrixParameters>
+</matrixParameters>
+ <maxPagesLoops>10</maxPagesLoops>
+ <method>GET</method>
+ <methodFieldName/>
+ <paginationEnabled>Y</paginationEnabled>
+ <parameters>
+</parameters>
+ <proxyHost/>
+ <proxyPort/>
+ <readTimeout/>
+ <result>
+ <code>result_status</code>
+ <name>result</name>
+ <response_header/>
+ <response_time/>
+ </result>
+ <resultSplitPath/>
+ <trustStoreFile/>
+ <trustStorePassword>Encrypted </trustStorePassword>
+ <url>/envelope</url>
+ <urlField/>
+ <urlInField>N</urlInField>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count pages</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <all_rows>N</all_rows>
+ <ignore_aggregate>N</ignore_aggregate>
+ <field_ignore/>
+ <directory>${java.io.tmpdir}</directory>
+ <prefix>grp</prefix>
+ <add_linenr>N</add_linenr>
+ <linenr_fieldname/>
+ <give_back_row>Y</give_back_row>
+ <group>
+ </group>
+ <fields>
+ <field>
+ <aggregate>page_count</aggregate>
+ <subject>result</subject>
+ <type>COUNT_ANY</type>
+ <valuefield/>
+ </field>
+ <field>
+ <aggregate>distinct_pages</aggregate>
+ <subject>result</subject>
+ <type>COUNT_DISTINCT</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>528</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>check pagination</name>
+ <type>FilterRows</type>
+ <description>Fails unless exactly 3 different pages were fetched: the
total count stops paging once 6
+ items were read, so the empty page is never requested.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to/>
+ <send_false_to/>
+ <compare>
+ <condition>
+ <negated>N</negated>
+ <conditions>
+ <condition>
+ <negated>Y</negated>
+ <leftvalue>page_count</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>3</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ <condition>
+ <negated>Y</negated>
+ <operator>OR</operator>
+ <leftvalue>distinct_pages</leftvalue>
+ <function>=</function>
+ <rightvalue/>
+ <value>
+ <name>constant</name>
+ <type>Integer</type>
+ <text>3</text>
+ <length>-1</length>
+ <precision>0</precision>
+ <isnull>N</isnull>
+ <mask>#</mask>
+ </value>
+ </condition>
+ </conditions>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>656</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/http/main-0054-rest-pagination-page-number-zero-based.hwf
b/integration-tests/http/main-0054-rest-pagination-page-number-zero-based.hwf
new file mode 100644
index 0000000000..9039ff549f
--- /dev/null
+++
b/integration-tests/http/main-0054-rest-pagination-page-number-zero-based.hwf
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0054-rest-pagination-page-number-zero-based</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>PAGE_NUMBER pagination with a first page number of 0 and a page
size parameter (#9548).</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 10:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0054-rest-pagination-page-number-zero-based.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+
<filename>${PROJECT_HOME}/0054-rest-pagination-page-number-zero-based.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0054-rest-pagination-page-number-zero-based.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/http/main-0055-rest-pagination-items-path.hwf
b/integration-tests/http/main-0055-rest-pagination-items-path.hwf
new file mode 100644
index 0000000000..cf5ecc4b6d
--- /dev/null
+++ b/integration-tests/http/main-0055-rest-pagination-items-path.hwf
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0055-rest-pagination-items-path</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Paging over an API that wraps its items in an envelope (#9549):
stop on the items path, and stop on the total count.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/17 10:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/17 10:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0055-rest-pagination-items-path.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0055-rest-pagination-items-path.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>256</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0055-rest-pagination-total-count.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0055-rest-pagination-total-count.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>512</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0055-rest-pagination-items-path.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0055-rest-pagination-items-path.hpl</from>
+ <to>0055-rest-pagination-total-count.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/http/metadata/restconnection/paging-envelope-items.json
b/integration-tests/http/metadata/restconnection/paging-envelope-items.json
new file mode 100644
index 0000000000..cbb515de65
--- /dev/null
+++ b/integration-tests/http/metadata/restconnection/paging-envelope-items.json
@@ -0,0 +1,13 @@
+{
+ "name": "paging-envelope-items",
+ "pagination_type": "PAGE_NUMBER",
+ "page_param_name": "page",
+ "first_page_number": "0",
+ "page_size_param_name": "pageSize",
+ "default_limit": 2,
+ "auth_type": "No Auth",
+ "base_url": "http://${PAGING_HOSTNAME}",
+ "test_url": "",
+ "ignoreSsl": false,
+ "items_json_path": "$.list"
+}
diff --git
a/integration-tests/http/metadata/restconnection/paging-envelope-total-count.json
b/integration-tests/http/metadata/restconnection/paging-envelope-total-count.json
new file mode 100644
index 0000000000..75f3fe1e63
--- /dev/null
+++
b/integration-tests/http/metadata/restconnection/paging-envelope-total-count.json
@@ -0,0 +1,14 @@
+{
+ "name": "paging-envelope-total-count",
+ "pagination_type": "PAGE_NUMBER",
+ "page_param_name": "page",
+ "first_page_number": "0",
+ "page_size_param_name": "pageSize",
+ "default_limit": 2,
+ "auth_type": "No Auth",
+ "base_url": "http://${PAGING_HOSTNAME}",
+ "test_url": "",
+ "ignoreSsl": false,
+ "items_json_path": "$.list",
+ "total_count_json_path": "$.count"
+}
diff --git
a/integration-tests/http/metadata/restconnection/paging-envelope-zero-based.json
b/integration-tests/http/metadata/restconnection/paging-envelope-zero-based.json
new file mode 100644
index 0000000000..48b1af4505
--- /dev/null
+++
b/integration-tests/http/metadata/restconnection/paging-envelope-zero-based.json
@@ -0,0 +1,12 @@
+{
+ "name": "paging-envelope-zero-based",
+ "pagination_type": "PAGE_NUMBER",
+ "page_param_name": "page",
+ "first_page_number": "0",
+ "page_size_param_name": "pageSize",
+ "default_limit": 2,
+ "auth_type": "No Auth",
+ "base_url": "http://${PAGING_HOSTNAME}",
+ "test_url": "",
+ "ignoreSsl": false
+}
diff --git
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
index 36a924c731..0df2d1d8e6 100644
---
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
+++
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
@@ -261,6 +261,49 @@ public class RestConnection extends HopMetadataBase
implements IHopMetadata {
@HopMetadataProperty(key = "next_page_url_x_path")
private String nextPageUrlXPath;
+ /**
+ * Number of the first page requested with {@link
RestPaginationType#PAGE_NUMBER}, for APIs that
+ * count pages from zero. Empty means 1, which is what page-number paging
has always started at. A
+ * String so it can hold a variable.
+ */
+ @HopMetadataProperty(key = "first_page_number")
+ private String firstPageNumber;
+
+ /**
+ * Query parameter that carries the page size with {@link
RestPaginationType#PAGE_NUMBER}, sent
+ * with {@link #defaultLimit} as its value. A key of its own rather than
{@link #limitParamName}:
+ * the editor keeps the fields of other paging types, so a page-number
connection can still hold a
+ * limit parameter from an earlier offset/limit setup, and reusing it would
make such a connection
+ * start sending a parameter it never sent before. Empty sends no page size.
+ */
+ @HopMetadataProperty(key = "page_size_param_name")
+ private String pageSizeParamName;
+
+ /**
+ * JsonPath to the items in a page. When set, a page where it finds nothing
ends {@link
+ * RestPaginationType#PAGE_NUMBER}, {@link RestPaginationType#OFFSET_LIMIT}
and {@link
+ * RestPaginationType#CURSOR} paging, also when the response body itself is
not empty (an envelope
+ * such as {@code {"count":87,"list":[]}}). Empty keeps the old rule: the
result split path, or
+ * else an empty body.
+ */
+ @HopMetadataProperty(key = "items_json_path")
+ private String itemsJsonPath;
+
+ /** XPath to the items in a page, the XML counterpart of {@link
#itemsJsonPath}. */
+ @HopMetadataProperty(key = "items_x_path")
+ private String itemsXPath;
+
+ /**
+ * JsonPath to the total number of items the API will return, for example
{@code $.count}. Paging
+ * stops once that many items were read, rather than after one more, empty,
page.
+ */
+ @HopMetadataProperty(key = "total_count_json_path")
+ private String totalCountJsonPath;
+
+ /** XPath to the total number of items, the XML counterpart of {@link
#totalCountJsonPath}. */
+ @HopMetadataProperty(key = "total_count_x_path")
+ private String totalCountXPath;
+
public RestConnection(IVariables variables) {
this.variables = variables;
}
diff --git
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnectionEditor.java
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnectionEditor.java
index 41ace53116..ad2118817f 100644
---
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnectionEditor.java
+++
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnectionEditor.java
@@ -149,6 +149,12 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
private TextVar wCursorXPath;
private TextVar wNextPageUrlJsonPath;
private TextVar wNextPageUrlXPath;
+ private TextVar wFirstPageNumber;
+ private TextVar wPageSizeParamName;
+ private TextVar wItemsJsonPath;
+ private TextVar wItemsXPath;
+ private TextVar wTotalCountJsonPath;
+ private TextVar wTotalCountXPath;
private final int middle;
private final int margin;
@@ -614,14 +620,23 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
wPaginationTabItem.setText(
BaseMessages.getString(PKG,
"RestConnectionEditor.Tab.Pagination.Title"));
- Composite wPagInner = new Composite(wTabFolder, SWT.NONE);
+ // Scrolled: with the items and total count rows the tab no longer fits a
small editor.
+ // Vertical only: the info text wraps, so the height depends on the width
it gets.
+ ScrolledComposite wsPag = new ScrolledComposite(wTabFolder, SWT.V_SCROLL);
+ Composite wPagInner = new Composite(wsPag, SWT.NONE);
FormLayout pagLayout = new FormLayout();
pagLayout.marginWidth = PropsUi.getFormMargin();
pagLayout.marginHeight = PropsUi.getFormMargin();
wPagInner.setLayout(pagLayout);
PropsUi.setLook(wPagInner);
buildPaginationWidgets(wPagInner);
- wPaginationTabItem.setControl(wPagInner);
+ wsPag.setContent(wPagInner);
+ wsPag.setExpandHorizontal(true);
+ wsPag.setExpandVertical(true);
+ wsPag.addListener(
+ SWT.Resize,
+ e ->
wsPag.setMinHeight(wPagInner.computeSize(wsPag.getClientArea().width,
SWT.DEFAULT).y));
+ wPaginationTabItem.setControl(wsPag);
wTabFolder.setSelection(wConnectionTabItem);
@@ -661,7 +676,13 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
wCursorJsonPath,
wCursorXPath,
wNextPageUrlJsonPath,
- wNextPageUrlXPath
+ wNextPageUrlXPath,
+ wFirstPageNumber,
+ wPageSizeParamName,
+ wItemsJsonPath,
+ wItemsXPath,
+ wTotalCountJsonPath,
+ wTotalCountXPath
};
enableControls(controls);
}
@@ -820,6 +841,22 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
wPageParamName.setLayoutData(fdP);
lastControl = wPageParamName;
+ wFirstPageNumber =
+ addPaginationTextRow(
+ p,
+ lastControl,
+ "RestConnectionEditor.Pagination.FirstPage.Label",
+ "RestConnectionEditor.Pagination.FirstPage.Tooltip");
+ lastControl = wFirstPageNumber;
+
+ wPageSizeParamName =
+ addPaginationTextRow(
+ p,
+ lastControl,
+ "RestConnectionEditor.Pagination.PageSizeParam.Label",
+ "RestConnectionEditor.Pagination.PageSizeParam.Tooltip");
+ lastControl = wPageSizeParamName;
+
Label wlOff = new Label(p, SWT.RIGHT);
wlOff.setText(BaseMessages.getString(PKG,
"RestConnectionEditor.Pagination.OffsetParam.Label"));
PropsUi.setLook(wlOff);
@@ -948,10 +985,60 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
fdNx.left = new FormAttachment(middle, margin);
fdNx.right = new FormAttachment(100, -margin);
wNextPageUrlXPath.setLayoutData(fdNx);
+ lastControl = wNextPageUrlXPath;
+
+ wItemsJsonPath =
+ addPaginationTextRow(
+ p,
+ lastControl,
+ "RestConnectionEditor.Pagination.ItemsJson.Label",
+ "RestConnectionEditor.Pagination.ItemsJson.Tooltip");
+ wItemsXPath =
+ addPaginationTextRow(
+ p,
+ wItemsJsonPath,
+ "RestConnectionEditor.Pagination.ItemsXPath.Label",
+ "RestConnectionEditor.Pagination.ItemsXPath.Tooltip");
+ wTotalCountJsonPath =
+ addPaginationTextRow(
+ p,
+ wItemsXPath,
+ "RestConnectionEditor.Pagination.TotalCountJson.Label",
+ "RestConnectionEditor.Pagination.TotalCountJson.Tooltip");
+ wTotalCountXPath =
+ addPaginationTextRow(
+ p,
+ wTotalCountJsonPath,
+ "RestConnectionEditor.Pagination.TotalCountXPath.Label",
+ "RestConnectionEditor.Pagination.TotalCountXPath.Tooltip");
refreshPaginationSensitiveFields();
}
+ /** One label and text field below {@code above} on the Pagination tab. */
+ private TextVar addPaginationTextRow(
+ Composite p, Control above, String labelKey, String toolTipKey) {
+ Label label = new Label(p, SWT.RIGHT);
+ label.setText(BaseMessages.getString(PKG, labelKey));
+ label.setToolTipText(BaseMessages.getString(PKG, toolTipKey));
+ PropsUi.setLook(label);
+ FormData fdLabel = new FormData();
+ fdLabel.top = new FormAttachment(above, margin);
+ fdLabel.left = new FormAttachment(0, margin);
+ fdLabel.right = new FormAttachment(middle, -margin);
+ label.setLayoutData(fdLabel);
+
+ TextVar text = new TextVar(variables, p, SWT.SINGLE | SWT.LEFT |
SWT.BORDER);
+ PropsUi.setLook(text);
+ text.setToolTipText(label.getToolTipText());
+ FormData fdText = new FormData();
+ fdText.top = new FormAttachment(label, 0, SWT.CENTER);
+ fdText.left = new FormAttachment(middle, margin);
+ fdText.right = new FormAttachment(100, -margin);
+ text.setLayoutData(fdText);
+ return text;
+ }
+
private void refreshPaginationSensitiveFields() {
if (wPaginationType == null || wPaginationType.isDisposed()) {
return;
@@ -972,16 +1059,26 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
|| headerCursor;
boolean offset = RestPaginationType.OFFSET_LIMIT.equals(t);
boolean cursor = RestPaginationType.CURSOR.equals(t) || bodyCursor ||
headerCursor;
+ boolean page = RestPaginationType.PAGE_NUMBER.equals(t);
boolean limitBatch = offset || bodyCursor || headerCursor;
+ // The modes that stop on an empty page, and so can use an items path or a
total count.
+ boolean itemsStop = offset || page || RestPaginationType.CURSOR.equals(t);
wPageParamName.setEnabled(pageOrCursor);
wOffsetParamName.setEnabled(offset);
wLimitParamName.setEnabled(limitBatch);
- wDefaultLimit.setEnabled(limitBatch);
+ // The default batch size is also the page size that PAGE_NUMBER sends.
+ wDefaultLimit.setEnabled(limitBatch || page);
wCursorJsonPath.setEnabled(cursor);
wCursorXPath.setEnabled(cursor);
wNextPageUrlJsonPath.setEnabled(bodyNextUrl);
wNextPageUrlXPath.setEnabled(bodyNextUrl);
+ wFirstPageNumber.setEnabled(page);
+ wPageSizeParamName.setEnabled(page);
+ wItemsJsonPath.setEnabled(itemsStop);
+ wItemsXPath.setEnabled(itemsStop);
+ wTotalCountJsonPath.setEnabled(itemsStop);
+ wTotalCountXPath.setEnabled(itemsStop);
}
private static int parseUnsignedIntSafe(String text) {
@@ -1581,6 +1678,12 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
wCursorXPath.setText(Const.NVL(metadata.getCursorXPath(), ""));
wNextPageUrlJsonPath.setText(Const.NVL(metadata.getNextPageUrlJsonPath(), ""));
wNextPageUrlXPath.setText(Const.NVL(metadata.getNextPageUrlXPath(), ""));
+ wFirstPageNumber.setText(Const.NVL(metadata.getFirstPageNumber(), ""));
+ wPageSizeParamName.setText(Const.NVL(metadata.getPageSizeParamName(),
""));
+ wItemsJsonPath.setText(Const.NVL(metadata.getItemsJsonPath(), ""));
+ wItemsXPath.setText(Const.NVL(metadata.getItemsXPath(), ""));
+ wTotalCountJsonPath.setText(Const.NVL(metadata.getTotalCountJsonPath(),
""));
+ wTotalCountXPath.setText(Const.NVL(metadata.getTotalCountXPath(), ""));
refreshPaginationSensitiveFields();
} finally {
@@ -1659,6 +1762,12 @@ public class RestConnectionEditor extends
MetadataEditor<RestConnection> {
connection.setCursorXPath(wCursorXPath.getText());
connection.setNextPageUrlJsonPath(wNextPageUrlJsonPath.getText());
connection.setNextPageUrlXPath(wNextPageUrlXPath.getText());
+ connection.setFirstPageNumber(wFirstPageNumber.getText());
+ connection.setPageSizeParamName(wPageSizeParamName.getText());
+ connection.setItemsJsonPath(wItemsJsonPath.getText());
+ connection.setItemsXPath(wItemsXPath.getText());
+ connection.setTotalCountJsonPath(wTotalCountJsonPath.getText());
+ connection.setTotalCountXPath(wTotalCountXPath.getText());
}
@Override
diff --git
a/plugins/misc/rest/src/main/resources/org/apache/hop/metadata/rest/messages/messages_en_US.properties
b/plugins/misc/rest/src/main/resources/org/apache/hop/metadata/rest/messages/messages_en_US.properties
index 2c013b46a8..987f392e37 100644
---
a/plugins/misc/rest/src/main/resources/org/apache/hop/metadata/rest/messages/messages_en_US.properties
+++
b/plugins/misc/rest/src/main/resources/org/apache/hop/metadata/rest/messages/messages_en_US.properties
@@ -66,11 +66,23 @@ RestConnectionEditor.Pagination.PageParam.Tooltip=Query
parameter for PAGE_NUMBE
RestConnectionEditor.Pagination.OffsetParam.Label=Offset query parameter name
(OFFSET_LIMIT)
RestConnectionEditor.Pagination.LimitParam.Label=Limit / batch-size parameter
name
RestConnectionEditor.Pagination.DefaultLimit.Label=Default batch size (0 = Hop
default fallback)
-RestConnectionEditor.Pagination.DefaultLimit.Tooltip=Page size for
OFFSET_LIMIT (query) or BODY_CURSOR (request body). When 0, Hop falls back to
100 for BODY_CURSOR.
+RestConnectionEditor.Pagination.DefaultLimit.Tooltip=Page size for
OFFSET_LIMIT (query) or BODY_CURSOR (request body). When 0, Hop falls back to
100 for BODY_CURSOR. PAGE_NUMBER sends it as the page size, only when the page
size parameter name is filled in as well.
RestConnectionEditor.Pagination.CursorJson.Label=Cursor JsonPath (CURSOR /
BODY_CURSOR / HEADER_CURSOR + JSON)
RestConnectionEditor.Pagination.CursorXPath.Label=Cursor XPath (CURSOR /
BODY_CURSOR / HEADER_CURSOR + XML)
RestConnectionEditor.Pagination.NextUrlJson.Label=Next page URL JsonPath
(BODY_NEXT_URL + JSON)
RestConnectionEditor.Pagination.NextUrlXPath.Label=Next page URL XPath
(BODY_NEXT_URL + XML)
+RestConnectionEditor.Pagination.FirstPage.Label=First page number (PAGE_NUMBER)
+RestConnectionEditor.Pagination.FirstPage.Tooltip=Number of the first page to
request. Leave empty for 1. Use 0 for an API that counts pages from zero.
+RestConnectionEditor.Pagination.PageSizeParam.Label=Page size parameter name
(PAGE_NUMBER)
+RestConnectionEditor.Pagination.PageSizeParam.Tooltip=Query parameter that
carries the page size, for example pageSize. The value is the default batch
size. Leave empty to send no page size.
+RestConnectionEditor.Pagination.ItemsJson.Label=Items JsonPath (PAGE_NUMBER /
OFFSET_LIMIT / CURSOR + JSON)
+RestConnectionEditor.Pagination.ItemsJson.Tooltip=Where the items are in a
page, for example $.list. A page where this finds nothing ends paging, also
when the response wraps its items in an envelope. Leave empty to stop on an
empty result split, or an empty response body.
+RestConnectionEditor.Pagination.ItemsXPath.Label=Items XPath (PAGE_NUMBER /
OFFSET_LIMIT / CURSOR + XML)
+RestConnectionEditor.Pagination.ItemsXPath.Tooltip=Where the items are in an
XML page, for example /envelope/list/item. Select the items themselves, not the
element around them: /envelope/list still ends paging on an empty <list/>, but
counts each page as one item, so the total count cannot stop paging early. A
page where this finds nothing ends paging.
+RestConnectionEditor.Pagination.TotalCountJson.Label=Total count JsonPath
(PAGE_NUMBER / OFFSET_LIMIT / CURSOR + JSON)
+RestConnectionEditor.Pagination.TotalCountJson.Tooltip=Where the API reports
the total number of items, for example $.count. Paging stops once that many
items were read, so the last, empty, page is not requested. Only used together
with an items path, which counts the items.
+RestConnectionEditor.Pagination.TotalCountXPath.Label=Total count XPath
(PAGE_NUMBER / OFFSET_LIMIT / CURSOR + XML)
+RestConnectionEditor.Pagination.TotalCountXPath.Tooltip=Where an XML page
reports the total number of items, for example /envelope/count. Paging stops
once that many items were read, so the last, empty, page is not requested. Only
used together with an items XPath, which counts the items.
RestConnectionEditor.OAuth2.TokenUrl=Token URL
RestConnectionEditor.OAuth2.Grant=Grant type
RestConnectionEditor.OAuth2.ClientId=Client ID
diff --git
a/plugins/misc/rest/src/test/java/org/apache/hop/metadata/rest/RestConnectionPaginationTest.java
b/plugins/misc/rest/src/test/java/org/apache/hop/metadata/rest/RestConnectionPaginationTest.java
new file mode 100644
index 0000000000..397925a4af
--- /dev/null
+++
b/plugins/misc/rest/src/test/java/org/apache/hop/metadata/rest/RestConnectionPaginationTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.metadata.rest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import org.apache.hop.core.encryption.Encr;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.encryption.TwoWayPasswordEncoderPlugin;
+import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.metadata.serializer.json.JsonMetadataParser;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.json.simple.JSONObject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The paging options added for APIs that count pages from zero and wrap their
items in an envelope
+ * (issues #9548 and #9549) are new keys only: a connection saved before them
loads without them.
+ */
+class RestConnectionPaginationTest {
+
+ @BeforeAll
+ static void initPasswordEncoder() throws Exception {
+ PluginRegistry.getInstance()
+ .registerPluginClass(
+ HopTwoWayPasswordEncoder.class.getName(),
+ TwoWayPasswordEncoderPluginType.class,
+ TwoWayPasswordEncoderPlugin.class);
+ Encr.init("Hop");
+ }
+
+ @Test
+ void aConnectionSavedBeforeTheseOptionsExistedLoadsWithoutThem() throws
Exception {
+ RestConnection connection =
+ fromJson(
+
"{\"name\":\"old\",\"pagination_type\":\"PAGE_NUMBER\",\"page_param_name\":\"page\","
+ + "\"base_url\":\"http://paging\"}");
+
+ assertEquals(RestPaginationType.PAGE_NUMBER,
connection.getPaginationType());
+ // Empty first page number means page one, as before.
+ assertNull(connection.getFirstPageNumber());
+ assertNull(connection.getPageSizeParamName());
+ assertNull(connection.getItemsJsonPath());
+ assertNull(connection.getItemsXPath());
+ assertNull(connection.getTotalCountJsonPath());
+ assertNull(connection.getTotalCountXPath());
+ assertEquals(0, connection.getDefaultLimit());
+ }
+
+ @Test
+ void theNewOptionsSurviveASaveAndLoad() throws Exception {
+ RestConnection connection = new RestConnection();
+ connection.setName("ironclad");
+ connection.setPaginationType(RestPaginationType.PAGE_NUMBER);
+ connection.setFirstPageNumber("0");
+ connection.setPageSizeParamName("pageSize");
+ connection.setDefaultLimit(100);
+ connection.setItemsJsonPath("$.list");
+ connection.setItemsXPath("/page/list/item");
+ connection.setTotalCountJsonPath("$.count");
+ connection.setTotalCountXPath("/page/count");
+
+ JSONObject json = parser().getJsonObject(connection);
+ assertEquals("0", json.get("first_page_number"));
+ assertEquals("pageSize", json.get("page_size_param_name"));
+ assertEquals("$.list", json.get("items_json_path"));
+ assertEquals("$.count", json.get("total_count_json_path"));
+
+ RestConnection loaded = fromJson(json.toJSONString());
+
+ assertEquals("0", loaded.getFirstPageNumber());
+ assertEquals("pageSize", loaded.getPageSizeParamName());
+ assertEquals(100, loaded.getDefaultLimit());
+ assertEquals("$.list", loaded.getItemsJsonPath());
+ assertEquals("/page/list/item", loaded.getItemsXPath());
+ assertEquals("$.count", loaded.getTotalCountJsonPath());
+ assertEquals("/page/count", loaded.getTotalCountXPath());
+ }
+
+ private static JsonMetadataParser<RestConnection> parser() {
+ return new JsonMetadataParser<>(RestConnection.class, new
MemoryMetadataProvider());
+ }
+
+ private static RestConnection fromJson(String json) throws Exception {
+ JsonParser jsonParser = new JsonFactory().createParser(json);
+ jsonParser.nextToken();
+ return parser().loadJsonObject(RestConnection.class, jsonParser);
+ }
+}
diff --git
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
index 6aae481e47..bf3da32a7c 100644
---
a/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
+++
b/plugins/transforms/rest/src/main/java/org/apache/hop/pipeline/transforms/rest/Rest.java
@@ -21,6 +21,7 @@ import static org.apache.hop.core.Const.NVL;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import java.io.BufferedReader;
@@ -49,7 +50,6 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Supplier;
import javax.net.ssl.SSLContext;
import javax.xml.XMLConstants;
-import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
@@ -98,6 +98,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.rest.common.RestConst;
import org.json.simple.JSONObject;
import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
@@ -118,6 +120,12 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
private final ObjectMapper paginationJsonMapper = new ObjectMapper();
+ /** Whether the paging stop settings were checked, which logs the ones that
have no effect. */
+ private boolean pageStopSetupChecked;
+
+ /** Whether a page where the items path and the split path disagree was
logged. */
+ private boolean itemsAndRowsMismatchLogged;
+
/**
* Outcome of one HTTP invocation (body, timing, serialization of headers
needed for paging and
* output fields).
@@ -167,6 +175,17 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
int pageNumber = 1;
long offset = 0;
+ /** Items read so far, counted with the items path, for the total count. */
+ long itemsRead = 0;
+
+ /**
+ * The connection's items and total count paths, resolved once per input
row for the JSON or XML
+ * response this transform reads. {@code null} when not set or not usable.
+ */
+ String itemsExpression;
+
+ String totalCountExpression;
+
/** Next full URL supplied by RFC 5988 Link header paging. */
String linkNextUrl;
@@ -927,6 +946,14 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
connection != null ? connection.getPaginationType() :
RestPaginationType.NONE;
PaginationState state = new PaginationState();
state.effectiveLimit = resolvePagingLimit(connection);
+ // Only for PAGE_NUMBER: the editor keeps the fields of other paging
types, so a connection
+ // switched to another type can still hold a first page number that no
longer resolves.
+ if (RestPaginationType.PAGE_NUMBER.equals(pagingType)) {
+ state.pageNumber = resolveFirstPageNumber(connection);
+ }
+ if (stopsOnEmptyPage(pagingType)) {
+ resolvePageStopExpressions(connection, state);
+ }
String resolvedSplitPath =
resolveSplitPathOrPagingExpression(meta.getResultSplitPath());
int maxLoops =
@@ -984,10 +1011,14 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
break;
}
}
- totalEmitted += emitPagedResultRows(rowData, resolvedSplitPath, ex);
+ // Parsed at most once, however many paths (split, items, total count,
cursor) read it.
+ ParsedBody page = new ParsedBody(ex.body);
+ int rowsFromPage = emitPagedResultRows(rowData, resolvedSplitPath, ex,
page);
+ totalEmitted += rowsFromPage;
hasMore =
- continuePaginationAfterExchange(connection, pagingType, state,
resolvedSplitPath, ex);
+ continuePaginationAfterExchange(
+ connection, pagingType, state, resolvedSplitPath, ex, page,
rowsFromPage);
loopIdx++;
if (!hasMore || loopIdx >= maxLoops && isDetailed()) {
if (loopIdx >= maxLoops && hasMore && isDetailed()) {
@@ -1010,6 +1041,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
}
RestPaginationType type = connection.getPaginationType();
if (RestPaginationType.OFFSET_LIMIT.equals(type)
+ || RestPaginationType.PAGE_NUMBER.equals(type)
|| RestPaginationType.BODY_CURSOR.equals(type)
|| RestPaginationType.HEADER_CURSOR.equals(type)) {
int lim = connection.getDefaultLimit();
@@ -1018,6 +1050,32 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
return 100;
}
+ /**
+ * The page number {@link RestPaginationType#PAGE_NUMBER} paging starts at:
the connection's first
+ * page number, or 1 when it is empty. A value that is not a whole number of
zero or more is an
+ * error rather than a silent 1, which would skip or repeat a page without
anyone noticing.
+ */
+ private int resolveFirstPageNumber(RestConnection connection) throws
HopException {
+ if (connection == null || Utils.isEmpty(connection.getFirstPageNumber())) {
+ return 1;
+ }
+ String value = Const.trim(resolve(connection.getFirstPageNumber()));
+ if (Utils.isEmpty(value)) {
+ return 1;
+ }
+ try {
+ int first = Integer.parseInt(value);
+ if (first >= 0) {
+ return first;
+ }
+ } catch (NumberFormatException e) {
+ // Reported below.
+ }
+ throw new HopException(
+ BaseMessages.getString(
+ PKG, "Rest.Error.PaginationFirstPageNumber", connection.getName(),
value));
+ }
+
private LinkedHashMap<String, String> buildPagingQuery(
RestConnection conn, RestPaginationType type, PaginationState state) {
LinkedHashMap<String, String> q = new LinkedHashMap<>();
@@ -1031,8 +1089,16 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
q.put(offKey, Long.toString(state.offset));
q.put(limKey, Integer.toString(state.effectiveLimit));
}
- case PAGE_NUMBER ->
- q.put(NVL(resolve(conn.getPageParamName()), "page"),
Integer.toString(state.pageNumber));
+ case PAGE_NUMBER -> {
+ q.put(NVL(resolve(conn.getPageParamName()), "page"),
Integer.toString(state.pageNumber));
+ // The page size goes out only when both its name and its value are
filled in: page-number
+ // paging never sent one before, and an API that does not know the
parameter may reject it.
+ // Its own parameter name, not the offset/limit one, which an old
connection may still hold.
+ String limKey = resolve(conn.getPageSizeParamName());
+ if (!Utils.isEmpty(limKey) && conn.getDefaultLimit() > 0) {
+ q.put(limKey, Integer.toString(state.effectiveLimit));
+ }
+ }
case CURSOR -> {
if (!Utils.isEmpty(state.cursorToken)) {
q.put(NVL(resolve(conn.getPageParamName()), "cursor"),
state.cursorToken);
@@ -1105,7 +1171,9 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
RestPaginationType pagingType,
PaginationState state,
String resolvedSplitPath,
- RestExchangeResult ex)
+ RestExchangeResult ex,
+ ParsedBody page,
+ int rowsFromPage)
throws HopException {
if (conn == null || pagingType == null ||
RestPaginationType.NONE.equals(pagingType)) {
@@ -1113,8 +1181,6 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
}
boolean httpOk = ex.status >= 200 && ex.status < 300;
- boolean hadPayload = exchangeHadPayload(ex, resolvedSplitPath);
-
if (!httpOk) {
return false;
}
@@ -1125,28 +1191,28 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
return !Utils.isEmpty(state.linkNextUrl);
}
case BODY_NEXT_URL -> {
- state.linkNextUrl = extractNextPageUrlFromBody(conn,
ex.body).orElse(null);
+ state.linkNextUrl = extractNextPageUrlFromBody(conn,
page).orElse(null);
return !Utils.isEmpty(state.linkNextUrl);
}
case OFFSET_LIMIT -> {
- if (!hadPayload) {
+ if (!pageHadItemsAndMoreRemain(state, page, resolvedSplitPath,
rowsFromPage)) {
return false;
}
state.offset += state.effectiveLimit;
return true;
}
case PAGE_NUMBER -> {
- if (!hadPayload) {
+ if (!pageHadItemsAndMoreRemain(state, page, resolvedSplitPath,
rowsFromPage)) {
return false;
}
state.pageNumber++;
return true;
}
case CURSOR -> {
- if (!hadPayload) {
+ if (!pageHadItemsAndMoreRemain(state, page, resolvedSplitPath,
rowsFromPage)) {
return false;
}
- return extractCursorForNext(conn, ex.body)
+ return extractCursorForNext(conn, page)
.map(
tok -> {
state.cursorToken = tok;
@@ -1155,7 +1221,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
.orElse(false);
}
case BODY_CURSOR, HEADER_CURSOR -> {
- return extractCursorForNext(conn, ex.body)
+ return extractCursorForNext(conn, page)
.map(
tok -> {
state.cursorToken = tok;
@@ -1172,18 +1238,286 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
}
}
+ /** The paging types that ask for pages until one is empty, and so can use
an items path. */
+ private static boolean stopsOnEmptyPage(RestPaginationType pagingType) {
+ return RestPaginationType.PAGE_NUMBER.equals(pagingType)
+ || RestPaginationType.OFFSET_LIMIT.equals(pagingType)
+ || RestPaginationType.CURSOR.equals(pagingType);
+ }
+
+ /**
+ * Resolves the connection's items and total count paths once for this input
row, and on the first
+ * row logs the settings that have no effect, which would otherwise only
show as paging that runs
+ * on to the maximum number of page requests.
+ */
+ private void resolvePageStopExpressions(RestConnection conn, PaginationState
state) {
+ state.itemsExpression = pagingExpression(conn.getItemsJsonPath(),
conn.getItemsXPath());
+ state.totalCountExpression =
+ pagingExpression(conn.getTotalCountJsonPath(),
conn.getTotalCountXPath());
+ boolean totalCountWithoutItems =
+ !Utils.isEmpty(state.totalCountExpression) &&
Utils.isEmpty(state.itemsExpression);
+
+ if (!pageStopSetupChecked) {
+ pageStopSetupChecked = true;
+ String appType = NVL(resolve(meta.getApplicationType()), "");
+ logUnusedXPath(conn.getItemsXPath(), state.itemsExpression, appType);
+ logUnusedXPath(conn.getTotalCountXPath(), state.totalCountExpression,
appType);
+ if (totalCountWithoutItems) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "Rest.Log.PaginationTotalCountWithoutItemsPath",
state.totalCountExpression));
+ }
+ }
+ // The items are counted with the items path. A split path can give more
than one row per
+ // item, so counting its rows against the total would stop paging too
early.
+ if (totalCountWithoutItems) {
+ state.totalCountExpression = null;
+ }
+ }
+
+ /** Logs an XPath that is set but not used, because the response is not read
as XML. */
+ private void logUnusedXPath(String xPath, String usedExpression, String
appType) {
+ String resolved = resolveSplitPathOrPagingExpression(xPath);
+ if (!Utils.isEmpty(resolved) && Utils.isEmpty(usedExpression)) {
+ logBasic(BaseMessages.getString(PKG, "Rest.Log.PaginationXPathNotUsed",
resolved, appType));
+ }
+ }
+
/**
- * @return whether this response appears to carry at least one item (whole
body when no
- * split-path, or extracted elements when splitting).
+ * Whether page, offset and cursor paging should ask for another page after
this one.
+ *
+ * <p>A page is empty when the connection's items path finds nothing in it.
Without an items path
+ * the result split path decides, and without either an empty body does —
which an API that wraps
+ * its results in an envelope never sends (#9549). The items path wins over
the split path because
+ * it describes the API, while the split path describes the rows this
transform wants.
+ *
+ * <p>With an items path and a total count path, paging also stops once that
many items were read,
+ * so the last, empty, page is not fetched at all.
+ *
+ * @param rowsFromPage the rows the split path gave for this page, compared
with the items found
*/
- private boolean exchangeHadPayload(RestExchangeResult ex, String
resolvedSplitPath)
+ private boolean pageHadItemsAndMoreRemain(
+ PaginationState state, ParsedBody page, String resolvedSplitPath, int
rowsFromPage)
throws HopException {
- List<String> items =
- Utils.isEmpty(resolvedSplitPath) ? null : splitResultItems(ex.body,
resolvedSplitPath);
- if (items != null) {
- return !items.isEmpty();
+ boolean splitting = !Utils.isEmpty(resolvedSplitPath);
+ if (Utils.isEmpty(state.itemsExpression)) {
+ // The split path already ran over this page to emit its rows.
+ return splitting ? rowsFromPage > 0 : !Utils.isEmpty(NVL(page.text(),
"").trim());
+ }
+ long items = countPageItems(page, state.itemsExpression);
+ if (splitting) {
+ logItemsAndRowsMismatch(state, resolvedSplitPath, items, rowsFromPage);
+ }
+ if (items == 0) {
+ return false;
+ }
+ state.itemsRead += items;
+
+ Long totalCount =
+ Utils.isEmpty(state.totalCountExpression)
+ ? null
+ : readTotalCount(page, state.totalCountExpression);
+ if (totalCount != null && state.itemsRead >= totalCount) {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG, "Rest.Log.PaginationTotalCountReached", state.itemsRead,
totalCount));
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * The items path decides when paging stops and the split path which rows a
page gives. When they
+ * point at different things, paging either runs on over pages that give no
rows, or stops after a
+ * first page that did give rows. Both are logged once, naming both paths.
+ */
+ private void logItemsAndRowsMismatch(
+ PaginationState state, String resolvedSplitPath, long items, int
rowsFromPage) {
+ if (itemsAndRowsMismatchLogged) {
+ return;
+ }
+ String message = null;
+ if (items > 0 && rowsFromPage == 0) {
+ message =
+ BaseMessages.getString(
+ PKG,
+ "Rest.Log.PaginationItemsWithoutRows",
+ state.itemsExpression,
+ items,
+ resolvedSplitPath);
+ } else if (items == 0 && rowsFromPage > 0 && state.itemsRead == 0) {
+ // itemsRead is still 0 only on the first page: every page that
continues adds its items.
+ message =
+ BaseMessages.getString(
+ PKG,
+ "Rest.Log.PaginationRowsWithoutItems",
+ state.itemsExpression,
+ resolvedSplitPath,
+ rowsFromPage);
+ }
+ if (message != null) {
+ itemsAndRowsMismatchLogged = true;
+ logBasic(message);
+ }
+ }
+
+ /**
+ * Picks the JsonPath or the XPath of a pair of paging expressions: the
XPath for an XML response
+ * when one is set, otherwise the JsonPath. A connection can hold both when
JSON and XML
+ * transforms share it. A JsonPath with no XPath next to it is always used,
so a mistake in it
+ * fails loudly rather than being ignored. Returns {@code null} when neither
applies.
+ */
+ private String pagingExpression(String jsonPath, String xPath) {
+ String appType = NVL(resolve(meta.getApplicationType()), "");
+ String json = resolveSplitPathOrPagingExpression(jsonPath);
+ String xml = resolveSplitPathOrPagingExpression(xPath);
+ if (!Utils.isEmpty(xml) && isXmlApplicationType(appType)) {
+ return xml;
+ }
+ if (!Utils.isEmpty(json) && (shouldParseResponseAsJson(appType, json) ||
Utils.isEmpty(xml))) {
+ return json;
+ }
+ return null;
+ }
+
+ private static boolean isXmlApplicationType(String appType) {
+ return RestMeta.APPLICATION_TYPE_XML.equals(appType)
+ || RestMeta.APPLICATION_TYPE_TEXT_XML.equals(appType)
+ || RestMeta.APPLICATION_TYPE_ATOM_XML.equals(appType)
+ || RestMeta.APPLICATION_TYPE_SVG_XML.equals(appType)
+ || RestMeta.APPLICATION_TYPE_XHTML.equals(appType);
+ }
+
+ /**
+ * The number of items the items path finds in a page. A JsonPath to an
array counts its elements,
+ * like the result split path. An XPath counts the nodes it selects, except
empty ones: an element
+ * without attributes, child elements or text, such as the {@code <list/>}
of an empty page when
+ * the path points at the container rather than at the items in it.
+ */
+ private long countPageItems(ParsedBody page, String expression) throws
HopException {
+ String appType = NVL(resolve(meta.getApplicationType()), "");
+ try {
+ if (shouldParseResponseAsJson(appType, expression)) {
+ Object raw = page.json().read(expression);
+ if (raw instanceof Iterable<?> iterable) {
+ long count = 0;
+ for (Object element : iterable) {
+ if (element != null) {
+ count++;
+ }
+ }
+ return count;
+ }
+ return raw == null ? 0 : 1;
+ }
+ if (isXmlApplicationType(appType)) {
+ NodeList nodes =
+ (NodeList)
+ XPathFactory.newInstance()
+ .newXPath()
+ .compile(expression)
+ .evaluate(page.xml(), XPathConstants.NODESET);
+ long count = 0;
+ for (int i = 0; i < nodes.getLength(); i++) {
+ if (!isEmptyXmlNode(nodes.item(i))) {
+ count++;
+ }
+ }
+ return count;
+ }
+ } catch (Exception e) {
+ throw new HopException(
+ BaseMessages.getString(PKG, "Rest.Error.PaginationItems",
expression), e);
+ }
+ throw new HopException(
+ BaseMessages.getString(PKG, "Rest.Error.PaginationItemsType",
expression, appType));
+ }
+
+ private static boolean isEmptyXmlNode(Node node) {
+ if (node instanceof Element element) {
+ if (element.hasAttributes()) {
+ return false;
+ }
+ for (Node child = element.getFirstChild(); child != null; child =
child.getNextSibling()) {
+ if (child.getNodeType() == Node.ELEMENT_NODE) {
+ return false;
+ }
+ }
+ }
+ return Utils.isEmpty(Const.trim(node.getTextContent()));
+ }
+
+ /**
+ * The total number of items the API reports through the connection's total
count path, or {@code
+ * null} when the page does not hold a whole number there. A page the path
cannot be read from is
+ * logged and treated the same way: paging then goes on until a page has no
items.
+ */
+ private Long readTotalCount(ParsedBody page, String expression) {
+ String value;
+ try {
+ // The same JSON-or-XML rule as the items, so the count and the items
agree.
+ if (shouldParseResponseAsJson(NVL(resolve(meta.getApplicationType()),
""), expression)) {
+ Object raw = page.json().read(expression);
+ if (raw instanceof Number number) {
+ return number.longValue();
+ }
+ value = raw == null ? null : raw.toString();
+ } else {
+ value =
XPathFactory.newInstance().newXPath().compile(expression).evaluate(page.xml());
+ }
+ } catch (Exception e) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "Rest.Log.PaginationTotalCountUnreadable", expression,
e.getMessage()));
+ return null;
+ }
+ if (Utils.isEmpty(Const.trim(value))) {
+ return null;
+ }
+ try {
+ return Long.parseLong(Const.trim(value));
+ } catch (NumberFormatException e) {
+ logBasic(
+ BaseMessages.getString(
+ PKG, "Rest.Log.PaginationTotalCountNotANumber", expression,
value));
+ return null;
+ }
+ }
+
+ /**
+ * One page's response body, parsed as JSON or XML on first use and then
kept, so that the split
+ * path, the items path, the total count and the cursor do not each parse it
again.
+ */
+ private final class ParsedBody {
+ private final String text;
+ private DocumentContext json;
+ private Document xml;
+
+ ParsedBody(String text) {
+ this.text = text;
+ }
+
+ String text() {
+ return text;
+ }
+
+ DocumentContext json() {
+ if (json == null) {
+ json = JsonPath.using(JSON_PATH_CONFIGURATION).parse(NVL(text, "{}"));
+ }
+ return json;
+ }
+
+ Document xml() throws Exception {
+ if (xml == null) {
+ try (StringReader sr = new StringReader(NVL(text, ""))) {
+ xml = documentBuilderFactory().newDocumentBuilder().parse(new
InputSource(sr));
+ }
+ }
+ return xml;
}
- return !Utils.isEmpty(NVL(ex.body, "").trim());
}
/** Uses RFC 5988 / RFC 8288 Web Link parsing ({@code LinkHeaderPaging}) for
{@code rel=next}. */
@@ -1315,7 +1649,8 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
}
private int emitPagedResultRows(
- Object[] rowTemplate, String resolvedSplitPath, RestExchangeResult ex)
throws HopException {
+ Object[] rowTemplate, String resolvedSplitPath, RestExchangeResult ex,
ParsedBody page)
+ throws HopException {
String splitPathResolved = Utils.isEmpty(resolvedSplitPath) ? null :
resolvedSplitPath;
if (splitPathResolved == null) {
@@ -1323,7 +1658,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
return 1;
}
- List<String> chunks = splitResultItems(ex.body, splitPathResolved);
+ List<String> chunks = splitResultItems(page, splitPathResolved);
if (chunks.isEmpty()) {
return 0;
}
@@ -1348,7 +1683,8 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
return emitted;
}
- private List<String> splitResultItems(String body, String jsonOrXPathExpr)
throws HopException {
+ private List<String> splitResultItems(ParsedBody page, String
jsonOrXPathExpr)
+ throws HopException {
String appType = NVL(resolve(meta.getApplicationType()), "");
List<String> out = new ArrayList<>();
@@ -1356,23 +1692,12 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
// Prefer JsonPath when the expression is JsonPath or the request type
is JSON. Request
// Content-Type (e.g. FORM URLENCODED) does not dictate the response
payload format.
if (shouldParseResponseAsJson(appType, jsonOrXPathExpr)) {
- Object raw =
- JsonPath.using(JSON_PATH_CONFIGURATION).parse(NVL(body,
"{}")).read(jsonOrXPathExpr);
+ Object raw = page.json().read(jsonOrXPathExpr);
appendStructuredJsonPieces(out, raw);
return out;
}
- if (RestMeta.APPLICATION_TYPE_XML.equals(appType)
- || RestMeta.APPLICATION_TYPE_TEXT_XML.equals(appType)
- || RestMeta.APPLICATION_TYPE_ATOM_XML.equals(appType)
- || RestMeta.APPLICATION_TYPE_SVG_XML.equals(appType)
- || RestMeta.APPLICATION_TYPE_XHTML.equals(appType)) {
-
- DocumentBuilderFactory df = documentBuilderFactory();
- DocumentBuilder db = df.newDocumentBuilder();
- Document doc;
- try (StringReader sr = new StringReader(NVL(body, ""))) {
- doc = db.parse(new InputSource(sr));
- }
+ if (isXmlApplicationType(appType)) {
+ Document doc = page.xml();
XPathFactory xpf = XPathFactory.newInstance();
XPath xp = xpf.newXPath();
XPathExpression expr = xp.compile(jsonOrXPathExpr);
@@ -1440,7 +1765,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
}
}
- private java.util.Optional<String> extractCursorForNext(RestConnection conn,
String body) {
+ private java.util.Optional<String> extractCursorForNext(RestConnection conn,
ParsedBody page) {
if (conn == null) {
return java.util.Optional.empty();
}
@@ -1449,10 +1774,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
String cursorJsonPath = conn.getCursorJsonPath();
if (shouldParseResponseAsJson(appType, resolve(cursorJsonPath))
&& !Utils.isEmpty(cursorJsonPath)) {
- Object raw =
- JsonPath.using(JSON_PATH_CONFIGURATION)
- .parse(NVL(body, "{}"))
- .read(resolve(cursorJsonPath));
+ Object raw = page.json().read(resolve(cursorJsonPath));
if (raw == null) {
return java.util.Optional.empty();
}
@@ -1467,11 +1789,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
if (Utils.isEmpty(conn.getCursorXPath())) {
return java.util.Optional.empty();
}
- DocumentBuilderFactory df = documentBuilderFactory();
- Document doc;
- try (StringReader sr = new StringReader(NVL(body, ""))) {
- doc = df.newDocumentBuilder().parse(new InputSource(sr));
- }
+ Document doc = page.xml();
XPathFactory xpf = XPathFactory.newInstance();
XPathExpression expr =
xpf.newXPath().compile(resolveSplitPathOrPagingExpression(conn.getCursorXPath()));
@@ -1485,7 +1803,8 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
return java.util.Optional.empty();
}
- private java.util.Optional<String> extractNextPageUrlFromBody(RestConnection
conn, String body) {
+ private java.util.Optional<String> extractNextPageUrlFromBody(
+ RestConnection conn, ParsedBody page) {
if (conn == null) {
return java.util.Optional.empty();
}
@@ -1494,10 +1813,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
String nextPageUrlJsonPath = conn.getNextPageUrlJsonPath();
if (shouldParseResponseAsJson(appType, resolve(nextPageUrlJsonPath))
&& !Utils.isEmpty(nextPageUrlJsonPath)) {
- Object raw =
- JsonPath.using(JSON_PATH_CONFIGURATION)
- .parse(NVL(body, "{}"))
- .read(resolveSplitPathOrPagingExpression(nextPageUrlJsonPath));
+ Object raw =
page.json().read(resolveSplitPathOrPagingExpression(nextPageUrlJsonPath));
return normalizeNextPageUrl(raw);
}
@@ -1507,11 +1823,7 @@ public class Rest extends BaseTransform<RestMeta,
RestData> {
if (Utils.isEmpty(conn.getNextPageUrlXPath())) {
return java.util.Optional.empty();
}
- DocumentBuilderFactory df = documentBuilderFactory();
- Document doc;
- try (StringReader sr = new StringReader(NVL(body, ""))) {
- doc = df.newDocumentBuilder().parse(new InputSource(sr));
- }
+ Document doc = page.xml();
XPathFactory xpf = XPathFactory.newInstance();
XPathExpression expr =
xpf.newXPath().compile(resolveSplitPathOrPagingExpression(conn.getNextPageUrlXPath()));
diff --git
a/plugins/transforms/rest/src/main/resources/org/apache/hop/pipeline/transforms/rest/messages/messages_en_US.properties
b/plugins/transforms/rest/src/main/resources/org/apache/hop/pipeline/transforms/rest/messages/messages_en_US.properties
index 203c068618..8903ebd4d1 100644
---
a/plugins/transforms/rest/src/main/resources/org/apache/hop/pipeline/transforms/rest/messages/messages_en_US.properties
+++
b/plugins/transforms/rest/src/main/resources/org/apache/hop/pipeline/transforms/rest/messages/messages_en_US.properties
@@ -60,6 +60,16 @@ Rest.Log.PagingEmptyJsonArrayFirstPage=REST paging: HTTP
success with an empty J
Rest.Log.PagingEmptyCollectionFirstPage=REST paging: HTTP success with an
empty collection at split path "{0}"; no rows to emit.
Rest.Log.LinkPaginationStoppedRepeatedUrl=REST Link header pagination stopped:
the next request URL was already fetched ({0}), avoiding an infinite loop when
rel="next" repeats or cycles.
Rest.Log.PaginationFetchedPage=REST pagination: fetched results page {0} from
[{1}]
+Rest.Log.PaginationTotalCountReached=REST pagination: stopping after {0}
items, the total count the API reported is {1}.
+Rest.Log.PaginationTotalCountNotANumber=REST pagination: the total count path
"{0}" returned "{1}", which is not a whole number. Paging continues until a
page has no items.
+Rest.Log.PaginationTotalCountUnreadable=REST pagination: could not read the
total count with "{0}" from the response ({1}). Paging continues until a page
has no items.
+Rest.Log.PaginationTotalCountWithoutItemsPath=REST pagination: the total count
path "{0}" is not used, because the connection has no items path to count the
items with. Set the items path as well.
+Rest.Log.PaginationXPathNotUsed=REST pagination: the XPath "{0}" on the
connection is not used, because the application type "{1}" is not an XML type.
Set an XML application type, or use a JsonPath.
+Rest.Log.PaginationItemsWithoutRows=REST pagination: the items path "{0}"
found {1} items in a page, but the result split path "{2}" gave no rows for it.
Paging goes on while the items path finds items: check that both paths point at
the same items.
+Rest.Log.PaginationRowsWithoutItems=REST pagination: the items path "{0}"
found no items in the first page, so paging stops, but the result split path
"{1}" gave {2} rows for it. If the API has more pages, check the items path.
+Rest.Error.PaginationItems=REST pagination: could not read the items with
"{0}" from the response.
+Rest.Error.PaginationItemsType=REST pagination: the items path "{0}" needs a
JsonPath (starting with $) or an XML application type, got "{1}".
+Rest.Error.PaginationFirstPageNumber=REST connection "{0}": the first page
number "{1}" is not a whole number of zero or more.
Rest.Error.PagingSplitProducedZeroRows=REST paging with a result split path
produced no rows. Last HTTP status: {0}. Split path used: "{1}". First response
excerpt: {2}
Rest.Error.PagingAuthFailed=REST paging stopped: HTTP {0} (authentication or
permission denied). Check the REST connection (Bearer token, API key header,
and resolved project/environment variables). Response excerpt: {1}
Rest.Error.PagingSplitProducedZeroRowsNoResponse=REST paging with a result
split path produced no rows and no HTTP response was recorded (unexpected).
diff --git
a/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingEnvelopeTest.java
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingEnvelopeTest.java
new file mode 100644
index 0000000000..4861367d46
--- /dev/null
+++
b/plugins/transforms/rest/src/test/java/org/apache/hop/pipeline/transforms/rest/RestPagingEnvelopeTest.java
@@ -0,0 +1,607 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.rest;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.core5.http.ClassicHttpRequest;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.io.HttpClientResponseHandler;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.rest.RestConnection;
+import org.apache.hop.metadata.rest.RestPaginationType;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.rest.fields.ResultField;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Page-number paging against an API that counts pages from zero, wants a page
size, and wraps its
+ * items in an envelope that is never empty (issues #9548 and #9549). The fake
API below behaves
+ * like the Ironclad public API: {@code GET ?page=N&pageSize=M} returns {@code
+ * {"page":N,"pageSize":M,"count":6,"list":[...]}}, with {@code "list":[]}
past the last page.
+ */
+class RestPagingEnvelopeTest {
+
+ private static final int TOTAL_ITEMS = 6;
+
+ @BeforeEach
+ void setUp() {
+ if (!HopLogStore.isInitialized()) {
+ HopLogStore.init();
+ }
+ }
+
+ // ------------------------------------------------------------------ #9548
+
+ @Test
+ void zeroBasedPagesWithAPageSizeReadEveryItemOnce() throws Exception {
+ RestConnection connection = pageNumberConnection();
+ connection.setFirstPageNumber("0");
+ connection.setPageSizeParamName("pageSize");
+ connection.setDefaultLimit(2);
+ PagingRest rest = build(connection, "$.list[*]", 10,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(
+ List.of("page=0&pageSize=2", "page=1&pageSize=2", "page=2&pageSize=2",
"page=3&pageSize=2"),
+ rest.queries);
+ assertEquals(TOTAL_ITEMS, rest.emitted.size());
+ }
+
+ @Test
+ void anOldConnectionStillStartsAtPageOneWithoutAPageSize() throws Exception {
+ // Saved before #9548: no first page number and no page size parameter.
The editor keeps the
+ // fields of other paging modes, so a limit parameter and a default limit
left over from an
+ // earlier OFFSET_LIMIT setup must not make it send a page size it never
sent before.
+ RestConnection connection = pageNumberConnection();
+ connection.setLimitParamName("pageSize");
+ connection.setDefaultLimit(2);
+ PagingRest rest = build(connection, null, 1, uri -> "{\"list\":[1]}");
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(List.of("page=1"), rest.queries);
+ }
+
+ @Test
+ void aDefaultLimitWithoutAParameterNameSendsNoPageSize() throws Exception {
+ RestConnection connection = pageNumberConnection();
+ connection.setDefaultLimit(50);
+ PagingRest rest = build(connection, null, 1, uri -> "{\"list\":[1]}");
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(List.of("page=1"), rest.queries);
+ }
+
+ @Test
+ void theFirstPageNumberCanBeAVariable() throws Exception {
+ RestConnection connection = pageNumberConnection();
+ connection.setFirstPageNumber("${FIRST_PAGE}");
+ PagingRest rest = build(connection, null, 1, uri -> "{\"list\":[1]}");
+ rest.setVariable("FIRST_PAGE", "0");
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(List.of("page=0"), rest.queries);
+ }
+
+ @Test
+ void aFirstPageNumberThatIsNotANumberIsAnError() throws Exception {
+ RestConnection connection = pageNumberConnection();
+ connection.setFirstPageNumber("first");
+ PagingRest rest = build(connection, null, 1, uri -> "{\"list\":[1]}");
+
+ assertThrows(HopException.class, () -> rest.runPaginationLoop(new Object[]
{"row"}));
+ assertTrue(rest.queries.isEmpty());
+ }
+
+ // ------------------------------------------------------------------ #9549
+
+ @Test
+ void withoutAnItemsPathAnEnvelopeOnlyStopsAtTheSafeguard() throws Exception {
+ // Today's behaviour, kept: the body is never empty, so only maxPagesLoops
ends it.
+ PagingRest rest = build(zeroBasedEnvelope(), null, 7,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(7, rest.queries.size());
+ }
+
+ @Test
+ void anItemsPathEndsPagingOnTheFirstEmptyPage() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest = build(connection, null, 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ // One row per page, the empty last page included.
+ assertEquals(4, rest.queries.size());
+ assertEquals(4, rest.emitted.size());
+ }
+
+ @Test
+ void aTotalCountStopsAfterTheLastFullPage() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ connection.setTotalCountJsonPath("$.count");
+ PagingRest rest = build(connection, null, 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(3, rest.queries.size(), "the empty page after the last item
is not requested");
+ assertEquals(3, rest.emitted.size());
+ }
+
+ @Test
+ void aTotalCountIsOnlyUsedWithAnItemsPath() throws Exception {
+ // A split path can give several rows per item, so its rows are not
counted against the
+ // total: paging stops on the empty page instead.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setTotalCountJsonPath("$.count");
+ PagingRest rest = build(connection, "$.list[*]", 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ assertEquals(TOTAL_ITEMS, rest.emitted.size());
+ }
+
+ @Test
+ void aSplitPathWithSeveralRowsPerItemReadsEveryPage() throws Exception {
+ // Two rows per item: counting rows would reach the total of 6 after page
two of three.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setTotalCountJsonPath("$.count");
+ PagingRest rest =
+ build(
+ connection,
+ "$.list[*].tags[*]",
+ 50,
+ uri ->
+ envelopeJson(uri)
+ .replaceAll("\\{\"id\":(\\d+)}",
"{\"id\":$1,\"tags\":[\"a\",\"b\"]}"));
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ assertEquals(2 * TOTAL_ITEMS, rest.emitted.size());
+ }
+
+ @Test
+ void theItemsPathDecidesTheStopAndTheSplitPathTheRows() throws Exception {
+ // The split path picks one field per item and so always finds something;
the items path
+ // still sees the empty page.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest = build(connection, "$.page", 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ assertEquals(4, rest.emitted.size());
+ }
+
+ @Test
+ void anItemsPathAlsoEndsOffsetPaging() throws Exception {
+ RestConnection connection = new RestConnection();
+ connection.setPaginationType(RestPaginationType.OFFSET_LIMIT);
+ connection.setDefaultLimit(2);
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest =
+ build(
+ connection,
+ null,
+ 50,
+ uri -> {
+ Map<String, String> q = query(uri);
+ int offset = Integer.parseInt(q.get("offset"));
+ return envelope(offset / 2, 2);
+ });
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(
+ List.of("offset=0&limit=2", "offset=2&limit=2", "offset=4&limit=2",
"offset=6&limit=2"),
+ rest.queries);
+ }
+
+ @Test
+ void xmlItemsAndTotalCountUseTheXPaths() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list/item");
+ connection.setTotalCountXPath("/page/count");
+ PagingRest rest =
+ build(
+ connection,
+ null,
+ 50,
+ uri -> {
+ Map<String, String> q = query(uri);
+ int page = Integer.parseInt(q.get("page"));
+ StringBuilder xml = new
StringBuilder("<page><count>6</count><list>");
+ for (int id = page * 2 + 1; id <= Math.min(page * 2 + 2,
TOTAL_ITEMS); id++) {
+ xml.append("<item>").append(id).append("</item>");
+ }
+ return xml.append("</list></page>").toString();
+ });
+ rest.getMeta().setApplicationType(RestMeta.APPLICATION_TYPE_XML);
+ rest.data().mediaType = ContentType.APPLICATION_XML;
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(3, rest.queries.size());
+ }
+
+ @Test
+ void anXmlResponseUsesTheXPathsWhenJsonPathsAreSetToo() throws Exception {
+ // A connection shared by JSON and XML transforms holds both kinds of path.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ connection.setItemsXPath("/page/list/item");
+ connection.setTotalCountJsonPath("$.count");
+ connection.setTotalCountXPath("/page/count");
+ PagingRest rest = build(connection, null, 50,
RestPagingEnvelopeTest::envelopeXml);
+ rest.getMeta().setApplicationType(RestMeta.APPLICATION_TYPE_XML);
+ rest.data().mediaType = ContentType.APPLICATION_XML;
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(3, rest.queries.size());
+ }
+
+ // ------------------------------------------------------------------ review
of #9573
+
+ @Test
+ void aFirstPageNumberLeftOverFromPageNumberPagingIsIgnoredByOtherTypes()
throws Exception {
+ // The editor keeps the fields of other paging types: START_PAGE is not
set, and the first
+ // page number does not apply to offset paging, so it must not fail the
transform.
+ RestConnection connection = new RestConnection();
+ connection.setName("paging");
+ connection.setPaginationType(RestPaginationType.OFFSET_LIMIT);
+ connection.setFirstPageNumber("${START_PAGE}");
+ connection.setDefaultLimit(2);
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest =
+ build(
+ connection,
+ null,
+ 50,
+ uri -> envelope(Integer.parseInt(query(uri).get("offset")) / 2,
2));
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ }
+
+ @Test
+ void anItemsXPathToTheContainerStopsOnTheEmptyPage() throws Exception {
+ // /page/list selects the <list/> of the empty page too; it must not count
as an item.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list");
+ PagingRest rest = xmlRest(connection);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ }
+
+ @Test
+ void anItemsXPathToTheContainerNeverStopsOnTheTotalCountTooEarly() throws
Exception {
+ // A container counts as one item per page, which is never more than the
items in it.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list");
+ connection.setTotalCountXPath("/page/count");
+ PagingRest rest = xmlRest(connection);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ }
+
+ @Test
+ void anItemsXPathCountsAnEmptyItemWithAttributes() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list/item");
+ connection.setTotalCountXPath("/page/count");
+ PagingRest rest =
+ build(
+ connection,
+ null,
+ 50,
+ uri -> envelopeXml(uri).replaceAll("<item>(\\d+)</item>", "<item
id=\"$1\"/>"));
+ rest.getMeta().setApplicationType(RestMeta.APPLICATION_TYPE_XML);
+ rest.data().mediaType = ContentType.APPLICATION_XML;
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(3, rest.queries.size());
+ }
+
+ @Test
+ void aTotalCountWithoutAnItemsPathIsLogged() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setTotalCountJsonPath("$.count");
+ PagingRest rest = build(connection, null, 7,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(1, rest.logged("\"$.count\" is not used"),
rest.logs.toString());
+ }
+
+ @Test
+ void anXPathWithANonXmlApplicationTypeIsLogged() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list/item");
+ PagingRest rest = xmlRest(connection);
+ rest.getMeta().setApplicationType(RestMeta.APPLICATION_TYPE_TEXT_PLAIN);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(1, rest.logged("\"/page/list/item\" on the connection is not
used"));
+ }
+
+ @Test
+ void aTotalCountThatCannotBeReadIsLoggedAndPagingGoesOn() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsXPath("/page/list/item");
+ connection.setTotalCountXPath("/page/count[");
+ PagingRest rest = xmlRest(connection);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(4, rest.queries.size());
+ assertTrue(rest.logged("could not read the total count") > 0,
rest.logs.toString());
+ }
+
+ @Test
+ void itemsWithoutRowsAreLoggedOnce() throws Exception {
+ // The split path points at something the API does not send.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest = build(connection, "$.data[*]", 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ assertThrows(HopException.class, () -> rest.runPaginationLoop(new Object[]
{"row"}));
+
+ assertEquals(4, rest.queries.size());
+ assertEquals(1, rest.logged("but the result split path \"$.data[*]\" gave
no rows"));
+ }
+
+ @Test
+ void rowsWithoutItemsOnTheFirstPageAreLogged() throws Exception {
+ // The items path points at something the API does not send.
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.items");
+ PagingRest rest = build(connection, "$.list[*]", 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(1, rest.queries.size());
+ assertEquals(1, rest.logged("found no items in the first page"));
+ }
+
+ @Test
+ void anItemsPathAndASplitPickingPageFieldsLogNothing() throws Exception {
+ RestConnection connection = zeroBasedEnvelope();
+ connection.setItemsJsonPath("$.list");
+ PagingRest rest = build(connection, "$.page", 50,
RestPagingEnvelopeTest::envelopeJson);
+
+ rest.runPaginationLoop(new Object[] {"row"});
+
+ assertEquals(0, rest.logged("the items path"), rest.logs.toString());
+ }
+
+ // ------------------------------------------------------------------ helpers
+
+ private PagingRest xmlRest(RestConnection connection) throws Exception {
+ PagingRest rest = build(connection, null, 50,
RestPagingEnvelopeTest::envelopeXml);
+ rest.getMeta().setApplicationType(RestMeta.APPLICATION_TYPE_XML);
+ rest.data().mediaType = ContentType.APPLICATION_XML;
+ return rest;
+ }
+
+ private static String envelopeXml(URI uri) {
+ int page = Integer.parseInt(query(uri).get("page"));
+ StringBuilder xml = new StringBuilder("<page><count>" + TOTAL_ITEMS +
"</count><list>");
+ for (int id = page * 2 + 1; id <= Math.min(page * 2 + 2, TOTAL_ITEMS);
id++) {
+ xml.append("<item>").append(id).append("</item>");
+ }
+ return xml.append("</list></page>").toString();
+ }
+
+ private static RestConnection pageNumberConnection() {
+ RestConnection connection = new RestConnection();
+ connection.setName("paging");
+ connection.setPaginationType(RestPaginationType.PAGE_NUMBER);
+ connection.setPageParamName("page");
+ return connection;
+ }
+
+ private static RestConnection zeroBasedEnvelope() {
+ RestConnection connection = pageNumberConnection();
+ connection.setFirstPageNumber("0");
+ connection.setPageSizeParamName("pageSize");
+ connection.setDefaultLimit(2);
+ return connection;
+ }
+
+ /** The Ironclad-style envelope for a zero-based page of the given size. */
+ private static String envelopeJson(URI uri) {
+ Map<String, String> q = query(uri);
+ return envelope(Integer.parseInt(q.get("page")),
Integer.parseInt(q.get("pageSize")));
+ }
+
+ private static String envelope(int page, int pageSize) {
+ StringBuilder list = new StringBuilder();
+ for (int id = page * pageSize + 1; id <= Math.min((page + 1) * pageSize,
TOTAL_ITEMS); id++) {
+ if (!list.isEmpty()) {
+ list.append(',');
+ }
+ list.append("{\"id\":").append(id).append('}');
+ }
+ return "{\"page\":"
+ + page
+ + ",\"pageSize\":"
+ + pageSize
+ + ",\"count\":"
+ + TOTAL_ITEMS
+ + ",\"list\":["
+ + list
+ + "]}";
+ }
+
+ private static Map<String, String> query(URI uri) {
+ Map<String, String> q = new LinkedHashMap<>();
+ if (uri.getRawQuery() != null) {
+ for (String pair : uri.getRawQuery().split("&")) {
+ int eq = pair.indexOf('=');
+ q.put(pair.substring(0, eq), pair.substring(eq + 1));
+ }
+ }
+ return q;
+ }
+
+ private PagingRest build(
+ RestConnection connection, String splitPath, int maxPages, Function<URI,
String> api)
+ throws Exception {
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("TestRest");
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("TestRest");
+ pipelineMeta.addTransform(transformMeta);
+
+ RestMeta meta = new RestMeta();
+ meta.setDefault();
+ meta.setMethod(RestMeta.HTTP_METHOD_GET);
+ meta.setUrl("http://example.com/items");
+ meta.setPaginationEnabled(true);
+ meta.setMaxPagesLoops(maxPages);
+ meta.setResultSplitPath(splitPath);
+ meta.setResultField(new ResultField());
+ meta.getResultField().setFieldName("result");
+ meta.getResultField().setCode("status");
+
+ RestData data = new RestData();
+ data.mediaType = ContentType.APPLICATION_JSON;
+ data.method = RestMeta.HTTP_METHOD_GET;
+ data.realUrl = "http://example.com/items";
+ data.resultFieldName = "result";
+ data.resultCodeFieldName = "status";
+
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("field1"));
+ data.inputRowMeta = rowMeta;
+ data.outputRowMeta = rowMeta.clone();
+
+ PagingRest rest =
+ new PagingRest(transformMeta, meta, data, 0, pipelineMeta, new
LocalPipelineEngine());
+ rest.setMetadataProvider(mock(IHopMetadataProvider.class));
+ data.client = fakeApi(api, rest.queries);
+
+ // The connection is normally loaded from metadata in init().
+ Field field = Rest.class.getDeclaredField("connection");
+ field.setAccessible(true);
+ field.set(rest, connection);
+ return rest;
+ }
+
+ /** A client that answers each request with whatever the API function
returns for its URI. */
+ private static CloseableHttpClient fakeApi(Function<URI, String> api,
List<String> queries)
+ throws Exception {
+ CloseableHttpClient client = mock(CloseableHttpClient.class);
+ when(client.execute(any(ClassicHttpRequest.class),
any(HttpClientResponseHandler.class)))
+ .thenAnswer(
+ invocation -> {
+ ClassicHttpRequest request = invocation.getArgument(0);
+ URI uri = request.getUri();
+ queries.add(uri.getRawQuery());
+ BasicClassicHttpResponse response = new
BasicClassicHttpResponse(200);
+ response.setEntity(
+ new ByteArrayEntity(
+ api.apply(uri).getBytes(StandardCharsets.UTF_8),
+ ContentType.APPLICATION_JSON));
+ HttpClientResponseHandler<?> handler = invocation.getArgument(1);
+ return handler.handleResponse(response);
+ });
+ return client;
+ }
+
+ /** Collects the emitted rows. */
+ static class PagingRest extends Rest {
+ final List<Object[]> emitted = new ArrayList<>();
+ final List<String> queries = new ArrayList<>();
+ final List<String> logs = new ArrayList<>();
+
+ PagingRest(
+ TransformMeta transformMeta,
+ RestMeta meta,
+ RestData data,
+ int copyNr,
+ PipelineMeta pipelineMeta,
+ Pipeline pipeline) {
+ super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
+ }
+
+ RestData data() {
+ return (RestData) getData();
+ }
+
+ @Override
+ public void putRow(IRowMeta rowMeta, Object[] row) {
+ emitted.add(row);
+ }
+
+ @Override
+ public void logBasic(String message) {
+ logs.add(message);
+ super.logBasic(message);
+ }
+
+ /** How many basic log lines contain the given text. */
+ long logged(String text) {
+ return logs.stream().filter(line -> line.contains(text)).count();
+ }
+ }
+}