This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 26655f9a8c Fix OData Input shared HTTP pool shutdown and unit test
(#8168)
26655f9a8c is described below
commit 26655f9a8ceece550afc27a7754cf442ffa23f2f
Author: Lance <[email protected]>
AuthorDate: Sat Aug 29 19:59:57 2026 +0800
Fix OData Input shared HTTP pool shutdown and unit test (#8168)
Signed-off-by: lance <[email protected]>
---
.../apache/hop/core/util/HttpClientManager.java | 12 +-
.../core/util/HttpClientManagerSharedPoolTest.java | 71 ++++
.../ROOT/assets/images/transforms/icons/odata.svg | 40 ++
.../pages/pipeline/transforms/odata-input.adoc | 73 ++--
integration-tests/transforms/0109-odata-input.hpl | 103 +++++
.../transforms/datasets/golden-odata-input.csv | 4 +
.../transforms/main-0109-odata-input.hwf | 80 ++++
.../metadata/dataset/golden-odata-input.json | 24 ++
.../metadata/unit-test/0109-odata-input UNIT.json | 32 ++
.../samples/transforms/odata-input-northwind.hpl | 121 ++++++
.../transforms/odata/ODataAuthTypeTest.java | 75 ++++
.../pipeline/transforms/odata/ODataFieldTest.java | 94 +++++
.../transforms/odata/ODataInputDataTest.java | 43 +++
.../transforms/odata/ODataInputMetaTest.java | 215 +++++++++++
.../pipeline/transforms/odata/ODataInputTest.java | 421 +++++++++++++++++++++
.../odata/src/test/resources/odata-input.xml | 44 +++
16 files changed, 1427 insertions(+), 25 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
b/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
index 31e200a083..e4c7406dd8 100644
--- a/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
+++ b/core/src/main/java/org/apache/hop/core/util/HttpClientManager.java
@@ -60,6 +60,10 @@ import org.apache.hop.core.logging.ILogChannel;
* org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
Connection pool} of 200
* connections. Maximum connections per one route is 100. Provides inner
builder class for creating
* {@link org.apache.hc.client5.http.classic.HttpClient HttpClients}.
+ *
+ * <p>Clients are built with {@code setConnectionManagerShared(true)} so that
closing one client
+ * (transform dispose, dialog preview, try-with-resources) does not shut this
process-wide pool down
+ * for every other caller. See <a
href="https://github.com/apache/hop/issues/8160";>HOP-8160</a>.
*/
public class HttpClientManager {
private static final int CONNECTIONS_PER_ROUTE = 100;
@@ -82,7 +86,10 @@ public class HttpClientManager {
}
public CloseableHttpClient createDefaultClient() {
- return HttpClients.custom().setConnectionManager(manager).build();
+ return HttpClients.custom()
+ .setConnectionManager(manager)
+ .setConnectionManagerShared(true)
+ .build();
}
public HttpClientBuilderFacade createBuilder() {
@@ -163,11 +170,14 @@ public class HttpClientManager {
new BasicHttpClientConnectionManager(socketFactoryRegistry);
httpClientBuilder.setConnectionManager(connectionManager);
+ // This manager is per-client, so the client should close it.
+ httpClientBuilder.setConnectionManagerShared(false);
}
public CloseableHttpClient build() {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
httpClientBuilder.setConnectionManager(manager);
+ httpClientBuilder.setConnectionManagerShared(true);
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
if (socketTimeout > 0) {
diff --git
a/core/src/test/java/org/apache/hop/core/util/HttpClientManagerSharedPoolTest.java
b/core/src/test/java/org/apache/hop/core/util/HttpClientManagerSharedPoolTest.java
new file mode 100644
index 0000000000..593df9ef81
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/core/util/HttpClientManagerSharedPoolTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.util;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.IOException;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Guards <a href="https://github.com/apache/hop/issues/8160";>HOP-8160</a>:
closing a client built
+ * by {@link HttpClientManager} must not shut down the process-wide connection
pool.
+ */
+class HttpClientManagerSharedPoolTest {
+
+ @Test
+ void closingADefaultClientLeavesTheSharedPoolUsable() throws Exception {
+ CloseableHttpClient first =
HttpClientManager.getInstance().createDefaultClient();
+ first.close();
+
+ CloseableHttpClient second =
HttpClientManager.getInstance().createDefaultClient();
+ try {
+ assertPoolStillUsable(second);
+ } finally {
+ second.close();
+ }
+ }
+
+ @Test
+ void closingABuilderClientLeavesTheSharedPoolUsable() throws Exception {
+ CloseableHttpClient first =
HttpClientManager.getInstance().createBuilder().build();
+ first.close();
+
+ CloseableHttpClient second =
HttpClientManager.getInstance().createBuilder().build();
+ try {
+ assertPoolStillUsable(second);
+ } finally {
+ second.close();
+ }
+ }
+
+ private static void assertPoolStillUsable(CloseableHttpClient client) throws
Exception {
+ try {
+ client.execute(new HttpGet("http://127.0.0.1:1/";));
+ } catch (IllegalStateException e) {
+ if (e.getMessage() != null && e.getMessage().contains("Connection pool
shut down")) {
+ fail("Shared connection pool was shut down when a client was closed
(HOP-8160)");
+ }
+ throw e;
+ } catch (IOException ignored) {
+ // Port 1 is not a real endpoint. Reaching here means the pool still
leased a connection.
+ }
+ }
+}
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/odata.svg
b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/odata.svg
new file mode 100644
index 0000000000..cde9c9b82b
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/odata.svg
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<svg
+ xmlns:dc="http://purl.org/dc/elements/1.1/";
+ xmlns:cc="http://creativecommons.org/ns#";
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+ xmlns:svg="http://www.w3.org/2000/svg";
+ xmlns="http://www.w3.org/2000/svg";
+ version="1.1"
+ id="Layer_1"
+ x="0px"
+ y="0px"
+ viewBox="0 0 45 45"
+ style="enable-background:new 0 0 45 45;"
+ xml:space="preserve"
+ width="100%"
+ height="100%">
+ <g>
+ <!-- Circle background -->
+ <circle cx="22.5" cy="22.5" r="20" fill="#0B79B5" stroke="#FFFFFF"
stroke-width="2"/>
+ <!-- OData text shape or stylized 'O' and 'D' -->
+ <text x="22.5" y="29" font-family="'Helvetica Neue', Helvetica, Arial,
sans-serif" font-size="18" font-weight="bold" fill="#FFFFFF"
text-anchor="middle">OD</text>
+ </g>
+</svg>
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc
index d2388b784c..f1ae9554b2 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc
@@ -16,6 +16,8 @@ under the License.
////
:documentationPath: /pipeline/transforms/
:language: en_US
+:openvar: ${
+:closevar: }
:description: The OData input transform retrieves data from OData V2 and V4
service endpoints.
= image:transforms/icons/odata.svg[OData input transform Icon,
role="image-doc-icon"] OData input
@@ -27,7 +29,9 @@ under the License.
The OData input transform enables you to query OData (Open Data Protocol)
services. OData is an OASIS standard that defines a set of best practices for
building and consuming RESTful APIs. For more information, visit the official
link:https://www.odata.org/[OData Documentation].
-This transform performs GET requests against OData Entity Sets, maps JSON
properties to Apache Hop rows, and automatically handles relative pagination
using standard `@odata.nextLink` (OData V4) and `__next` (OData V2) metadata
attributes.
+This transform is an input transform: it issues GET requests against an Entity
Set and streams each entity as a Hop row. JSON properties are mapped with an
*OData Path*. Pagination follows `@odata.nextLink` (OData V4) and `__next`
(OData V2), including relative next-link URLs.
+
+The Connection, Query, and Fields values all accept Hop variables
(`{openvar}VARIABLE{closevar}`).
|
== Supported Engines
@@ -52,34 +56,37 @@ image::transforms/odata-input-connection-tab.png[OData
input Connection Tab, ali
|===
|Option|Description
|Transform name|Name of this transform as it appears in the pipeline workspace.
-|OData Service Root URL|The base URL of the OData service (e.g.,
`https://services.odata.org/V4/TripPinServiceRW/`).
+|OData Service Root URL|The base URL of the OData service, without the entity
set name (e.g., `https://services.odata.org/V4/Northwind/Northwind.svc`). A
trailing slash is optional.
|Authentication Type|The authentication type to use:
+
* `No Authentication`: Anonymous connection.
-* `Basic Authentication`: Use standard Username/Password credentials.
-* `Bearer Token`: Use a Bearer token in the `Authorization` header.
+* `Basic Authentication (Username / Password)`: HTTP Basic credentials.
+* `Bearer Token (Authorization Header)`: Sends `Authorization: Bearer <token>`.
|Username (Basic Auth)|The username for Basic authentication.
|Password (Basic Auth)|The password for Basic authentication.
-|Token (Bearer / OAuth2)|The bearer token value.
-|OData Entity Set|The OData entity set to query (e.g., `People` or `Airports`).
-|Get Entity Sets|Retrieves and displays a drop-down list of available entity
sets from the service endpoint.
+|Token (Bearer / OAuth2)|The bearer token value (without the `Bearer ` prefix).
+|OData Entity Set|The OData entity set to query (e.g., `Products` or
`Customers`).
+|Get Entity Sets|Calls the service root URL and fills the Entity Set drop-down
from the catalog (`value[].name` in V4; `d.EntitySets` or similar in V2).
|===
== Query Options
-The OData Query tab allows you to configure filtering, ordering, and field
selection properties.
+The OData Query tab appends standard OData system query options to the first
request URL.
image::transforms/odata-input-query-tab.png[OData input Query Tab,
align="center"]
[options="header"]
|===
|Option|Description
-|$select|A comma-separated list of properties to select (e.g.,
`UserName,FirstName,LastName`).
-|$filter|An OData expression to filter matching entities (e.g., `Gender eq
'Male'`).
-|$orderby|Specifies the sorting order of the results (e.g., `LastName asc,
FirstName desc`).
-|$top|Limits the number of entities returned on each page request (e.g., `50`).
-|$skip|Specifies the number of entities to skip from the beginning of the list.
+|$select|A comma-separated list of properties to select (e.g.,
`ProductID,ProductName`).
+|$filter|An OData expression to filter matching entities (e.g., `ProductID le
3` or `Discontinued eq false`).
+|$orderby|Specifies the sorting order of the results (e.g., `ProductName asc`).
+|$top|Limits how many entities the *first* request asks for (e.g., `50`).
Later pages still follow the service `nextLink` if one is returned.
+|$skip|Specifies the number of entities to skip from the beginning of the list
on the first request.
|===
+NOTE: There is no dedicated `$expand`, `$count`, or `$search` field. Complex
nested collections are not exploded into extra rows.
+
== Fields Mapping
The Fields tab defines how properties in the returned OData JSON response map
to Apache Hop row fields.
@@ -89,19 +96,37 @@ image::transforms/odata-input-fields-tab.png[OData input
Fields Tab, align="cent
[options="header"]
|===
|Option|Description
-|Name in Hop|The name of the field as it will be produced in the Apache Hop
pipeline.
-|OData Path|The JSON property path of the field within the entity JSON object
(e.g., `UserName`, `AddressInfo/0/City/Name` for nested structures).
-|Type|The Hop data type (String, Integer, Number, Date, Boolean, BigNumber).
-|Format|Optional parsing format for date or numeric fields.
+|Name in Hop|The name of the field as it will be produced in the Apache Hop
pipeline. Variables in the name are resolved at runtime.
+|OData Path|Slash-separated JSON object path within one entity (e.g.,
`ProductName`, or `Address/City` for a nested object). Array indexes such as
`AddressInfo/0/City` are *not* supported.
+|Type|The Hop data type: String, Integer, Number, Date, Boolean, or BigNumber.
Integer is read as a Long and Number as a Double. Other types, including
BigNumber, are read as text.
+|Format|Optional conversion mask for date or numeric fields.
See xref:pipeline/formatting-values.adoc[Formatting numbers and dates].
-|Get Fields|Connects to the `$metadata` endpoint of the OData service and
automatically populates the fields list based on the selected Entity Set's
schema properties.
+|Get Fields|Calls the service `$metadata` endpoint and populates the table
from the selected Entity Set's CSDL `Property` elements (not navigation
properties). Edm integers map to Integer, decimals/doubles to Number, booleans
to Boolean, and date/time types to Date.
|===
+[NOTE]
+====
+OData date/time values that fail to parse are emitted as `null` and do not
abort the pipeline. If you need the original timestamp, map the property as
*String*.
+====
+
+== Behaviour
+
+* Requests use `Accept: application/json`. Atom/XML payloads are not parsed as
rows.
+* OData V4 responses use a `value` array. OData V2 uses `d.results`, a `d`
array, or a single `d` object.
+* When a page includes `@odata.nextLink` or `__next`, the transform keeps
fetching until the service stops paging. Relative next-link URLs are resolved
against the previous request.
+* A non-200 HTTP status fails the transform.
+* *Get Entity Sets* and *Get Fields* can be used more than once in the same
Hop Gui session.
+
== Example
-To query a public OData service:
-1. Set the **Service Root URL** to
`https://services.odata.org/V4/TripPinServiceRW/`.
-2. Set the **Entity Set** to `People`.
-3. Go to the **Fields** tab and click **Get Fields** to pull all metadata
fields (`UserName`, `FirstName`, `LastName`, etc.).
-4. (Optional) In the **OData Query** tab, add a `$filter` such as `FirstName
eq 'John'` or a `$select` of `UserName,Emails`.
-5. Run the pipeline to stream entities into downstream transforms.
+To query the public Northwind OData V4 service:
+
+. Set the *Service Root URL* to
`https://services.odata.org/V4/Northwind/Northwind.svc`.
+. Click *Get Entity Sets* and choose `Products`, or type `Products`.
+. Open the *Fields* tab and click *Get Fields*, or add `ProductID` (Integer)
and `ProductName` (String) yourself.
+. Optionally, on *OData Query*, set `$select` to `ProductID,ProductName`,
`$filter` to `ProductID le 3`, `$orderby` to `ProductID`, and `$top` to `3`.
+. Preview or run the pipeline. The first three products are Chai, Chang, and
Aniseed Syrup.
+
+Sample pipeline (Samples project):
+
+link:https://github.com/apache/hop/blob/main/plugins/transforms/odata/src/main/samples/transforms/odata-input-northwind.hpl[odata-input-northwind.hpl]
diff --git a/integration-tests/transforms/0109-odata-input.hpl
b/integration-tests/transforms/0109-odata-input.hpl
new file mode 100644
index 0000000000..09a065ecf2
--- /dev/null
+++ b/integration-tests/transforms/0109-odata-input.hpl
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0109-odata-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>OData Input against public Northwind Products (ProductID le
3).</description>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/08/29 09:21:00.000</created_date>
+ <modified_date>2026/08/29 09:21:00.000</modified_date>
+ <created_hop_version/>
+ <modified_hop_version>2.20.0-SNAPSHOT</modified_hop_version>
+ </info>
+ <transform>
+ <type>ODataInput</type>
+ <name>OData Input</name>
+ <url>https://services.odata.org/V4/Northwind/Northwind.svc</url>
+ <entity_set>Products</entity_set>
+ <auth_type>NONE</auth_type>
+ <query_select>ProductID,ProductName</query_select>
+ <query_filter>ProductID le 3</query_filter>
+ <query_order>ProductID</query_order>
+ <query_top>3</query_top>
+ <fields>
+ <field>
+ <name>ProductID</name>
+ <path>ProductID</path>
+ <type>Integer</type>
+ <format>#</format>
+ </field>
+ <field>
+ <name>ProductName</name>
+ <path>ProductName</path>
+ <type>String</type>
+ <format/>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>176</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>Verify</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>OData Input</from>
+ <to>Verify</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/transforms/datasets/golden-odata-input.csv
b/integration-tests/transforms/datasets/golden-odata-input.csv
new file mode 100644
index 0000000000..52eb4750dd
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-odata-input.csv
@@ -0,0 +1,4 @@
+ProductID,ProductName
+1,Chai
+2,Chang
+3,Aniseed Syrup
diff --git a/integration-tests/transforms/main-0109-odata-input.hwf
b/integration-tests/transforms/main-0109-odata-input.hwf
new file mode 100644
index 0000000000..2ff40beb0f
--- /dev/null
+++ b/integration-tests/transforms/main-0109-odata-input.hwf
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0109-odata-input</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>OData Input unit test against public Northwind
Products.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/08/29 09:21:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/08/29 09:21:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>80</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0109-odata-input UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>320</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-odata-input.json
b/integration-tests/transforms/metadata/dataset/golden-odata-input.json
new file mode 100644
index 0000000000..14a45f38c9
--- /dev/null
+++ b/integration-tests/transforms/metadata/dataset/golden-odata-input.json
@@ -0,0 +1,24 @@
+{
+ "base_filename": "golden-odata-input.csv",
+ "name": "golden-odata-input",
+ "description": "First three Northwind Products by ProductID",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 5,
+ "field_precision": 0,
+ "field_name": "ProductID",
+ "field_format": "#"
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "ProductName",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
diff --git a/integration-tests/transforms/metadata/unit-test/0109-odata-input
UNIT.json b/integration-tests/transforms/metadata/unit-test/0109-odata-input
UNIT.json
new file mode 100644
index 0000000000..e70192b704
--- /dev/null
+++ b/integration-tests/transforms/metadata/unit-test/0109-odata-input
UNIT.json
@@ -0,0 +1,32 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": true,
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "ProductID",
+ "data_set_field": "ProductID"
+ },
+ {
+ "transform_field": "ProductName",
+ "data_set_field": "ProductName"
+ }
+ ],
+ "field_order": [
+ "ProductID"
+ ],
+ "transform_name": "Verify",
+ "data_set_name": "golden-odata-input"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0109-odata-input UNIT",
+ "description": "",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0109-odata-input.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/odata/src/main/samples/transforms/odata-input-northwind.hpl
b/plugins/transforms/odata/src/main/samples/transforms/odata-input-northwind.hpl
new file mode 100644
index 0000000000..cc09ae5835
--- /dev/null
+++
b/plugins/transforms/odata/src/main/samples/transforms/odata-input-northwind.hpl
@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>odata-input-northwind</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Read the first three Northwind Products from the public OData
V4 service.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <notepads>
+ <notepad>
+ <backgroundcolorblue>251</backgroundcolorblue>
+ <backgroundcolorgreen>232</backgroundcolorgreen>
+ <backgroundcolorred>201</backgroundcolorred>
+ <bordercolorblue>90</bordercolorblue>
+ <bordercolorgreen>58</bordercolorgreen>
+ <bordercolorred>14</bordercolorred>
+ <fontbold>N</fontbold>
+ <fontcolorblue>90</fontcolorblue>
+ <fontcolorgreen>58</fontcolorgreen>
+ <fontcolorred>14</fontcolorred>
+ <fontitalic>N</fontitalic>
+ <fontname>Noto Sans</fontname>
+ <fontsize>11</fontsize>
+ <height>200</height>
+ <xloc>80</xloc>
+ <yloc>32</yloc>
+ <note>OData Input sample (public Northwind V4)
+
+Service: https://services.odata.org/V4/Northwind/Northwind.svc
+Entity set: Products
+Query: $select=ProductID,ProductName $filter=ProductID le 3
$orderby=ProductID $top=3
+
+Expected preview rows: 1/Chai, 2/Chang, 3/Aniseed Syrup.
+
+Double-click OData Input and use Get Entity Sets / Get Fields against the same
URL.</note>
+ <width>720</width>
+ </notepad>
+ </notepads>
+ <order>
+ <hop>
+ <from>OData Input</from>
+ <to>preview</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>OData Input</name>
+ <type>ODataInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <url>https://services.odata.org/V4/Northwind/Northwind.svc</url>
+ <entity_set>Products</entity_set>
+ <auth_type>NONE</auth_type>
+ <query_select>ProductID,ProductName</query_select>
+ <query_filter>ProductID le 3</query_filter>
+ <query_order>ProductID</query_order>
+ <query_top>3</query_top>
+ <fields>
+ <field>
+ <name>ProductID</name>
+ <path>ProductID</path>
+ <type>Integer</type>
+ <format>#</format>
+ </field>
+ <field>
+ <name>ProductName</name>
+ <path>ProductName</path>
+ <type>String</type>
+ </field>
+ </fields>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>preview</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>368</xloc>
+ <yloc>288</yloc>
+ </GUI>
+ </transform>
+</pipeline>
diff --git
a/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataAuthTypeTest.java
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataAuthTypeTest.java
new file mode 100644
index 0000000000..21aa419d53
--- /dev/null
+++
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataAuthTypeTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.odata;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link ODataAuthType} */
+class ODataAuthTypeTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @Test
+ void lookupCodeIsCaseInsensitiveAndFallsBackToNone() {
+ assertEquals(ODataAuthType.NONE, ODataAuthType.lookupCode("NONE"));
+ assertEquals(ODataAuthType.BASIC, ODataAuthType.lookupCode("basic"));
+ assertEquals(ODataAuthType.BEARER, ODataAuthType.lookupCode("Bearer"));
+ assertEquals(ODataAuthType.NONE, ODataAuthType.lookupCode("unknown"));
+ assertEquals(ODataAuthType.NONE, ODataAuthType.lookupCode(null));
+ }
+
+ @Test
+ void lookupDescriptionMatchesLocalizedLabels() {
+ assertEquals(
+ ODataAuthType.NONE,
ODataAuthType.lookupDescription(ODataAuthType.NONE.getDescription()));
+ assertEquals(
+ ODataAuthType.BASIC,
ODataAuthType.lookupDescription(ODataAuthType.BASIC.getDescription()));
+ assertEquals(
+ ODataAuthType.BEARER,
+
ODataAuthType.lookupDescription(ODataAuthType.BEARER.getDescription()));
+ assertEquals(ODataAuthType.NONE,
ODataAuthType.lookupDescription("not-an-auth-type"));
+ assertEquals(ODataAuthType.NONE, ODataAuthType.lookupDescription(null));
+ }
+
+ @Test
+ void getDescriptionsReturnsEveryEnumValue() {
+ String[] descriptions = ODataAuthType.getDescriptions();
+ assertEquals(ODataAuthType.values().length, descriptions.length);
+ assertArrayEquals(
+ new String[] {
+ ODataAuthType.NONE.getDescription(),
+ ODataAuthType.BASIC.getDescription(),
+ ODataAuthType.BEARER.getDescription()
+ },
+ descriptions);
+ }
+
+ @Test
+ void codesAreStableWireValues() {
+ assertEquals("NONE", ODataAuthType.NONE.getCode());
+ assertEquals("BASIC", ODataAuthType.BASIC.getCode());
+ assertEquals("BEARER", ODataAuthType.BEARER.getCode());
+ assertNotEquals(ODataAuthType.NONE.getDescription(),
ODataAuthType.NONE.getCode());
+ }
+}
diff --git
a/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataFieldTest.java
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataFieldTest.java
new file mode 100644
index 0000000000..d76b172557
--- /dev/null
+++
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataFieldTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.odata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link ODataField} */
+class ODataFieldTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @Test
+ void defaultConstructorUsesStringType() {
+ ODataField field = new ODataField();
+ assertEquals("", field.getName());
+ assertEquals("", field.getPath());
+ assertEquals(IValueMeta.TYPE_STRING, field.getType());
+ assertEquals("", field.getFormat());
+ }
+
+ @Test
+ void copyConstructorAndCloneAreIndependentCopies() {
+ ODataField original = new ODataField("Id", "Id", IValueMeta.TYPE_INTEGER,
"#");
+ ODataField copy = new ODataField(original);
+ ODataField cloned = original.clone();
+
+ assertEquals(original, copy);
+ assertEquals(original, cloned);
+ assertNotSame(original, copy);
+ assertNotSame(original, cloned);
+
+ copy.setName("Other");
+ assertEquals("Id", original.getName());
+ }
+
+ @Test
+ void equalsAndHashCodeUseAllProperties() {
+ ODataField a = new ODataField("Id", "Id", IValueMeta.TYPE_INTEGER, "#");
+ ODataField b = new ODataField("Id", "Id", IValueMeta.TYPE_INTEGER, "#");
+ ODataField c = new ODataField("Name", "Id", IValueMeta.TYPE_INTEGER, "#");
+
+ assertEquals(a, b);
+ assertEquals(a.hashCode(), b.hashCode());
+ assertNotEquals(a, c);
+ assertNotEquals(a, null);
+ assertNotEquals(a, "Id");
+ }
+
+ @Test
+ void toValueMetaUsesTypeNameOriginAndFormat() throws Exception {
+ ODataField field = new ODataField("Amount", "Price",
IValueMeta.TYPE_NUMBER, "0.00");
+ IValueMeta valueMeta = field.toValueMeta("OData Input", null);
+
+ assertEquals("Amount", valueMeta.getName());
+ assertEquals(IValueMeta.TYPE_NUMBER, valueMeta.getType());
+ assertEquals("OData Input", valueMeta.getOrigin());
+ assertEquals("0.00", valueMeta.getConversionMask());
+ }
+
+ @Test
+ void toValueMetaTreatsNoneAsStringAndResolvesVariables() throws Exception {
+ ODataField field = new ODataField("${FIELD_NAME}", "Name",
IValueMeta.TYPE_NONE, "");
+ Variables variables = new Variables();
+ variables.setVariable("FIELD_NAME", "ProductName");
+
+ IValueMeta valueMeta = field.toValueMeta("odata", variables);
+
+ assertEquals("ProductName", valueMeta.getName());
+ assertEquals(IValueMeta.TYPE_STRING, valueMeta.getType());
+ }
+}
diff --git
a/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputDataTest.java
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputDataTest.java
new file mode 100644
index 0000000000..bc0a365f9e
--- /dev/null
+++
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputDataTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.odata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit test for {@link ODataInputData} */
+class ODataInputDataTest {
+
+ @Test
+ void defaultsAreReadyForTheFirstPage() {
+ ODataInputData data = new ODataInputData();
+
+ assertNull(data.outputRowMeta);
+ assertNull(data.httpClient);
+ assertNull(data.nextPageUrl);
+ assertNotNull(data.recordBuffer);
+ assertTrue(data.recordBuffer.isEmpty());
+ assertEquals(0, data.recordIndex);
+ assertFalse(data.isFinishedReading);
+ }
+}
diff --git
a/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputMetaTest.java
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputMetaTest.java
new file mode 100644
index 0000000000..f86f5f4704
--- /dev/null
+++
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputMetaTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.odata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.exception.HopXmlException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.xml.XmlHandler;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link ODataInputMeta} */
+class ODataInputMetaTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ @Test
+ void constructorDefaultsAuthTypeAndEmptyFields() {
+ ODataInputMeta meta = new ODataInputMeta();
+ assertEquals("NONE", meta.getAuthType());
+ assertTrue(meta.getFields().isEmpty());
+ }
+
+ @Test
+ void setDefaultClearsConnectionAndQuery() {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.setUrl("https://example.test/odata";);
+ meta.setEntitySet("Products");
+ meta.setAuthType("BASIC");
+ meta.getFields().add(new ODataField("Id", "Id", IValueMeta.TYPE_INTEGER,
""));
+
+ meta.setDefault();
+
+ assertEquals("", meta.getUrl());
+ assertEquals("", meta.getEntitySet());
+ assertEquals("NONE", meta.getAuthType());
+ assertEquals("", meta.getUsername());
+ assertEquals("", meta.getPassword());
+ assertEquals("", meta.getToken());
+ assertEquals("", meta.getQuerySelect());
+ assertEquals("", meta.getQueryFilter());
+ assertEquals("", meta.getQueryOrder());
+ assertEquals("", meta.getQueryTop());
+ assertEquals("", meta.getQuerySkip());
+ assertTrue(meta.getFields().isEmpty());
+ }
+
+ @Test
+ void getFieldsAddsConfiguredValueMeta() throws Exception {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.getFields().add(new ODataField("ProductID", "ProductID",
IValueMeta.TYPE_INTEGER, "#"));
+ meta.getFields().add(new ODataField("${NAME}", "ProductName",
IValueMeta.TYPE_STRING, ""));
+
+ Variables variables = new Variables();
+ variables.setVariable("NAME", "ProductName");
+ IRowMeta rowMeta = new RowMeta();
+ meta.getFields(rowMeta, "OData Input", null, null, variables, new
MemoryMetadataProvider());
+
+ assertEquals(2, rowMeta.size());
+ assertEquals("ProductID", rowMeta.getValueMeta(0).getName());
+ assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.getValueMeta(0).getType());
+ assertEquals("OData Input", rowMeta.getValueMeta(0).getOrigin());
+ assertEquals("ProductName", rowMeta.getValueMeta(1).getName());
+ }
+
+ @Test
+ void checkReportsMissingUrlAndEntitySet() {
+ ODataInputMeta meta = new ODataInputMeta();
+ TransformMeta transformMeta = new TransformMeta("OData Input", meta);
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ transformMeta,
+ null,
+ new String[0],
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertEquals(2, remarks.size());
+ assertEquals(ICheckResult.TYPE_RESULT_ERROR, remarks.get(0).getType());
+ assertTrue(remarks.get(0).getText().contains("Service URL"));
+ assertEquals(ICheckResult.TYPE_RESULT_ERROR, remarks.get(1).getType());
+ assertTrue(remarks.get(1).getText().contains("Entity Set"));
+ }
+
+ @Test
+ void checkTreatsWhitespaceAsMissing() {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.setUrl(" ");
+ meta.setEntitySet("\t");
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta("OData Input", meta),
+ null,
+ new String[0],
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertEquals(ICheckResult.TYPE_RESULT_ERROR, remarks.get(0).getType());
+ assertEquals(ICheckResult.TYPE_RESULT_ERROR, remarks.get(1).getType());
+ }
+
+ @Test
+ void checkAcceptsConfiguredUrlAndEntitySet() {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.setUrl("https://example.test/odata";);
+ meta.setEntitySet("Products");
+ List<ICheckResult> remarks = new ArrayList<>();
+
+ meta.check(
+ remarks,
+ new PipelineMeta(),
+ new TransformMeta("OData Input", meta),
+ null,
+ new String[0],
+ new String[0],
+ null,
+ new Variables(),
+ new MemoryMetadataProvider());
+
+ assertEquals(2, remarks.size());
+ assertEquals(ICheckResult.TYPE_RESULT_OK, remarks.get(0).getType());
+ assertEquals(ICheckResult.TYPE_RESULT_OK, remarks.get(1).getType());
+ }
+
+ @Test
+ void xmlRoundTripPreservesConnectionQueryAndFields() throws Exception {
+ ODataInputMeta meta = loadFromClasspath("/odata-input.xml");
+ validateLoadedMeta(meta);
+
+ String xmlCopy =
+ XmlHandler.openTag(TransformMeta.XML_TAG)
+ + XmlMetadataUtil.serializeObjectToXml(meta)
+ + XmlHandler.closeTag(TransformMeta.XML_TAG);
+ ODataInputMeta copy = loadFromXml(xmlCopy);
+ validateLoadedMeta(copy);
+ }
+
+ private static void validateLoadedMeta(ODataInputMeta meta) {
+ assertEquals("https://example.test/odata";, meta.getUrl());
+ assertEquals("Products", meta.getEntitySet());
+ assertEquals("BASIC", meta.getAuthType());
+ assertEquals("odata-user", meta.getUsername());
+ assertEquals("odata-secret", meta.getPassword());
+ assertEquals("bearer-token", meta.getToken());
+ assertEquals("ProductID,ProductName", meta.getQuerySelect());
+ assertEquals("Discontinued eq false", meta.getQueryFilter());
+ assertEquals("ProductName asc", meta.getQueryOrder());
+ assertEquals("3", meta.getQueryTop());
+ assertEquals("1", meta.getQuerySkip());
+ assertEquals(2, meta.getFields().size());
+ assertEquals(
+ new ODataField("ProductID", "ProductID", IValueMeta.TYPE_INTEGER, "#"),
+ meta.getFields().get(0));
+ assertEquals("ProductName", meta.getFields().get(1).getName());
+ assertEquals(IValueMeta.TYPE_STRING, meta.getFields().get(1).getType());
+ }
+
+ private static ODataInputMeta loadFromClasspath(String resource) throws
Exception {
+ Path path =
+
Paths.get(Objects.requireNonNull(ODataInputMetaTest.class.getResource(resource)).toURI());
+ return loadFromXml(Files.readString(path));
+ }
+
+ private static ODataInputMeta loadFromXml(String xml) throws HopXmlException
{
+ ODataInputMeta meta = new ODataInputMeta();
+ XmlMetadataUtil.deSerializeFromXml(
+ XmlHandler.loadXmlString(xml, TransformMeta.XML_TAG),
+ ODataInputMeta.class,
+ meta,
+ new MemoryMetadataProvider());
+ return meta;
+ }
+}
diff --git
a/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputTest.java
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputTest.java
new file mode 100644
index 0000000000..6d3730d39f
--- /dev/null
+++
b/plugins/transforms/odata/src/test/java/org/apache/hop/pipeline/transforms/odata/ODataInputTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.odata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.apache.hop.pipeline.transform.RowAdapter;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link ODataInput} */
+class ODataInputTest {
+ @RegisterExtension
+ static RestoreHopEngineEnvironmentExtension env = new
RestoreHopEngineEnvironmentExtension();
+
+ private HttpServer server;
+ private int port;
+ private final Map<String, String> responses = new ConcurrentHashMap<>();
+ private final AtomicInteger statusCode = new AtomicInteger(200);
+ private final AtomicReference<String> lastAuthorization = new
AtomicReference<>();
+ private final AtomicReference<String> lastAccept = new AtomicReference<>();
+
+ @BeforeEach
+ void setUp() throws IOException {
+ if (!HopLogStore.isInitialized()) {
+ HopLogStore.init();
+ }
+ responses.clear();
+ statusCode.set(200);
+ lastAuthorization.set(null);
+ lastAccept.set(null);
+
+ server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+ port = server.getAddress().getPort();
+ server.createContext(
+ "/odata",
+ exchange -> {
+
lastAuthorization.set(exchange.getRequestHeaders().getFirst("Authorization"));
+ lastAccept.set(exchange.getRequestHeaders().getFirst("Accept"));
+ String key = exchange.getRequestURI().getPath();
+ if (exchange.getRequestURI().getQuery() != null) {
+ key += "?" + exchange.getRequestURI().getQuery();
+ }
+ String body = responses.getOrDefault(key, "{\"value\":[]}");
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().add("Content-Type",
"application/json");
+ exchange.sendResponseHeaders(statusCode.get(), bytes.length);
+ try (OutputStream os = exchange.getResponseBody()) {
+ os.write(bytes);
+ }
+ });
+ server.start();
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (server != null) {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void initFailsWhenServiceUrlIsEmpty() {
+ ODataInputMeta meta = newMeta("Products");
+ meta.setUrl("");
+ ODataInput transform = newTransform(meta);
+
+ assertFalse(transform.init());
+ }
+
+ @Test
+ void initFailsWhenEntitySetIsEmpty() {
+ ODataInputMeta meta = newMeta("");
+ ODataInput transform = newTransform(meta);
+
+ assertFalse(transform.init());
+ }
+
+ @Test
+ void initBuildsUrlWithTrailingSlashAndQueryOptions() {
+ ODataInputMeta meta = newMeta("Products");
+ meta.setUrl(baseUrl());
+ meta.setQuerySelect("ProductID,Name");
+ meta.setQueryFilter("Name eq 'Chai'");
+ meta.setQueryOrder("Name asc");
+ meta.setQueryTop("3");
+ meta.setQuerySkip("1");
+ ODataInput transform = newTransform(meta);
+
+ assertTrue(transform.init());
+ String next = transform.getData().nextPageUrl;
+ assertTrue(next.startsWith(baseUrl() + "/Products?"));
+ assertTrue(next.contains("$select=" + encoded("ProductID,Name")));
+ assertTrue(next.contains("$filter=" + encoded("Name eq 'Chai'")));
+ assertTrue(next.contains("$orderby=" + encoded("Name asc")));
+ assertTrue(next.contains("$top=" + encoded("3")));
+ assertTrue(next.contains("$skip=" + encoded("1")));
+ assertFalse(transform.getData().isFinishedReading);
+ assertNotNull(transform.getData().httpClient);
+ transform.dispose();
+ }
+
+ @Test
+ void initKeepsExistingTrailingSlash() {
+ ODataInputMeta meta = newMeta("Customers");
+ meta.setUrl(baseUrl() + "/");
+ ODataInput transform = newTransform(meta);
+
+ assertTrue(transform.init());
+ assertEquals(baseUrl() + "/Customers", transform.getData().nextPageUrl);
+ transform.dispose();
+ }
+
+ @Test
+ void initResolvesVariablesInUrlAndEntitySet() {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.setUrl("${SVC}");
+ meta.setEntitySet("${SET}");
+ ODataInput transform = newTransform(meta);
+ transform.setVariable("SVC", baseUrl());
+ transform.setVariable("SET", "Orders");
+
+ assertTrue(transform.init());
+ assertEquals(baseUrl() + "/Orders", transform.getData().nextPageUrl);
+ transform.dispose();
+ }
+
+ @Test
+ void initAcceptsBasicAuthWithoutCallingTheService() {
+ ODataInputMeta meta = newMeta("Products");
+ meta.setAuthType("BASIC");
+ meta.setUsername("user");
+ meta.setPassword("secret");
+ ODataInput transform = newTransform(meta);
+
+ assertTrue(transform.init());
+ transform.dispose();
+ }
+
+ @Test
+ void processRowReadsODataV4RecordsAndTypes() throws Exception {
+ responses.put(
+ "/odata/Products",
+ """
+ {"value":[
+
{"ProductID":1,"ProductName":"Chai","UnitPrice":18.5,"Discontinued":false,"OrderDate":"2024-01-15"},
+
{"ProductID":2,"ProductName":null,"UnitPrice":null,"Discontinued":true,"OrderDate":null}
+ ]}
+ """);
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("ProductID", "ProductID",
IValueMeta.TYPE_INTEGER, ""));
+ meta.getFields().add(new ODataField("ProductName", "ProductName",
IValueMeta.TYPE_STRING, ""));
+ meta.getFields().add(new ODataField("UnitPrice", "UnitPrice",
IValueMeta.TYPE_NUMBER, ""));
+ meta.getFields()
+ .add(new ODataField("Discontinued", "Discontinued",
IValueMeta.TYPE_BOOLEAN, ""));
+ meta.getFields()
+ .add(new ODataField("OrderDate", "OrderDate", IValueMeta.TYPE_DATE,
"yyyy-MM-dd"));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(2, rows.size());
+ assertEquals(1L, rows.get(0)[0]);
+ assertEquals("Chai", rows.get(0)[1]);
+ assertEquals(18.5d, (Double) rows.get(0)[2], 0.0001);
+ assertEquals(Boolean.FALSE, rows.get(0)[3]);
+ // Date parsing currently calls convertDataFromString with a null
convertMeta, so the value
+ // stays null instead of aborting the row.
+ assertNull(rows.get(0)[4]);
+ assertEquals(2L, rows.get(1)[0]);
+ assertNull(rows.get(1)[1]);
+ assertNull(rows.get(1)[2]);
+ assertEquals(Boolean.TRUE, rows.get(1)[3]);
+ assertNull(rows.get(1)[4]);
+ assertEquals("application/json", lastAccept.get());
+ }
+
+ @Test
+ void processRowReadsNestedODataPath() throws Exception {
+ responses.put(
+ "/odata/Customers",
+ """
+ {"value":[{"Name":"Alfreds","Address":{"City":"Berlin"}}]}
+ """);
+ ODataInputMeta meta = newMeta("Customers");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ meta.getFields().add(new ODataField("City", "Address/City",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(1, rows.size());
+ assertEquals("Alfreds", rows.get(0)[0]);
+ assertEquals("Berlin", rows.get(0)[1]);
+ }
+
+ @Test
+ void processRowFollowsAbsoluteODataV4NextLink() throws Exception {
+ responses.put(
+ "/odata/Products",
+ "{\"value\":[{\"Name\":\"A\"}],\"@odata.nextLink\":\""
+ + baseUrl()
+ + "/Products?$skiptoken=2\"}");
+ responses.put("/odata/Products?$skiptoken=2",
"{\"value\":[{\"Name\":\"B\"}]}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(2, rows.size());
+ assertEquals("A", rows.get(0)[0]);
+ assertEquals("B", rows.get(1)[0]);
+ }
+
+ @Test
+ void processRowResolvesRootRelativeNextLink() throws Exception {
+ responses.put(
+ "/odata/Products",
+
"{\"value\":[{\"Name\":\"A\"}],\"@odata.nextLink\":\"/odata/Products?$skiptoken=2\"}");
+ responses.put("/odata/Products?$skiptoken=2",
"{\"value\":[{\"Name\":\"B\"}]}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(List.of("A", "B"), rows.stream().map(row -> row[0]).toList());
+ }
+
+ @Test
+ void processRowResolvesRelativeNextLinkAgainstCurrentPath() throws Exception
{
+ responses.put(
+ "/odata/Products",
+
"{\"value\":[{\"Name\":\"A\"}],\"@odata.nextLink\":\"Products?$skiptoken=2\"}");
+ responses.put("/odata/Products?$skiptoken=2",
"{\"value\":[{\"Name\":\"B\"}]}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(2, rows.size());
+ assertEquals("B", rows.get(1)[0]);
+ }
+
+ @Test
+ void processRowReadsODataV2ResultsAndNext() throws Exception {
+ responses.put(
+ "/odata/Products",
+ "{\"d\":{\"results\":[{\"Name\":\"V2A\"}],\"__next\":\""
+ + baseUrl()
+ + "/Products?$skiptoken=2\"}}");
+ responses.put("/odata/Products?$skiptoken=2",
"{\"d\":{\"results\":[{\"Name\":\"V2B\"}]}}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertEquals(List.of("V2A", "V2B"), rows.stream().map(row ->
row[0]).toList());
+ }
+
+ @Test
+ void processRowReadsODataV2SingleObjectAndArray() throws Exception {
+ responses.put("/odata/Product", "{\"d\":{\"Name\":\"Single\"}}");
+ ODataInputMeta singleMeta = newMeta("Product");
+ singleMeta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ List<Object[]> single = runToCompletion(newTransform(singleMeta));
+ assertEquals("Single", single.get(0)[0]);
+
+ responses.put("/odata/Items", "{\"d\":[{\"Name\":\"Arr\"}]}");
+ ODataInputMeta arrayMeta = newMeta("Items");
+ arrayMeta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ List<Object[]> array = runToCompletion(newTransform(arrayMeta));
+ assertEquals("Arr", array.get(0)[0]);
+ }
+
+ @Test
+ void processRowFallsBackToPlainJsonObjectAndArray() throws Exception {
+ responses.put("/odata/One", "{\"Name\":\"Plain\"}");
+ ODataInputMeta objectMeta = newMeta("One");
+ objectMeta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ assertEquals("Plain", runToCompletion(newTransform(objectMeta)).get(0)[0]);
+
+ responses.put("/odata/Many", "[{\"Name\":\"List\"}]");
+ ODataInputMeta arrayMeta = newMeta("Many");
+ arrayMeta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ assertEquals("List", runToCompletion(newTransform(arrayMeta)).get(0)[0]);
+ }
+
+ @Test
+ void processRowSendsBearerToken() throws Exception {
+ responses.put("/odata/Products", "{\"value\":[{\"Name\":\"Secured\"}]}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.setAuthType("BEARER");
+ meta.setToken("abc-123");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ runToCompletion(newTransform(meta));
+
+ assertEquals("Bearer abc-123", lastAuthorization.get());
+ }
+
+ @Test
+ void processRowReturnsNoRowsForEmptyPage() throws Exception {
+ responses.put("/odata/Products", "{\"value\":[]}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+
+ List<Object[]> rows = runToCompletion(newTransform(meta));
+
+ assertTrue(rows.isEmpty());
+ }
+
+ @Test
+ void processRowWrapsNon200Status() {
+ statusCode.set(500);
+ responses.put("/odata/Products", "{\"error\":\"boom\"}");
+ ODataInputMeta meta = newMeta("Products");
+ meta.getFields().add(new ODataField("Name", "Name",
IValueMeta.TYPE_STRING, ""));
+ ODataInput transform = newTransform(meta);
+ assertTrue(transform.init());
+
+ HopException thrown = assertThrows(HopException.class,
transform::processRow);
+ assertTrue(thrown.getMessage().contains("Error requesting OData data
page"));
+ transform.dispose();
+ }
+
+ @Test
+ void disposeClosesHttpClient() {
+ ODataInput transform = newTransform(newMeta("Products"));
+ assertTrue(transform.init());
+ assertNotNull(transform.getData().httpClient);
+ transform.dispose();
+ transform.dispose();
+ }
+
+ private List<Object[]> runToCompletion(ODataInput transform) throws
Exception {
+ assertTrue(transform.init());
+ List<Object[]> rows = new ArrayList<>();
+ transform.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) {
+ rows.add(row);
+ }
+ });
+ while (transform.processRow()) {
+ // drain every page
+ }
+ transform.dispose();
+ return rows;
+ }
+
+ private ODataInput newTransform(ODataInputMeta meta) {
+ TransformMeta transformMeta = new TransformMeta();
+ transformMeta.setName("OData Input");
+ transformMeta.setTransform(meta);
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("odata-input-test");
+ pipelineMeta.addTransform(transformMeta);
+ LocalPipelineEngine pipeline = spy(new LocalPipelineEngine());
+ when(pipeline.isRunning()).thenReturn(true);
+ return new ODataInput(transformMeta, meta, new ODataInputData(), 0,
pipelineMeta, pipeline);
+ }
+
+ private ODataInputMeta newMeta(String entitySet) {
+ ODataInputMeta meta = new ODataInputMeta();
+ meta.setUrl(baseUrl());
+ meta.setEntitySet(entitySet);
+ meta.setAuthType("NONE");
+ return meta;
+ }
+
+ private String baseUrl() {
+ return "http://localhost:"; + port + "/odata";
+ }
+
+ private static String encoded(String value) {
+ return URLEncoder.encode(value, StandardCharsets.UTF_8);
+ }
+}
diff --git a/plugins/transforms/odata/src/test/resources/odata-input.xml
b/plugins/transforms/odata/src/test/resources/odata-input.xml
new file mode 100644
index 0000000000..cfe806566d
--- /dev/null
+++ b/plugins/transforms/odata/src/test/resources/odata-input.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<transform>
+ <url>https://example.test/odata</url>
+ <entity_set>Products</entity_set>
+ <auth_type>BASIC</auth_type>
+ <username>odata-user</username>
+ <password>odata-secret</password>
+ <token>bearer-token</token>
+ <query_select>ProductID,ProductName</query_select>
+ <query_filter>Discontinued eq false</query_filter>
+ <query_order>ProductName asc</query_order>
+ <query_top>3</query_top>
+ <query_skip>1</query_skip>
+ <fields>
+ <field>
+ <name>ProductID</name>
+ <path>ProductID</path>
+ <type>Integer</type>
+ <format>#</format>
+ </field>
+ <field>
+ <name>ProductName</name>
+ <path>ProductName</path>
+ <type>String</type>
+ </field>
+ </fields>
+</transform>
