This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 1a5eafd010 Fix for issue #2320 : The Execution Information Perspective
from Remote Location returns a HopException (IT bug fix)
new 566db9f125 Merge pull request #2418 from mattcasters/master
1a5eafd010 is described below
commit 1a5eafd010f4138d44ba428cc23cb96a869b8d3a
Author: Matt Casters <[email protected]>
AuthorDate: Fri Feb 17 14:15:23 2023 +0100
Fix for issue #2320 : The Execution Information Perspective from Remote
Location returns a HopException (IT bug fix)
---
.../remote/RemoteExecutionInfoLocation.java | 8 +-
.../main/java/org/apache/hop/server/HopServer.java | 2 +-
.../main/java/org/apache/hop/www/HopServer.java | 17 +-
.../hop/www/RegisterExecutionInfoServlet.java | 107 ++++++------
.../hop_server/0005-read-from-remote-location.hpl | 16 +-
.../hop_server/0005-verify-executions-logged.hpl | 190 +++++++++++++++++++++
.../main-0005-remote-execution-location.hwf | 51 +++++-
.../local-remote-location.json | 2 +-
.../remote-no-linked-resources.json | 7 +-
9 files changed, 321 insertions(+), 79 deletions(-)
diff --git
a/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
b/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
index 0e08720298..8e73898ac3 100644
---
a/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
+++
b/engine/src/main/java/org/apache/hop/execution/remote/RemoteExecutionInfoLocation.java
@@ -146,14 +146,14 @@ public class RemoteExecutionInfoLocation implements
IExecutionInfoLocation {
try {
validateSettings();
URI uri =
- new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH)
+ new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH+"/")
.addParameter(
RegisterExecutionInfoServlet.PARAMETER_TYPE,
RegisterExecutionInfoServlet.TYPE_EXECUTION)
.addParameter(RegisterExecutionInfoServlet.PARAMETER_LOCATION,
location.getName())
.build();
- server.sendJson(variables, uri.toString(), getJson(execution));
+ server.sendJson(variables, getJson(execution), uri.toString());
} catch (Exception e) {
throw new HopException("Error registering execution at remote location",
e);
}
@@ -185,7 +185,7 @@ public class RemoteExecutionInfoLocation implements
IExecutionInfoLocation {
try {
validateSettings();
URI uri =
- new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH)
+ new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH + "/")
.addParameter(
RegisterExecutionInfoServlet.PARAMETER_TYPE,
RegisterExecutionInfoServlet.TYPE_STATE)
@@ -203,7 +203,7 @@ public class RemoteExecutionInfoLocation implements
IExecutionInfoLocation {
try {
validateSettings();
URI uri =
- new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH)
+ new URIBuilder(RegisterExecutionInfoServlet.CONTEXT_PATH+"/")
.addParameter(
RegisterExecutionInfoServlet.PARAMETER_TYPE,
RegisterExecutionInfoServlet.TYPE_DATA)
diff --git a/engine/src/main/java/org/apache/hop/server/HopServer.java
b/engine/src/main/java/org/apache/hop/server/HopServer.java
index 127320edef..4887c0d0c9 100644
--- a/engine/src/main/java/org/apache/hop/server/HopServer.java
+++ b/engine/src/main/java/org/apache/hop/server/HopServer.java
@@ -532,7 +532,7 @@ public class HopServer extends HopMetadataBase implements
Cloneable, IXml, IHopM
HttpEntity entity = new ByteArrayEntity(content);
postMethod.setEntity(entity);
- postMethod.addHeader(new BasicHeader("Content-Type", contentType +
";charset=" + encoding));
+ postMethod.addHeader(new BasicHeader("Accept", contentType + ";charset=" +
encoding));
return postMethod;
}
diff --git a/engine/src/main/java/org/apache/hop/www/HopServer.java
b/engine/src/main/java/org/apache/hop/www/HopServer.java
index f40deab372..fe0ba00d1d 100644
--- a/engine/src/main/java/org/apache/hop/www/HopServer.java
+++ b/engine/src/main/java/org/apache/hop/www/HopServer.java
@@ -18,6 +18,16 @@
package org.apache.hop.www;
import com.google.common.annotations.VisibleForTesting;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.Socket;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileObject;
@@ -346,11 +356,8 @@ public class HopServer implements Runnable,
IHasHopMetadataProvider {
}
String hostname = variables.resolve(hopServer.getHostname());
String port = variables.resolve(hopServer.getPort());
- String shudownPort = variables.resolve(hopServer.getShutdownPort());
- parameters = List.of(hostname, port);
- if (StringUtils.isNotEmpty(shudownPort)) {
- parameters.add(shudownPort);
- }
+ String shutDownPort = variables.resolve(hopServer.getShutdownPort());
+ parameters = List.of(Const.NVL(hostname, ""), Const.NVL(port, ""),
Const.NVL(shutDownPort, ""));
}
private boolean handleQueryOptions() {
diff --git
a/engine/src/main/java/org/apache/hop/www/RegisterExecutionInfoServlet.java
b/engine/src/main/java/org/apache/hop/www/RegisterExecutionInfoServlet.java
index 0e1747647e..e587f91e5a 100644
--- a/engine/src/main/java/org/apache/hop/www/RegisterExecutionInfoServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/RegisterExecutionInfoServlet.java
@@ -17,6 +17,12 @@
package org.apache.hop.www;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.annotations.HopServerServlet;
@@ -30,13 +36,6 @@ import org.apache.hop.execution.ExecutionState;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-
@HopServerServlet(id = "registerExecInfo", name = "Register execution
information")
public class RegisterExecutionInfoServlet extends BaseHttpServlet implements
IHopServerPlugin {
private static final long serialVersionUID = -2817136625869923847L;
@@ -94,6 +93,8 @@ public class RegisterExecutionInfoServlet extends
BaseHttpServlet implements IHo
}
// Look up the location in the metadata.
+ // No caching of the location state is done.
+ // initialize() is always followed by a close().
//
MultiMetadataProvider provider =
pipelineMap.getHopServerConfig().getMetadataProvider();
IHopMetadataSerializer<ExecutionInfoLocation> serializer =
@@ -102,51 +103,57 @@ public class RegisterExecutionInfoServlet extends
BaseHttpServlet implements IHo
if (location == null) {
throw new HopException("Unable to find execution information location
" + locationName);
}
-
- // First read the complete JSON document in memory from the request
- //
- StringBuilder json = new StringBuilder(request.getContentLength());
- int c;
- while ((c = in.read()) != -1) {
- json.append((char) c);
+ try {
+ location.getExecutionInfoLocation().initialize(variables, provider);
+
+ // First read the complete JSON document in memory from the request
+ //
+ StringBuilder json = new StringBuilder(request.getContentLength());
+ int c;
+ while ((c = in.read()) != -1) {
+ json.append((char) c);
+ }
+
+ // What type of information are we receiving?
+ //
+ switch (type) {
+ case TYPE_EXECUTION:
+ Execution execution =
HopJson.newMapper().readValue(json.toString(), Execution.class);
+ location.getExecutionInfoLocation().registerExecution(execution);
+ break;
+ case TYPE_STATE:
+ ExecutionState state =
+ HopJson.newMapper().readValue(json.toString(),
ExecutionState.class);
+ location.getExecutionInfoLocation().updateExecutionState(state);
+ break;
+ case TYPE_DATA:
+ ExecutionData data =
+ HopJson.newMapper().readValue(json.toString(),
ExecutionData.class);
+ location.getExecutionInfoLocation().registerData(data);
+ break;
+ default:
+ throw new HopException(
+ "Unknown update type: "
+ + type
+ + " allowed are: "
+ + TYPE_EXECUTION
+ + ", "
+ + TYPE_STATE
+ + ", "
+ + TYPE_DATA);
+ }
+
+ // Log successful registration of execution, state or data
+ //
+ out.println(
+ new WebResult(
+ WebResult.STRING_OK, "Registration successful at location " +
locationName));
+
+ } finally {
+ location.getExecutionInfoLocation().close();
}
-
- // What type of information are we receiving?
- //
- switch (type) {
- case TYPE_EXECUTION:
- Execution execution = HopJson.newMapper().readValue(json.toString(),
Execution.class);
- location.getExecutionInfoLocation().registerExecution(execution);
- break;
- case TYPE_STATE:
- ExecutionState state =
- HopJson.newMapper().readValue(json.toString(),
ExecutionState.class);
- location.getExecutionInfoLocation().updateExecutionState(state);
- break;
- case TYPE_DATA:
- ExecutionData data = HopJson.newMapper().readValue(json.toString(),
ExecutionData.class);
- location.getExecutionInfoLocation().registerData(data);
- break;
- default:
- throw new HopException(
- "Unknown update type: "
- + type
- + " allowed are: "
- + TYPE_EXECUTION
- + ", "
- + TYPE_STATE
- + ", "
- + TYPE_DATA);
- }
-
- // Return the log channel id as well
- //
- out.println(
- new WebResult(
- WebResult.STRING_OK, "Registration successful at location " +
locationName));
-
} catch (Exception ex) {
-
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
out.println(new WebResult(WebResult.STRING_ERROR,
Const.getStackTracker(ex)));
}
}
diff --git a/integration-tests/hop_server/0005-read-from-remote-location.hpl
b/integration-tests/hop_server/0005-read-from-remote-location.hpl
index 0dbbbea742..b2fc36361d 100644
--- a/integration-tests/hop_server/0005-read-from-remote-location.hpl
+++ b/integration-tests/hop_server/0005-read-from-remote-location.hpl
@@ -58,11 +58,6 @@ limitations under the License.
<to>Validate</to>
<enabled>Y</enabled>
</hop>
- <hop>
- <from>Execution IDs</from>
- <to>Execution & State</to>
- <enabled>Y</enabled>
- </hop>
<hop>
<from>Execution & State</from>
<to>executionType=Pipeline</to>
@@ -78,6 +73,11 @@ limitations under the License.
<to>execution data</to>
<enabled>Y</enabled>
</hop>
+ <hop>
+ <from>Execution IDs</from>
+ <to>Execution & State</to>
+ <enabled>Y</enabled>
+ </hop>
</order>
<transform>
<name>Execution & State</name>
@@ -95,7 +95,7 @@ limitations under the License.
<operationType>GetExecutionAndState</operationType>
<attributes/>
<GUI>
- <xloc>416</xloc>
+ <xloc>384</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@@ -221,7 +221,7 @@ limitations under the License.
</fields>
<attributes/>
<GUI>
- <xloc>800</xloc>
+ <xloc>768</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@@ -349,7 +349,7 @@ limitations under the License.
</compare>
<attributes/>
<GUI>
- <xloc>592</xloc>
+ <xloc>560</xloc>
<yloc>48</yloc>
</GUI>
</transform>
diff --git a/integration-tests/hop_server/0005-verify-executions-logged.hpl
b/integration-tests/hop_server/0005-verify-executions-logged.hpl
new file mode 100644
index 0000000000..01a62f871a
--- /dev/null
+++ b/integration-tests/hop_server/0005-verify-executions-logged.hpl
@@ -0,0 +1,190 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0005-verify-executions-logged</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2023/02/17 13:40:27.812</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2023/02/17 13:40:27.812</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>list /tmp/executions</from>
+ <to>count</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count</from>
+ <to>count==0?</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>count==0?</from>
+ <to>Nothing logged: abort</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>list /tmp/executions</name>
+ <type>GetFileNames</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <doNotFailIfNoFile>N</doNotFailIfNoFile>
+ <dynamic_include_subfolders>N</dynamic_include_subfolders>
+ <exclude_wildcard_Field/>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask>.*</filemask>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/executions/</name>
+ </file>
+ <filefield>N</filefield>
+ <filename_Field/>
+ <filter>
+ <filterfiletype>all_files</filterfiletype>
+ </filter>
+ <isaddresult>Y</isaddresult>
+ <limit>0</limit>
+ <raiseAnExceptionIfNoFile>N</raiseAnExceptionIfNoFile>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <wildcard_Field/>
+ <attributes/>
+ <GUI>
+ <xloc>112</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <add_linenr>N</add_linenr>
+ <all_rows>N</all_rows>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <aggregate>count</aggregate>
+ <subject>filename</subject>
+ <type>COUNT_ANY</type>
+ <valuefield/>
+ </field>
+ </fields>
+ <give_back_row>Y</give_back_row>
+ <group>
+</group>
+ <ignore_aggregate>N</ignore_aggregate>
+ <linenr_fieldname/>
+ <prefix>grp</prefix>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>count==0?</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <compare>
+ <condition>
+ <conditions>
+</conditions>
+ <function>=</function>
+ <leftvalue>count</leftvalue>
+ <negated>N</negated>
+ <operator>-</operator>
+ <value>
+ <isnull>N</isnull>
+ <length>-1</length>
+ <mask>####0;-####0</mask>
+ <name>constant</name>
+ <precision>0</precision>
+ <text>0</text>
+ <type>Integer</type>
+ </value>
+ </condition>
+ </compare>
+ <send_true_to>Nothing logged: abort</send_true_to>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Nothing logged: abort</name>
+ <type>Abort</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <abort_option>ABORT_WITH_ERROR</abort_option>
+ <always_log_rows>Y</always_log_rows>
+ <message>No logging was performed on the remote server: no log folders
detected.</message>
+ <row_threshold>0</row_threshold>
+ <attributes/>
+ <GUI>
+ <xloc>608</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git
a/integration-tests/hop_server/main-0005-remote-execution-location.hwf
b/integration-tests/hop_server/main-0005-remote-execution-location.hwf
index 16c37fac5e..b38f10392e 100644
--- a/integration-tests/hop_server/main-0005-remote-execution-location.hwf
+++ b/integration-tests/hop_server/main-0005-remote-execution-location.hwf
@@ -72,7 +72,7 @@ limitations under the License.
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
- <xloc>288</xloc>
+ <xloc>272</xloc>
<yloc>64</yloc>
<attributes_hac/>
</action>
@@ -87,7 +87,7 @@ limitations under the License.
</test_name>
</test_names>
<parallel>N</parallel>
- <xloc>624</xloc>
+ <xloc>784</xloc>
<yloc>64</yloc>
<attributes_hac/>
</action>
@@ -147,6 +147,34 @@ limitations under the License.
<yloc>192</yloc>
<attributes_hac/>
</action>
+ <action>
+ <name>0005-verify-executions-logged</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0005-verify-executions-logged.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>remote-no-linked-resources</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>528</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
</actions>
<hops>
<hop>
@@ -157,22 +185,29 @@ limitations under the License.
<unconditional>Y</unconditional>
</hop>
<hop>
- <from>0005-write-to-remote-location</from>
- <to>0005-read-from-remote-location</to>
+ <from>0005-read-from-remote-location</from>
+ <to>0005-delete-from-remote-location</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
- <from>0005-read-from-remote-location</from>
- <to>0005-delete-from-remote-location</to>
+ <from>0005-delete-from-remote-location</from>
+ <to>0005-verify-deleted-on-remote-location</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
- <from>0005-delete-from-remote-location</from>
- <to>0005-verify-deleted-on-remote-location</to>
+ <from>0005-write-to-remote-location</from>
+ <to>0005-verify-executions-logged</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0005-verify-executions-logged</from>
+ <to>0005-read-from-remote-location</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
diff --git
a/integration-tests/hop_server/metadata/pipeline-run-configuration/local-remote-location.json
b/integration-tests/hop_server/metadata/pipeline-run-configuration/local-remote-location.json
index 2bacc67a42..8027e9a2a4 100644
---
a/integration-tests/hop_server/metadata/pipeline-run-configuration/local-remote-location.json
+++
b/integration-tests/hop_server/metadata/pipeline-run-configuration/local-remote-location.json
@@ -18,5 +18,5 @@
"name": "local-remote-location",
"description": "",
"dataProfile": "first-last",
- "executionInfoLocationName": "local"
+ "executionInfoLocationName": "remote"
}
\ No newline at end of file
diff --git
a/integration-tests/hop_server/metadata/pipeline-run-configuration/remote-no-linked-resources.json
b/integration-tests/hop_server/metadata/pipeline-run-configuration/remote-no-linked-resources.json
index 93d80a9ea0..5b38368d41 100644
---
a/integration-tests/hop_server/metadata/pipeline-run-configuration/remote-no-linked-resources.json
+++
b/integration-tests/hop_server/metadata/pipeline-run-configuration/remote-no-linked-resources.json
@@ -4,13 +4,16 @@
"export_resources": false,
"resources_target_folder": "",
"resources_source_folder": "",
+ "run_config": "local",
"server_poll_interval": "",
- "run_config": "",
"hop_server": "testserver",
"server_poll_delay": ""
}
},
+ "defaultSelection": false,
"configurationVariables": [],
"name": "remote-no-linked-resources",
- "description": ""
+ "description": "",
+ "dataProfile": "",
+ "executionInfoLocationName": ""
}
\ No newline at end of file
