This is an automated email from the ASF dual-hosted git repository.
hansva pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
from 12e809862c Merge pull request #3331 from sramazzina/GET-FILENAMES-FIX
new d359ad98a0 First draft of Beam Hive Metastore input, #3274
new 00089b816b Adding dependencies and some code changes, #3274
new 8edecf20a1 add environment for testing
new bc2a0d3bfc pass rows as a single string field, Dockerfile.hadoop chown
update. #3274
new 4907ef29cf Downgrade hive to work on Spark 3.4.1, #3274
new 37023ac617 avoid NPE, #3274
new c9f529bb5a Add documentation and some cleanup, #3274
new df5ce528fc Add support for Azure Blob Storage to Spark Runner, #3338
new 34879d4361 Merge pull request #3313 from hansva/3274
The 6257 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../plugins/databases/hive-assemblies/pom.xml | 29 ++-
assemblies/plugins/engines/beam/pom.xml | 86 +++++++++
.../plugins/engines/beam/src/assembly/assembly.xml | 39 ++++
.../integration-tests/integration-tests-spark.yaml | 97 +++++++++-
.../spark/{scripts/master.sh => Dockerfile.hadoop} | 31 +---
docker/integration-tests/spark/Dockerfile.master | 4 +-
docker/integration-tests/spark/Dockerfile.worker | 4 +-
.../spark/{scripts/master.sh => build.sh} | 35 ++--
docker/integration-tests/spark/config | 41 +++++
docker/integration-tests/spark/hive-site.xml | 70 ++++++++
.../spark/{scripts/master.sh => log4j.properties} | 27 +--
docker/integration-tests/spark/scripts/master.sh | 4 +-
.../pipeline/transforms/beamhivecataloginput.adoc | 58 ++++++
integration-tests/scripts/run-tests-docker.sh | 8 +-
integration-tests/spark/dev-env-config.json | 2 +-
integration-tests/spark/disabled.txt | 18 ++
plugins/databases/hive/pom.xml | 6 +-
plugins/engines/beam/pom.xml | 154 ++++++++++++++++
.../transform/BeamHiveMetastoreInputTransform.java | 127 +++++++++++++
.../spark/BeamSparkPipelineRunConfiguration.java | 72 +++++++-
.../hivecatalog/BeamHiveCatalogInput.java | 58 ++++++
.../hivecatalog/BeamHiveCatalogInputData.java | 23 +++
.../hivecatalog/BeamHiveCatalogInputDialog.java | 197 +++++++++++++++++++++
.../hivecatalog/BeamHiveCatalogInputMeta.java | 150 ++++++++++++++++
.../spark/messages/messages_en_US.properties | 6 +
.../messages/messages_de_DE.properties | 0
.../hivecatalog/messages/messages_en_US.properties | 28 +--
.../messages/messages_es_AR.properties | 0
.../messages/messages_es_ES.properties | 0
.../messages/messages_fr_FR.properties | 0
.../messages/messages_it_IT.properties | 0
.../messages/messages_ja_JP.properties | 0
.../messages/messages_ko_KR.properties | 0
.../messages/messages_nl_NL.properties | 0
.../messages/messages_no_NO.properties | 0
.../messages/messages_pl_PL.properties | 0
.../messages/messages_pt_BR.properties | 0
.../messages/messages_pt_PT.properties | 0
.../hivecatalog/messages/messages_zh_CN.properties | 22 +--
pom.xml | 3 +-
40 files changed, 1274 insertions(+), 125 deletions(-)
copy docker/integration-tests/spark/{scripts/master.sh => Dockerfile.hadoop}
(55%)
copy docker/integration-tests/spark/{scripts/master.sh => build.sh} (55%)
mode change 100644 => 100755
create mode 100644 docker/integration-tests/spark/config
create mode 100644 docker/integration-tests/spark/hive-site.xml
copy docker/integration-tests/spark/{scripts/master.sh => log4j.properties}
(55%)
create mode 100644
docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/beamhivecataloginput.adoc
create mode 100644 integration-tests/spark/disabled.txt
create mode 100644
plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/BeamHiveMetastoreInputTransform.java
create mode 100644
plugins/engines/beam/src/main/java/org/apache/hop/beam/transforms/hivecatalog/BeamHiveCatalogInput.java
create mode 100644
plugins/engines/beam/src/main/java/org/apache/hop/beam/transforms/hivecatalog/BeamHiveCatalogInputData.java
create mode 100644
plugins/engines/beam/src/main/java/org/apache/hop/beam/transforms/hivecatalog/BeamHiveCatalogInputDialog.java
create mode 100644
plugins/engines/beam/src/main/java/org/apache/hop/beam/transforms/hivecatalog/BeamHiveCatalogInputMeta.java
copy {core/src/test/resources/org/apache/hop/foo =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_de_DE.properties
(100%)
copy docker/integration-tests/spark/scripts/master.sh =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog/messages/messages_en_US.properties
(55%)
copy {core/src/test/resources/org/apache/hop/foo =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_es_AR.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/util =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_es_ES.properties
(100%)
copy {core/src/test/resources/org/apache/hop/foo =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_fr_FR.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/util =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_it_IT.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/util =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_ja_JP.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/util =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_ko_KR.properties
(100%)
copy {core/src/test/resources/org/apache/hop/foo =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_nl_NL.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/plugins =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_no_NO.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/logging =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_pl_PL.properties
(100%)
copy {core/src/test/resources/org/apache/hop/foo =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_pt_BR.properties
(100%)
copy {core/src/main/resources/org/apache/hop/core/logging =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog}/messages/messages_pt_PT.properties
(100%)
copy docker/integration-tests/spark/scripts/master.sh =>
plugins/engines/beam/src/main/resources/org/apache/hop/beam/transforms/hivecatalog/messages/messages_zh_CN.properties
(55%)
