This is an automated email from the ASF dual-hosted git repository. Amar3tto pushed a commit to branch icerbergio-benchmark in repository https://gitbox.apache.org/repos/asf/beam.git
commit 22b2f385a8a9a82ad5c2aa92217068ce01c8500a Author: Vitaly Terentyev <[email protected]> AuthorDate: Tue Jul 7 15:38:43 2026 +0400 Refactor LoadTestBase --- .../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +- it/build.gradle | 6 +- it/common/build.gradle | 10 ++ .../common}/bigquery/BigQueryResourceManager.java | 2 +- .../bigquery/BigQueryResourceManagerException.java | 2 +- .../bigquery/BigQueryResourceManagerUtils.java | 2 +- .../beam/it/common/bigquery}/package-info.java | 4 +- .../common}/dataflow/AbstractPipelineLauncher.java | 2 +- .../common}/dataflow/DefaultPipelineLauncher.java | 4 +- .../beam/it/common/dataflow}/package-info.java | 4 +- .../it/common}/monitoring/MonitoringClient.java | 2 +- .../beam/it/common}/monitoring/package-info.java | 2 +- .../org/apache/beam/it/common}/IOLoadTestBase.java | 7 +- .../apache/beam/it/common}/IOStressTestBase.java | 2 +- .../org/apache/beam/it/common}/LoadTestBase.java | 13 +- it/google-cloud-platform/build.gradle | 3 + .../gcp/bigquery/conditions/BigQueryRowsCheck.java | 2 +- .../it/gcp/dataflow/ClassicTemplateClient.java | 2 + .../beam/it/gcp/dataflow/FlexTemplateClient.java | 2 + .../apache/beam/it/gcp/bigquery/BigQueryIOLT.java | 3 +- .../apache/beam/it/gcp/bigquery/BigQueryIOST.java | 3 +- .../gcp/bigquery/BigQueryResourceManagerTest.java | 2 + .../bigquery/BigQueryResourceManagerUtilsTest.java | 4 +- .../beam/it/gcp/bigquery/BigQueryStreamingLT.java | 2 +- .../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 2 +- .../apache/beam/it/gcp/bigtable/BigTableIOST.java | 2 +- .../gcp/dataflow/AbstractPipelineLauncherTest.java | 1 + .../it/gcp/dataflow/ClassicTemplateClientTest.java | 9 +- .../gcp/dataflow/DefaultPipelineLauncherTest.java | 1 + .../it/gcp/dataflow/FlexTemplateClientTest.java | 8 +- .../org/apache/beam/it/gcp/pubsub/PubSubIOST.java | 2 +- .../org/apache/beam/it/gcp/pubsub/PubsubIOLT.java | 2 +- .../apache/beam/it/gcp/spanner/SpannerIOLT.java | 2 +- .../apache/beam/it/gcp/spanner/SpannerIOST.java | 2 +- .../apache/beam/it/gcp/storage/FileBasedIOLT.java | 2 +- it/iceberg/build.gradle | 172 +++++++++++++++++++++ it/iceberg/src/main/resources/test-artifact.json | 1 + .../org/apache/beam/it/iceberg/IcebergIOLT.java | 7 + settings.gradle.kts | 1 + 39 files changed, 255 insertions(+), 46 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml index db43441f2cd..0bc0a37ea4e 100644 --- a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml +++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Java_IO_Performance_Tests"] job_phrase: ["Run Java PostCommit IO Performance Tests"] - test_case: ["GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"] + test_case: ["IcebergPerformanceTest"] steps: - uses: actions/checkout@v7 - name: Setup repository diff --git a/it/build.gradle b/it/build.gradle index 42a9ad9f4ee..c550e26d305 100644 --- a/it/build.gradle +++ b/it/build.gradle @@ -34,4 +34,8 @@ tasks.register('BigTablePerformanceTest') { tasks.register('BigQueryStorageApiStreamingPerformanceTest') { dependsOn(":it:google-cloud-platform:BigQueryStorageApiStreamingPerformanceTest") -} \ No newline at end of file +} + +tasks.register('IcebergPerformanceTest') { + dependsOn(":it:iceberg:IcebergPerformanceTest") +} diff --git a/it/common/build.gradle b/it/common/build.gradle index c92ec551cb2..7b8559aa6e1 100644 --- a/it/common/build.gradle +++ b/it/common/build.gradle @@ -28,9 +28,16 @@ ext.summary = "Code used by all integration test utilities." dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(path: ":runners:google-cloud-dataflow-java") + implementation project(path: ":sdks:java:testing:test-utils") + implementation 'com.google.cloud:google-cloud-bigquery' + implementation 'com.google.cloud:google-cloud-monitoring' + provided 'com.google.api.grpc:proto-google-cloud-monitoring-v3' + implementation library.java.gax implementation library.java.google_api_services_dataflow implementation library.java.google_auth_library_credentials implementation library.java.google_auth_library_oauth2_http + implementation library.java.google_api_services_bigquery implementation library.java.vendored_guava_32_1_2_jre implementation library.java.slf4j_api implementation library.java.commons_lang3 @@ -38,6 +45,8 @@ dependencies { implementation library.java.google_code_gson implementation library.java.google_http_client implementation library.java.guava + implementation library.java.protobuf_java_util + implementation library.java.protobuf_java // TODO: excluding Guava until Truth updates it to >32.1.x testImplementation(library.java.truth) { @@ -46,4 +55,5 @@ dependencies { testImplementation library.java.junit testImplementation library.java.mockito_inline testRuntimeOnly library.java.slf4j_simple + testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") } diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManager.java similarity index 99% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java rename to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManager.java index 5ca6a8c4655..728a47b224b 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManager.java +++ b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManager.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.bigquery; +package org.apache.beam.it.common.bigquery; import com.google.api.gax.paging.Page; import com.google.auth.Credentials; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerException.java similarity index 96% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java rename to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerException.java index 61ef27e4526..ce95dc89155 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerException.java +++ b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerException.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.bigquery; +package org.apache.beam.it.common.bigquery; /** Custom exception for {@link BigQueryResourceManager} implementations. */ public class BigQueryResourceManagerException extends RuntimeException { diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerUtils.java similarity index 98% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java rename to it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerUtils.java index f2b6849caa6..56cb8201388 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtils.java +++ b/it/common/src/main/java/org/apache/beam/it/common/bigquery/BigQueryResourceManagerUtils.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.bigquery; +package org.apache.beam.it.common.bigquery; import static org.apache.beam.it.common.utils.ResourceManagerUtils.generateResourceId; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/bigquery/package-info.java similarity index 86% copy from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java copy to it/common/src/main/java/org/apache/beam/it/common/bigquery/package-info.java index 9cd79ea9cf1..f2d5978fb4e 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java +++ b/it/common/src/main/java/org/apache/beam/it/common/bigquery/package-info.java @@ -16,5 +16,5 @@ * limitations under the License. */ -/** Package for querying metrics from cloud monitoring. */ -package org.apache.beam.it.gcp.monitoring; +/** Package for managing BigQuery common resources within integration tests. */ +package org.apache.beam.it.common.bigquery; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java b/it/common/src/main/java/org/apache/beam/it/common/dataflow/AbstractPipelineLauncher.java similarity index 99% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java rename to it/common/src/main/java/org/apache/beam/it/common/dataflow/AbstractPipelineLauncher.java index 9d00fa67f14..a6efaeca5af 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncher.java +++ b/it/common/src/main/java/org/apache/beam/it/common/dataflow/AbstractPipelineLauncher.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.dataflow; +package org.apache.beam.it.common.dataflow; import static org.apache.beam.it.common.PipelineLauncher.JobState.ACTIVE_STATES; import static org.apache.beam.it.common.PipelineLauncher.JobState.FAILED; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java b/it/common/src/main/java/org/apache/beam/it/common/dataflow/DefaultPipelineLauncher.java similarity index 99% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java rename to it/common/src/main/java/org/apache/beam/it/common/dataflow/DefaultPipelineLauncher.java index 11a09c4ba74..a872868c12b 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java +++ b/it/common/src/main/java/org/apache/beam/it/common/dataflow/DefaultPipelineLauncher.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.dataflow; +package org.apache.beam.it.common.dataflow; import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; import static org.apache.beam.sdk.testing.TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS; @@ -41,7 +41,7 @@ import java.util.regex.Pattern; import java.util.stream.StreamSupport; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.utils.PipelineUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.runners.dataflow.DataflowPipelineJob; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.metrics.DistributionResult; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/dataflow/package-info.java similarity index 88% copy from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java copy to it/common/src/main/java/org/apache/beam/it/common/dataflow/package-info.java index 9cd79ea9cf1..eb10897523b 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java +++ b/it/common/src/main/java/org/apache/beam/it/common/dataflow/package-info.java @@ -16,5 +16,5 @@ * limitations under the License. */ -/** Package for querying metrics from cloud monitoring. */ -package org.apache.beam.it.gcp.monitoring; +/** Package for managing Dataflow jobs from integration tests. */ +package org.apache.beam.it.common.dataflow; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java b/it/common/src/main/java/org/apache/beam/it/common/monitoring/MonitoringClient.java similarity index 99% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java rename to it/common/src/main/java/org/apache/beam/it/common/monitoring/MonitoringClient.java index 66550b5a507..6b447d84512 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/MonitoringClient.java +++ b/it/common/src/main/java/org/apache/beam/it/common/monitoring/MonitoringClient.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp.monitoring; +package org.apache.beam.it.common.monitoring; import com.google.api.gax.core.CredentialsProvider; import com.google.cloud.monitoring.v3.MetricServiceClient; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java b/it/common/src/main/java/org/apache/beam/it/common/monitoring/package-info.java similarity index 94% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java rename to it/common/src/main/java/org/apache/beam/it/common/monitoring/package-info.java index 9cd79ea9cf1..731bbc3a6b2 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/monitoring/package-info.java +++ b/it/common/src/main/java/org/apache/beam/it/common/monitoring/package-info.java @@ -17,4 +17,4 @@ */ /** Package for querying metrics from cloud monitoring. */ -package org.apache.beam.it.gcp.monitoring; +package org.apache.beam.it.common.monitoring; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java b/it/common/src/test/java/org/apache/beam/it/common/IOLoadTestBase.java similarity index 96% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java rename to it/common/src/test/java/org/apache/beam/it/common/IOLoadTestBase.java index 14770a42973..fb9feb6663b 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOLoadTestBase.java +++ b/it/common/src/test/java/org/apache/beam/it/common/IOLoadTestBase.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp; +package org.apache.beam.it.common; import com.google.cloud.Timestamp; import java.io.IOException; @@ -23,9 +23,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.UUID; -import org.apache.beam.it.common.PipelineLauncher; -import org.apache.beam.it.common.TestProperties; -import org.apache.beam.it.gcp.dataflow.DefaultPipelineLauncher; + +import org.apache.beam.it.common.dataflow.DefaultPipelineLauncher; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.testutils.NamedTestResult; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOStressTestBase.java b/it/common/src/test/java/org/apache/beam/it/common/IOStressTestBase.java similarity index 99% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOStressTestBase.java rename to it/common/src/test/java/org/apache/beam/it/common/IOStressTestBase.java index 5c2fb74cd2f..a15e98d65d8 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOStressTestBase.java +++ b/it/common/src/test/java/org/apache/beam/it/common/IOStressTestBase.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp; +package org.apache.beam.it.common; import java.io.Serializable; import java.time.Duration; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java b/it/common/src/test/java/org/apache/beam/it/common/LoadTestBase.java similarity index 98% rename from it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java rename to it/common/src/test/java/org/apache/beam/it/common/LoadTestBase.java index cd9ef52ed83..7d7adbbae0a 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/LoadTestBase.java +++ b/it/common/src/test/java/org/apache/beam/it/common/LoadTestBase.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.beam.it.gcp; +package org.apache.beam.it.common; import static org.apache.beam.it.common.logging.LogStrings.formatForLogging; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.RUNNER_V2; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.RUNNER_V2; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; @@ -40,12 +40,11 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.regex.Pattern; -import org.apache.beam.it.common.PipelineLauncher; + import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; -import org.apache.beam.it.common.PipelineOperator; -import org.apache.beam.it.common.TestProperties; -import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; -import org.apache.beam.it.gcp.monitoring.MonitoringClient; +import org.apache.beam.it.common.bigquery.BigQueryResourceManager; +import org.apache.beam.it.common.dataflow.AbstractPipelineLauncher; +import org.apache.beam.it.common.monitoring.MonitoringClient; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle index 3a46f2b94d8..dd8376e3569 100644 --- a/it/google-cloud-platform/build.gradle +++ b/it/google-cloud-platform/build.gradle @@ -27,6 +27,8 @@ applyJavaNature( description = "Apache Beam :: IT :: Google Cloud Platform" ext.summary = "Integration test utilities for Google Cloud Platform." +evaluationDependsOn(":it:common") + dependencies { implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom) implementation project(path: ":sdks:java:core", configuration: "shadow") @@ -84,6 +86,7 @@ dependencies { implementation 'com.google.cloud:google-cloud-secretmanager' provided 'com.google.api.grpc:proto-google-cloud-secretmanager-v1' + testImplementation project(path: ":it:common", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:io:google-cloud-platform") testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:io:synthetic") diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java index 28626f7edd1..57b7d5ac99c 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/bigquery/conditions/BigQueryRowsCheck.java @@ -20,7 +20,7 @@ package org.apache.beam.it.gcp.bigquery.conditions; import com.google.auto.value.AutoValue; import com.google.cloud.bigquery.TableId; import org.apache.beam.it.conditions.ConditionCheck; -import org.apache.beam.it.gcp.bigquery.BigQueryResourceManager; +import org.apache.beam.it.common.bigquery.BigQueryResourceManager; import org.checkerframework.checker.nullness.qual.Nullable; /** ConditionCheck to validate if BigQuery has received a certain number of rows. */ diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java index 00aa4003817..296f7092cfa 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClient.java @@ -30,6 +30,8 @@ import com.google.auth.Credentials; import com.google.auth.http.HttpCredentialsAdapter; import dev.failsafe.Failsafe; import java.io.IOException; + +import org.apache.beam.it.common.dataflow.AbstractPipelineLauncher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java index 0a9731b13c3..3e7270dbd34 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClient.java @@ -32,6 +32,8 @@ import com.google.auth.Credentials; import com.google.auth.http.HttpCredentialsAdapter; import dev.failsafe.Failsafe; import java.io.IOException; + +import org.apache.beam.it.common.dataflow.AbstractPipelineLauncher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java index f123ffa5954..9a4af4082e2 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOLT.java @@ -43,8 +43,9 @@ import org.apache.avro.generic.GenericRecord; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.bigquery.BigQueryResourceManager; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.gcp.bigquery.AvroWriteRequest; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java index 14c83e48308..bbe4df61191 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java @@ -41,8 +41,9 @@ import org.apache.avro.generic.GenericRecord; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; +import org.apache.beam.it.common.bigquery.BigQueryResourceManager; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOStressTestBase; +import org.apache.beam.it.common.IOStressTestBase; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.gcp.bigquery.AvroWriteRequest; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java index 53987dbe267..3a633f00c6f 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerTest.java @@ -41,6 +41,8 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TimePartitioning; +import org.apache.beam.it.common.bigquery.BigQueryResourceManager; +import org.apache.beam.it.common.bigquery.BigQueryResourceManagerException; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; import org.junit.Before; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java index 432b7ef7362..7248fed0e87 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryResourceManagerUtilsTest.java @@ -17,10 +17,12 @@ */ package org.apache.beam.it.gcp.bigquery; -import static org.apache.beam.it.gcp.bigquery.BigQueryResourceManagerUtils.checkValidTableId; +import static org.apache.beam.it.common.bigquery.BigQueryResourceManagerUtils.checkValidTableId; import static org.junit.Assert.assertThrows; import java.util.Arrays; + +import org.apache.beam.it.common.bigquery.BigQueryResourceManagerUtils; import org.junit.Test; /** Unit tests for {@link BigQueryResourceManagerUtils}. */ diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java index 6e511bd8e5c..7bf759b42f9 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java @@ -41,7 +41,7 @@ import java.util.stream.Collectors; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java index 410c992fe2d..aea26f2f0a4 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOLT.java @@ -35,7 +35,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO; import org.apache.beam.sdk.testing.TestPipeline; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOST.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOST.java index 052a0839adf..85950772bce 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOST.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigtable/BigTableIOST.java @@ -38,7 +38,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOStressTestBase; +import org.apache.beam.it.common.IOStressTestBase; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java index f2c33d70116..df1c5cea640 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/AbstractPipelineLauncherTest.java @@ -34,6 +34,7 @@ import dev.failsafe.FailsafeException; import java.io.IOException; import java.net.SocketTimeoutException; import org.apache.beam.it.common.PipelineLauncher.JobState; +import org.apache.beam.it.common.dataflow.AbstractPipelineLauncher; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java index 88c35589f2b..a50c0951005 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/ClassicTemplateClientTest.java @@ -18,10 +18,10 @@ package org.apache.beam.it.gcp.dataflow; import static com.google.common.truth.Truth.assertThat; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -49,6 +49,7 @@ import java.util.Collections; import org.apache.beam.it.common.PipelineLauncher.JobState; import org.apache.beam.it.common.PipelineLauncher.LaunchConfig; import org.apache.beam.it.common.PipelineLauncher.LaunchInfo; +import org.apache.beam.it.common.dataflow.AbstractPipelineLauncher; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java index b6c0f8cdc58..52ff5c0a230 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncherTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.time.Instant; import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.dataflow.DefaultPipelineLauncher; import org.apache.beam.it.gcp.IOLoadTestBase; import org.apache.beam.it.gcp.IOLoadTestBase.PipelineMetricsType; import org.apache.beam.sdk.io.GenerateSequence; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java index 06f44437414..b94d52e45af 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/dataflow/FlexTemplateClientTest.java @@ -18,10 +18,10 @@ package org.apache.beam.it.gcp.dataflow; import static com.google.common.truth.Truth.assertThat; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; -import static org.apache.beam.it.gcp.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.LEGACY_RUNNER; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_JOB_ID; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_JOB_TYPE; +import static org.apache.beam.it.common.dataflow.AbstractPipelineLauncher.PARAM_RUNNER; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubSubIOST.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubSubIOST.java index cb96db40b74..74bd994055a 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubSubIOST.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubSubIOST.java @@ -38,7 +38,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOStressTestBase; +import org.apache.beam.it.common.IOStressTestBase; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Primitive; import org.apache.beam.sdk.io.Read; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubIOLT.java index 477c79dd3bc..a9c7b63e64c 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubIOLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/pubsub/PubsubIOLT.java @@ -38,7 +38,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.runners.direct.DirectOptions; import org.apache.beam.sdk.extensions.protobuf.Proto3SchemaMessages.Primitive; import org.apache.beam.sdk.io.Read; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOLT.java index 8f3de1b58f4..a29cf32eae1 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOLT.java @@ -38,7 +38,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.gcp.spanner.SpannerIO; import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOST.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOST.java index 76312b3e10f..8ce34d06de8 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOST.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/spanner/SpannerIOST.java @@ -38,7 +38,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOStressTestBase; +import org.apache.beam.it.common.IOStressTestBase; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.gcp.spanner.SpannerIO; diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java index 58ba88d900e..febcfc01f3f 100644 --- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java +++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/storage/FileBasedIOLT.java @@ -34,7 +34,7 @@ import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.TestProperties; import org.apache.beam.it.common.utils.ResourceManagerUtils; -import org.apache.beam.it.gcp.IOLoadTestBase; +import org.apache.beam.it.common.IOLoadTestBase; import org.apache.beam.sdk.io.Compression; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.TextIO; diff --git a/it/iceberg/build.gradle b/it/iceberg/build.gradle new file mode 100644 index 00000000000..1a61f86367b --- /dev/null +++ b/it/iceberg/build.gradle @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.beam.gradle.IoPerformanceTestUtilities + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it.iceberg', + exportJavadoc: false, + // iceberg ended support for Java 8 in 1.7.0 + requireJavaVersion: JavaVersion.VERSION_11, +) + +description = "Apache Beam :: IT :: Iceberg" +ext.summary = "Integration test utilities for Iceberg." + +def iceberg_version = "1.10.0" +def parquet_version = "1.16.0" +def orc_version = "1.9.6" +def hive_version = "3.1.3" + +def hadoopVersions = [ + "336": "3.3.6", +] + +hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")} + +evaluationDependsOn(":it:common") + +dependencies { + implementation library.java.vendored_guava_32_1_2_jre + implementation project(path: ":sdks:java:core", configuration: "shadow") + implementation project(path: ":model:pipeline", configuration: "shadow") + implementation library.java.avro + implementation library.java.slf4j_api + implementation library.java.joda_time + implementation "org.apache.parquet:parquet-column:$parquet_version" + implementation "org.apache.parquet:parquet-hadoop:$parquet_version" + implementation "org.apache.parquet:parquet-common:$parquet_version" + implementation project(":sdks:java:io:parquet") + implementation "org.apache.orc:orc-core:$orc_version" + implementation "org.apache.iceberg:iceberg-core:$iceberg_version" + implementation "org.apache.iceberg:iceberg-api:$iceberg_version" + implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version" + implementation "org.apache.iceberg:iceberg-orc:$iceberg_version" + implementation "org.apache.iceberg:iceberg-data:$iceberg_version" + implementation "org.apache.hadoop:hadoop-common:3.3.6" + + provided "org.immutables:value:2.8.8" + permitUnusedDeclared "org.immutables:value:2.8.8" + implementation library.java.vendored_calcite_1_40_0 + runtimeOnly "org.apache.iceberg:iceberg-gcp:$iceberg_version" + runtimeOnly "org.apache.iceberg:iceberg-aws:$iceberg_version" + runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:$iceberg_version" + runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version" + runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version" + runtimeOnly library.java.bigdataoss_gcs_connector + runtimeOnly library.java.bigdataoss_util_hadoop + runtimeOnly library.java.hadoop_client + + testImplementation project(":sdks:java:managed") + testImplementation library.java.bigdataoss_gcsio + testImplementation library.java.bigdataoss_util_hadoop + testImplementation "org.apache.parquet:parquet-avro:$parquet_version" + testImplementation "org.apache.iceberg:iceberg-data:$iceberg_version" + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation library.java.junit + testImplementation library.java.hamcrest + testImplementation project(path: ":sdks:java:extensions:avro") + testImplementation 'org.awaitility:awaitility:4.2.0' + + // Hive catalog test dependencies + testImplementation project(path: ":sdks:java:io:iceberg:hive") + testImplementation "org.apache.iceberg:iceberg-common:$iceberg_version" + testImplementation ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version") + testImplementation ("org.apache.hive:hive-metastore:$hive_version") + testImplementation "org.assertj:assertj-core:3.11.1" + testRuntimeOnly ("org.apache.hive.hcatalog:hive-hcatalog-core:$hive_version") { + exclude group: "org.apache.hive", module: "hive-exec" + exclude group: "org.apache.parquet", module: "parquet-hadoop-bundle" + } + + // BigQueryMetastore catalog dep + testImplementation project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") + testImplementation project(":sdks:java:io:google-cloud-platform") + testImplementation library.java.google_api_services_bigquery + testImplementation 'com.google.cloud:google-cloud-storage' + + testImplementation library.java.google_auth_library_oauth2_http + testRuntimeOnly library.java.slf4j_jdk14 + testImplementation project(path: ":runners:direct-java", configuration: "shadow") + testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java") + testRuntimeOnly project(path: ":sdks:java:harness") + hadoopVersions.each {kv -> + "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value" + "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-minicluster:$kv.value" + "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-hdfs-client:$kv.value" + "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value" + } + + implementation project(path: ":runners:google-cloud-dataflow-java") + implementation project(path: ":it:common") + implementation project(path: ":it:conditions") + implementation project(path: ":it:truthmatchers") + implementation project(path: ":sdks:java:testing:test-utils") + implementation library.java.failsafe + implementation library.java.google_api_services_dataflow + implementation(library.java.truth) { + exclude group: 'com.google.guava', module: 'guava' + } + implementation library.java.jackson_core + implementation library.java.jackson_databind + implementation 'org.apache.commons:commons-lang3:3.9' + implementation library.java.google_api_client + implementation library.java.google_http_client + implementation library.java.guava + implementation library.java.protobuf_java_util + implementation library.java.protobuf_java + implementation library.java.threetenbp + + testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") + testImplementation project(path: ":sdks:java:io:synthetic") + testImplementation project(path: ":it:common", configuration: "testRuntimeMigration") + testImplementation library.java.mockito_inline + testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration") + testRuntimeOnly library.java.slf4j_simple +} + +configurations.all { + // iceberg-core needs avro:1.12.0 + resolutionStrategy.force 'org.apache.avro:avro:1.12.0' + // TODO(https://github.com/apache/beam/issues/38515): + // Remove below pins when parquet-hadoop upgrades to hadoop-common:3.4.2 + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' +} + +hadoopVersions.each {kv -> + configurations."hadoopVersion$kv.key" { + resolutionStrategy { + force "org.apache.hadoop:hadoop-client:$kv.value" + force "org.apache.hadoop:hadoop-common:$kv.value" + force "org.apache.hadoop:hadoop-mapreduce-client-core:$kv.value" + force "org.apache.hadoop:hadoop-minicluster:$kv.value" + force "org.apache.hadoop:hadoop-hdfs:$kv.value" + force "org.apache.hadoop:hadoop-hdfs-client:$kv.value" + } + } +} + +tasks.register( + "IcebergPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'iceberg', 'IcebergIOLT', + ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp'] + + System.properties +) diff --git a/it/iceberg/src/main/resources/test-artifact.json b/it/iceberg/src/main/resources/test-artifact.json new file mode 100644 index 00000000000..551c80d14a6 --- /dev/null +++ b/it/iceberg/src/main/resources/test-artifact.json @@ -0,0 +1 @@ +["This is a test artifact."] \ No newline at end of file diff --git a/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java b/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java new file mode 100644 index 00000000000..d2e342ccecb --- /dev/null +++ b/it/iceberg/src/test/java/org/apache/beam/it/iceberg/IcebergIOLT.java @@ -0,0 +1,7 @@ +package org.apache.beam.it.iceberg; + +import org.apache.beam.it.common.IOLoadTestBase; + +public class IcebergIOLT extends IOLoadTestBase { + +} diff --git a/settings.gradle.kts b/settings.gradle.kts index d9dbbf9021e..5ee8f8a4d39 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -122,6 +122,7 @@ include(":it:conditions") include(":it:datadog") include(":it:elasticsearch") include(":it:google-cloud-platform") +include(":it:iceberg") include(":it:jdbc") include(":it:kafka") include(":it:testcontainers")
