This is an automated email from the ASF dual-hosted git repository. zhangyue19921010 pushed a commit to branch partition-bucket-index in repository https://gitbox.apache.org/repos/asf/hudi.git
commit f3a6f3620967aebca3fe458428f9a51b046c8de1 Author: Alex R <[email protected]> AuthorDate: Mon Mar 17 11:23:00 2025 -0700 [HUDI-9168] Refactor hudi-client-common to not import hudi-aws (#12969) --- hudi-aws/pom.xml | 5 ++ .../cloudwatch/CloudWatchMetricsReporter.java | 3 +- .../cloudwatch/CloudWatchReporter.java | 11 ++-- .../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 2 +- ...amoDBBasedImplicitPartitionKeyLockProvider.java | 2 +- .../java/org/apache/hudi/aws/utils/S3Utils.java | 27 ---------- .../cloudwatch/TestCloudWatchMetricsReporter.java | 61 +++++++++++++++++++--- .../cloudwatch/TestCloudWatchReporter.java | 10 ++-- .../org/apache/hudi/aws/utils/TestS3Utils.java | 59 --------------------- hudi-client/hudi-client-common/pom.xml | 12 ++--- ...ZookeeperBasedImplicitBasePathLockProvider.java | 2 +- .../java/org/apache/hudi/common/fs/FSUtils.java | 5 ++ .../hudi/metrics/MetricsReporterFactory.java | 2 +- .../hudi/metrics/TestMetricsReporterFactory.java | 38 ++++++++++++-- .../org/apache/hudi/common/fs/TestFSUtils.java | 32 ++++++++++++ 15 files changed, 153 insertions(+), 118 deletions(-) diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml index 1b2967e5b2c..7ee4a0a7c4a 100644 --- a/hudi-aws/pom.xml +++ b/hudi-aws/pom.xml @@ -42,6 +42,11 @@ </dependency> <!-- Hoodie --> + <dependency> + <groupId>org.apache.hudi</groupId> + <artifactId>hudi-client-common</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.hudi</groupId> <artifactId>hudi-common</artifactId> diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java similarity index 96% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java rename to hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java index 68e4951f74f..03f71410b36 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package org.apache.hudi.metrics.cloudwatch; +package org.apache.hudi.aws.metrics.cloudwatch; -import org.apache.hudi.aws.cloudwatch.CloudWatchReporter; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.metrics.MetricsReporter; diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java similarity index 97% rename from hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java rename to hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java index bf5ae9f91f3..037bab52db1 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.aws.cloudwatch; +package org.apache.hudi.aws.metrics.cloudwatch; import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; import org.apache.hudi.common.util.Option; @@ -196,17 +196,17 @@ public class CloudWatchReporter extends ScheduledReporter { for (Map.Entry<String, Histogram> entry : histograms.entrySet()) { processCounter(entry.getKey(), entry.getValue(), timestampMilliSec, metricsData); - //TODO: Publish other Histogram metrics to cloud watch + // TODO: Publish other Histogram metrics to cloud watch } for (Map.Entry<String, Meter> entry : meters.entrySet()) { processCounter(entry.getKey(), entry.getValue(), timestampMilliSec, metricsData); - //TODO: Publish other Meter metrics to cloud watch + // TODO: Publish other Meter metrics to cloud watch } for (Map.Entry<String, Timer> entry : timers.entrySet()) { processCounter(entry.getKey(), entry.getValue(), timestampMilliSec, metricsData); - //TODO: Publish other Timer metrics to cloud watch + // TODO: Publish other Timer metrics to cloud watch } report(metricsData); @@ -236,6 +236,9 @@ public class CloudWatchReporter extends ScheduledReporter { } catch (final Exception ex) { LOG.error("Error reporting metrics to CloudWatch. The data in this CloudWatch request " + "may have been discarded, and not made it to CloudWatch.", ex); + if (ex instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } } } } diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index abda5a9d988..ce14e87ea23 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -101,7 +101,7 @@ import java.util.concurrent.Future; import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.apache.hudi.aws.utils.S3Utils.s3aToS3; +import static org.apache.hudi.common.fs.FSUtils.s3aToS3; import static org.apache.hudi.common.util.MapUtils.containsAll; import static org.apache.hudi.common.util.MapUtils.isNullOrEmpty; import static org.apache.hudi.config.GlueCatalogSyncClientConfig.ALL_PARTITIONS_READ_PARALLELISM; diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java index d9a131cd096..bcb0f4571c0 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java @@ -31,7 +31,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import javax.annotation.concurrent.NotThreadSafe; -import static org.apache.hudi.aws.utils.S3Utils.s3aToS3; +import static org.apache.hudi.common.fs.FSUtils.s3aToS3; /** * A DynamoDB based lock. diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java b/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java deleted file mode 100644 index bfb208ee150..00000000000 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hudi.aws.utils; - -public final class S3Utils { - - public static String s3aToS3(String s3aUrl) { - return s3aUrl.replaceFirst("(?i)^s3a://", "s3://"); - } -} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java similarity index 53% rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java rename to hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java index 4b1aaffbf86..15cfb06e623 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.metrics.cloudwatch; +package org.apache.hudi.aws.metrics.cloudwatch; -import org.apache.hudi.aws.cloudwatch.CloudWatchReporter; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.metrics.MetricsReporterFactory; import org.apache.hudi.metrics.MetricsReporterType; @@ -27,19 +28,26 @@ import com.codahale.metrics.MetricRegistry; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class TestCloudWatchMetricsReporter { +class TestCloudWatchMetricsReporter { + + @Mock + private HoodieWriteConfig writeConfig; @Mock private HoodieMetricsConfig metricsConfig; @@ -51,7 +59,7 @@ public class TestCloudWatchMetricsReporter { private CloudWatchReporter reporter; @Test - public void testReporter() { + void testReporter() { when(metricsConfig.getCloudWatchReportPeriodSeconds()).thenReturn(30); CloudWatchMetricsReporter metricsReporter = new CloudWatchMetricsReporter(metricsConfig, registry, reporter); @@ -66,7 +74,23 @@ public class TestCloudWatchMetricsReporter { } @Test - public void testReporterViaReporterFactory() { + void testReporterUsingMetricsConfig() { + when(writeConfig.getMetricsConfig()).thenReturn(metricsConfig); + when(metricsConfig.getCloudWatchReportPeriodSeconds()).thenReturn(30); + CloudWatchMetricsReporter metricsReporter = new CloudWatchMetricsReporter(writeConfig, registry, reporter); + + metricsReporter.start(); + verify(reporter, times(1)).start(30, TimeUnit.SECONDS); + + metricsReporter.report(); + verify(reporter, times(1)).report(); + + metricsReporter.stop(); + verify(reporter, times(1)).stop(); + } + + @Test + void testReporterViaReporterFactory() { try { when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); // MetricsReporterFactory uses reflection to create CloudWatchMetricsReporter @@ -75,7 +99,32 @@ public class TestCloudWatchMetricsReporter { } catch (Exception e) { assertTrue(e.getCause() instanceof InvocationTargetException); assertTrue(Arrays.stream(((InvocationTargetException) e.getCause()).getTargetException().getStackTrace()).anyMatch( - ste -> ste.toString().contains("org.apache.hudi.aws.cloudwatch.CloudWatchReporter.getAmazonCloudWatchClient"))); + ste -> ste.toString().contains("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.getAmazonCloudWatchClient"))); + } + } + + @Test + void testCreateCloudWatchReporter() { + when(metricsConfig.getCloudWatchMetricPrefix()).thenReturn("prefix"); + when(metricsConfig.getCloudWatchMetricNamespace()).thenReturn("namespace"); + when(metricsConfig.getCloudWatchMaxDatumsPerRequest()).thenReturn(100); + TypedProperties props = new TypedProperties(); + when(metricsConfig.getProps()).thenReturn(props); + + CloudWatchReporter reporterMock = mock(CloudWatchReporter.class); + CloudWatchReporter.Builder builderMock = mock(CloudWatchReporter.Builder.class); + + try (MockedStatic<CloudWatchReporter> mockedStatic = Mockito.mockStatic(CloudWatchReporter.class)) { + mockedStatic.when(() -> CloudWatchReporter.forRegistry(registry)) + .thenReturn(builderMock); + when(builderMock.prefixedWith("prefix")).thenReturn(builderMock); + when(builderMock.namespace("namespace")).thenReturn(builderMock); + when(builderMock.maxDatumsPerRequest(100)).thenReturn(builderMock); + when(builderMock.build(props)).thenReturn(reporterMock); + + CloudWatchMetricsReporter metricsReporter = + new CloudWatchMetricsReporter(metricsConfig, registry); + assertNotNull(metricsReporter); } } } diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java similarity index 94% rename from hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java rename to hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java index 7061188a63c..0073f3687db 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.aws.cloudwatch; +package org.apache.hudi.aws.metrics.cloudwatch; import com.codahale.metrics.Clock; import com.codahale.metrics.Counter; @@ -49,10 +49,10 @@ import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import static org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_COUNT_TYPE_VALUE; -import static org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_GAUGE_TYPE_VALUE; -import static org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_METRIC_TYPE_KEY; -import static org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_TABLE_NAME_KEY; +import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_COUNT_TYPE_VALUE; +import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_GAUGE_TYPE_VALUE; +import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_METRIC_TYPE_KEY; +import static org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_TABLE_NAME_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java b/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java deleted file mode 100644 index a0bdd110d83..00000000000 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package org.apache.hudi.aws.utils; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class TestS3Utils { - - @Test - void testS3aToS3_AWS() { - // Test cases for AWS S3 URLs - assertEquals("s3://my-bucket/path/to/object", S3Utils.s3aToS3("s3a://my-bucket/path/to/object")); - assertEquals("s3://my-bucket", S3Utils.s3aToS3("s3a://my-bucket")); - assertEquals("s3://MY-BUCKET/PATH/TO/OBJECT", S3Utils.s3aToS3("s3a://MY-BUCKET/PATH/TO/OBJECT")); - assertEquals("s3://my-bucket/path/to/object", S3Utils.s3aToS3("S3a://my-bucket/path/to/object")); - assertEquals("s3://my-bucket/path/to/object", S3Utils.s3aToS3("s3A://my-bucket/path/to/object")); - assertEquals("s3://my-bucket/path/to/object", S3Utils.s3aToS3("S3A://my-bucket/path/to/object")); - assertEquals("s3://my-bucket/s3a://another-bucket/another/path", S3Utils.s3aToS3("s3a://my-bucket/s3a://another-bucket/another/path")); - } - - @ParameterizedTest - @ValueSource(strings = { - "gs://my-bucket/path/to/object", - "gs://my-bucket", - "gs://MY-BUCKET/PATH/TO/OBJECT", - "https://myaccount.blob.core.windows.net/mycontainer/path/to/blob";, - "https://myaccount.blob.core.windows.net/MYCONTAINER/PATH/TO/BLOB";, - "https://example.com/path/to/resource";, - "http://example.com";, - "ftp://example.com/resource";, - "", - "gs://my-bucket/path/to/s3a://object", - "gs://my-bucket s3a://my-object", - - }) - void testUriDoesNotChange(String uri) { - assertEquals(uri, S3Utils.s3aToS3(uri)); - } -} diff --git a/hudi-client/hudi-client-common/pom.xml b/hudi-client/hudi-client-common/pom.xml index 5178ae92253..67b12413407 100644 --- a/hudi-client/hudi-client-common/pom.xml +++ b/hudi-client/hudi-client-common/pom.xml @@ -48,12 +48,6 @@ <artifactId>hudi-io</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.hudi</groupId> - <artifactId>hudi-aws</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> <dependency> <groupId>org.apache.hudi</groupId> <artifactId>hudi-timeline-service</artifactId> @@ -131,6 +125,12 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hudi</groupId> + <artifactId>hudi-hadoop-mr</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.hudi</groupId> <artifactId>hudi-tests-common</artifactId> diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java index 199a9b6309d..9c4a57bc271 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java @@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.concurrent.NotThreadSafe; -import static org.apache.hudi.aws.utils.S3Utils.s3aToS3; +import static org.apache.hudi.common.fs.FSUtils.s3aToS3; /** * A zookeeper based lock. This {@link LockProvider} implementation allows to lock table operations diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java index 9edc1729dc0..0d78d03b029 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java @@ -738,6 +738,11 @@ public class FSUtils { ? new Path(null, path.toUri().getAuthority(), path.toUri().getPath()) : path; } + // Converts s3a to s3a + public static String s3aToS3(String s3aUrl) { + return s3aUrl.replaceFirst("(?i)^s3a://", "s3://"); + } + /** * Serializable function interface. * diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index 71509d7a9b7..d7d04903576 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -86,7 +86,7 @@ public class MetricsReporterFactory { reporter = new ConsoleMetricsReporter(registry); break; case CLOUDWATCH: - reporter = (MetricsReporter) ReflectionUtils.loadClass("org.apache.hudi.metrics.cloudwatch.CloudWatchMetricsReporter", + reporter = (MetricsReporter) ReflectionUtils.loadClass("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter", new Class[]{HoodieMetricsConfig.class, MetricRegistry.class}, metricsConfig, registry); break; case M3: diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java similarity index 75% rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java rename to hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java index 5afd27fbea3..d456748d976 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java +++ b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java @@ -20,6 +20,7 @@ package org.apache.hudi.metrics; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.config.metrics.HoodieMetricsConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metrics.custom.CustomizableMetricsReporter; @@ -32,17 +33,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class TestMetricsReporterFactory { +class TestMetricsReporterFactory { @Mock HoodieMetricsConfig metricsConfig; @@ -68,7 +75,27 @@ public class TestMetricsReporterFactory { } @Test - public void metricsReporterFactoryShouldReturnUserDefinedReporter() { + void metricsReporterFactoryShouldReturnCloudWatchReporter() { + when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH); + + MetricsReporter reporterMock = mock(MetricsReporter.class); + try (MockedStatic<ReflectionUtils> mockedStatic = Mockito.mockStatic(ReflectionUtils.class)) { + mockedStatic.when(() -> + ReflectionUtils.loadClass( + eq("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter"), + any(Class[].class), + eq(metricsConfig), + eq(registry) + ) + ).thenReturn(reporterMock); + + MetricsReporter actualReporter = MetricsReporterFactory.createReporter(metricsConfig, registry).get(); + assertSame(reporterMock, actualReporter); + } + } + + @Test + void metricsReporterFactoryShouldReturnUserDefinedReporter() { when(metricsConfig.getMetricReporterClassName()).thenReturn(DummyMetricsReporter.class.getName()); TypedProperties props = new TypedProperties(); @@ -82,7 +109,7 @@ public class TestMetricsReporterFactory { } @Test - public void metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() { + void metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() { when(metricsConfig.getMetricReporterClassName()).thenReturn(IllegalTestMetricsReporter.class.getName()); when(metricsConfig.getProps()).thenReturn(new TypedProperties()); assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(metricsConfig, registry)); @@ -96,14 +123,17 @@ public class TestMetricsReporterFactory { @Override public void start() { + // no-op } @Override public void report() { + // no-op } @Override public void stop() { + // no-op } } @@ -113,5 +143,3 @@ public class TestMetricsReporterFactory { } } } - - diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java index 19b63a3a5c6..c7d8a28bf76 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java @@ -47,6 +47,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.nio.file.Files; @@ -658,6 +659,37 @@ public class TestFSUtils extends HoodieCommonTestHarness { assertEquals(FSUtils.getPathWithoutScheme(new Path(path1)).toUri().toString(), "//test_bucket_one/table/base/path", "should return false since bucket names dont match"); } + @Test + void testS3aToS3_AWS() { + // Test cases for AWS S3 URLs + assertEquals("s3://my-bucket/path/to/object", FSUtils.s3aToS3("s3a://my-bucket/path/to/object")); + assertEquals("s3://my-bucket", FSUtils.s3aToS3("s3a://my-bucket")); + assertEquals("s3://MY-BUCKET/PATH/TO/OBJECT", FSUtils.s3aToS3("s3a://MY-BUCKET/PATH/TO/OBJECT")); + assertEquals("s3://my-bucket/path/to/object", FSUtils.s3aToS3("S3a://my-bucket/path/to/object")); + assertEquals("s3://my-bucket/path/to/object", FSUtils.s3aToS3("s3A://my-bucket/path/to/object")); + assertEquals("s3://my-bucket/path/to/object", FSUtils.s3aToS3("S3A://my-bucket/path/to/object")); + assertEquals("s3://my-bucket/s3a://another-bucket/another/path", FSUtils.s3aToS3("s3a://my-bucket/s3a://another-bucket/another/path")); + } + + @ParameterizedTest + @ValueSource(strings = { + "gs://my-bucket/path/to/object", + "gs://my-bucket", + "gs://MY-BUCKET/PATH/TO/OBJECT", + "https://myaccount.blob.core.windows.net/mycontainer/path/to/blob";, + "https://myaccount.blob.core.windows.net/MYCONTAINER/PATH/TO/BLOB";, + "https://example.com/path/to/resource";, + "http://example.com";, + "ftp://example.com/resource";, + "", + "gs://my-bucket/path/to/s3a://object", + "gs://my-bucket s3a://my-object", + }) + + void testUriDoesNotChange(String uri) { + assertEquals(uri, FSUtils.s3aToS3(uri)); + } + private StoragePath getHoodieTempDir() { return new StoragePath(baseUri.toString(), ".hoodie/.temp"); }
