This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d4cf109cea9 [HUDI-9168] Refactor hudi-client-common to not import
hudi-aws (#12969)
d4cf109cea9 is described below
commit d4cf109cea95e8201e4469ed4e42f33e99315aa7
Author: Alex R <[email protected]>
AuthorDate: Mon Mar 17 11:23:00 2025 -0700
[HUDI-9168] Refactor hudi-client-common to not import hudi-aws (#12969)
---
hudi-aws/pom.xml | 5 ++
.../cloudwatch/CloudWatchMetricsReporter.java | 3 +-
.../cloudwatch/CloudWatchReporter.java | 11 ++--
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 2 +-
...amoDBBasedImplicitPartitionKeyLockProvider.java | 2 +-
.../java/org/apache/hudi/aws/utils/S3Utils.java | 27 ----------
.../cloudwatch/TestCloudWatchMetricsReporter.java | 61 +++++++++++++++++++---
.../cloudwatch/TestCloudWatchReporter.java | 10 ++--
.../org/apache/hudi/aws/utils/TestS3Utils.java | 59 ---------------------
hudi-client/hudi-client-common/pom.xml | 12 ++---
...ZookeeperBasedImplicitBasePathLockProvider.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 5 ++
.../hudi/metrics/MetricsReporterFactory.java | 2 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 38 ++++++++++++--
.../org/apache/hudi/common/fs/TestFSUtils.java | 32 ++++++++++++
15 files changed, 153 insertions(+), 118 deletions(-)
diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml
index 1b2967e5b2c..7ee4a0a7c4a 100644
--- a/hudi-aws/pom.xml
+++ b/hudi-aws/pom.xml
@@ -42,6 +42,11 @@
</dependency>
<!-- Hoodie -->
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-client-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-common</artifactId>
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java
similarity index 96%
rename from
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java
rename to
hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java
index 68e4951f74f..03f71410b36 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/cloudwatch/CloudWatchMetricsReporter.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchMetricsReporter.java
@@ -16,9 +16,8 @@
* limitations under the License.
*/
-package org.apache.hudi.metrics.cloudwatch;
+package org.apache.hudi.aws.metrics.cloudwatch;
-import org.apache.hudi.aws.cloudwatch.CloudWatchReporter;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.MetricsReporter;
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java
similarity index 97%
rename from
hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java
rename to
hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java
index bf5ae9f91f3..037bab52db1 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/metrics/cloudwatch/CloudWatchReporter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.aws.cloudwatch;
+package org.apache.hudi.aws.metrics.cloudwatch;
import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory;
import org.apache.hudi.common.util.Option;
@@ -196,17 +196,17 @@ public class CloudWatchReporter extends ScheduledReporter
{
for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
processCounter(entry.getKey(), entry.getValue(), timestampMilliSec,
metricsData);
- //TODO: Publish other Histogram metrics to cloud watch
+ // TODO: Publish other Histogram metrics to cloud watch
}
for (Map.Entry<String, Meter> entry : meters.entrySet()) {
processCounter(entry.getKey(), entry.getValue(), timestampMilliSec,
metricsData);
- //TODO: Publish other Meter metrics to cloud watch
+ // TODO: Publish other Meter metrics to cloud watch
}
for (Map.Entry<String, Timer> entry : timers.entrySet()) {
processCounter(entry.getKey(), entry.getValue(), timestampMilliSec,
metricsData);
- //TODO: Publish other Timer metrics to cloud watch
+ // TODO: Publish other Timer metrics to cloud watch
}
report(metricsData);
@@ -236,6 +236,9 @@ public class CloudWatchReporter extends ScheduledReporter {
} catch (final Exception ex) {
LOG.error("Error reporting metrics to CloudWatch. The data in this
CloudWatch request "
+ "may have been discarded, and not made it to CloudWatch.", ex);
+ if (ex instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
}
}
}
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
index abda5a9d988..ce14e87ea23 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
@@ -101,7 +101,7 @@ import java.util.concurrent.Future;
import java.util.function.Consumer;
import java.util.stream.Collectors;
-import static org.apache.hudi.aws.utils.S3Utils.s3aToS3;
+import static org.apache.hudi.common.fs.FSUtils.s3aToS3;
import static org.apache.hudi.common.util.MapUtils.containsAll;
import static org.apache.hudi.common.util.MapUtils.isNullOrEmpty;
import static
org.apache.hudi.config.GlueCatalogSyncClientConfig.ALL_PARTITIONS_READ_PARALLELISM;
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java
index d9a131cd096..bcb0f4571c0 100644
---
a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java
+++
b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java
@@ -31,7 +31,7 @@ import
software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import javax.annotation.concurrent.NotThreadSafe;
-import static org.apache.hudi.aws.utils.S3Utils.s3aToS3;
+import static org.apache.hudi.common.fs.FSUtils.s3aToS3;
/**
* A DynamoDB based lock.
diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java
b/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java
deleted file mode 100644
index bfb208ee150..00000000000
--- a/hudi-aws/src/main/java/org/apache/hudi/aws/utils/S3Utils.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.aws.utils;
-
-public final class S3Utils {
-
- public static String s3aToS3(String s3aUrl) {
- return s3aUrl.replaceFirst("(?i)^s3a://", "s3://");
- }
-}
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
similarity index 53%
rename from
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
rename to
hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
index 4b1aaffbf86..15cfb06e623 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
+++
b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchMetricsReporter.java
@@ -16,9 +16,10 @@
* limitations under the License.
*/
-package org.apache.hudi.metrics.cloudwatch;
+package org.apache.hudi.aws.metrics.cloudwatch;
-import org.apache.hudi.aws.cloudwatch.CloudWatchReporter;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.metrics.MetricsReporterFactory;
import org.apache.hudi.metrics.MetricsReporterType;
@@ -27,19 +28,26 @@ import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
-public class TestCloudWatchMetricsReporter {
+class TestCloudWatchMetricsReporter {
+
+ @Mock
+ private HoodieWriteConfig writeConfig;
@Mock
private HoodieMetricsConfig metricsConfig;
@@ -51,7 +59,7 @@ public class TestCloudWatchMetricsReporter {
private CloudWatchReporter reporter;
@Test
- public void testReporter() {
+ void testReporter() {
when(metricsConfig.getCloudWatchReportPeriodSeconds()).thenReturn(30);
CloudWatchMetricsReporter metricsReporter = new
CloudWatchMetricsReporter(metricsConfig, registry, reporter);
@@ -66,7 +74,23 @@ public class TestCloudWatchMetricsReporter {
}
@Test
- public void testReporterViaReporterFactory() {
+ void testReporterUsingMetricsConfig() {
+ when(writeConfig.getMetricsConfig()).thenReturn(metricsConfig);
+ when(metricsConfig.getCloudWatchReportPeriodSeconds()).thenReturn(30);
+ CloudWatchMetricsReporter metricsReporter = new
CloudWatchMetricsReporter(writeConfig, registry, reporter);
+
+ metricsReporter.start();
+ verify(reporter, times(1)).start(30, TimeUnit.SECONDS);
+
+ metricsReporter.report();
+ verify(reporter, times(1)).report();
+
+ metricsReporter.stop();
+ verify(reporter, times(1)).stop();
+ }
+
+ @Test
+ void testReporterViaReporterFactory() {
try {
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH);
// MetricsReporterFactory uses reflection to create
CloudWatchMetricsReporter
@@ -75,7 +99,32 @@ public class TestCloudWatchMetricsReporter {
} catch (Exception e) {
assertTrue(e.getCause() instanceof InvocationTargetException);
assertTrue(Arrays.stream(((InvocationTargetException)
e.getCause()).getTargetException().getStackTrace()).anyMatch(
- ste ->
ste.toString().contains("org.apache.hudi.aws.cloudwatch.CloudWatchReporter.getAmazonCloudWatchClient")));
+ ste ->
ste.toString().contains("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.getAmazonCloudWatchClient")));
+ }
+ }
+
+ @Test
+ void testCreateCloudWatchReporter() {
+ when(metricsConfig.getCloudWatchMetricPrefix()).thenReturn("prefix");
+ when(metricsConfig.getCloudWatchMetricNamespace()).thenReturn("namespace");
+ when(metricsConfig.getCloudWatchMaxDatumsPerRequest()).thenReturn(100);
+ TypedProperties props = new TypedProperties();
+ when(metricsConfig.getProps()).thenReturn(props);
+
+ CloudWatchReporter reporterMock = mock(CloudWatchReporter.class);
+ CloudWatchReporter.Builder builderMock =
mock(CloudWatchReporter.Builder.class);
+
+ try (MockedStatic<CloudWatchReporter> mockedStatic =
Mockito.mockStatic(CloudWatchReporter.class)) {
+ mockedStatic.when(() -> CloudWatchReporter.forRegistry(registry))
+ .thenReturn(builderMock);
+ when(builderMock.prefixedWith("prefix")).thenReturn(builderMock);
+ when(builderMock.namespace("namespace")).thenReturn(builderMock);
+ when(builderMock.maxDatumsPerRequest(100)).thenReturn(builderMock);
+ when(builderMock.build(props)).thenReturn(reporterMock);
+
+ CloudWatchMetricsReporter metricsReporter =
+ new CloudWatchMetricsReporter(metricsConfig, registry);
+ assertNotNull(metricsReporter);
}
}
}
diff --git
a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java
b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java
similarity index 94%
rename from
hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java
rename to
hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java
index 7061188a63c..0073f3687db 100644
---
a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java
+++
b/hudi-aws/src/test/java/org/apache/hudi/aws/metrics/cloudwatch/TestCloudWatchReporter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.aws.cloudwatch;
+package org.apache.hudi.aws.metrics.cloudwatch;
import com.codahale.metrics.Clock;
import com.codahale.metrics.Counter;
@@ -49,10 +49,10 @@ import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-import static
org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_COUNT_TYPE_VALUE;
-import static
org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_GAUGE_TYPE_VALUE;
-import static
org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_METRIC_TYPE_KEY;
-import static
org.apache.hudi.aws.cloudwatch.CloudWatchReporter.DIMENSION_TABLE_NAME_KEY;
+import static
org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_COUNT_TYPE_VALUE;
+import static
org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_GAUGE_TYPE_VALUE;
+import static
org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_METRIC_TYPE_KEY;
+import static
org.apache.hudi.aws.metrics.cloudwatch.CloudWatchReporter.DIMENSION_TABLE_NAME_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
b/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
deleted file mode 100644
index a0bdd110d83..00000000000
--- a/hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the License.
- */
-
-package org.apache.hudi.aws.utils;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-class TestS3Utils {
-
- @Test
- void testS3aToS3_AWS() {
- // Test cases for AWS S3 URLs
- assertEquals("s3://my-bucket/path/to/object",
S3Utils.s3aToS3("s3a://my-bucket/path/to/object"));
- assertEquals("s3://my-bucket", S3Utils.s3aToS3("s3a://my-bucket"));
- assertEquals("s3://MY-BUCKET/PATH/TO/OBJECT",
S3Utils.s3aToS3("s3a://MY-BUCKET/PATH/TO/OBJECT"));
- assertEquals("s3://my-bucket/path/to/object",
S3Utils.s3aToS3("S3a://my-bucket/path/to/object"));
- assertEquals("s3://my-bucket/path/to/object",
S3Utils.s3aToS3("s3A://my-bucket/path/to/object"));
- assertEquals("s3://my-bucket/path/to/object",
S3Utils.s3aToS3("S3A://my-bucket/path/to/object"));
- assertEquals("s3://my-bucket/s3a://another-bucket/another/path",
S3Utils.s3aToS3("s3a://my-bucket/s3a://another-bucket/another/path"));
- }
-
- @ParameterizedTest
- @ValueSource(strings = {
- "gs://my-bucket/path/to/object",
- "gs://my-bucket",
- "gs://MY-BUCKET/PATH/TO/OBJECT",
- "https://myaccount.blob.core.windows.net/mycontainer/path/to/blob";,
- "https://myaccount.blob.core.windows.net/MYCONTAINER/PATH/TO/BLOB";,
- "https://example.com/path/to/resource";,
- "http://example.com";,
- "ftp://example.com/resource";,
- "",
- "gs://my-bucket/path/to/s3a://object",
- "gs://my-bucket s3a://my-object",
-
- })
- void testUriDoesNotChange(String uri) {
- assertEquals(uri, S3Utils.s3aToS3(uri));
- }
-}
diff --git a/hudi-client/hudi-client-common/pom.xml
b/hudi-client/hudi-client-common/pom.xml
index 5178ae92253..67b12413407 100644
--- a/hudi-client/hudi-client-common/pom.xml
+++ b/hudi-client/hudi-client-common/pom.xml
@@ -48,12 +48,6 @@
<artifactId>hudi-io</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-aws</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-timeline-service</artifactId>
@@ -131,6 +125,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-hadoop-mr</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-tests-common</artifactId>
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java
index 199a9b6309d..9c4a57bc271 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.NotThreadSafe;
-import static org.apache.hudi.aws.utils.S3Utils.s3aToS3;
+import static org.apache.hudi.common.fs.FSUtils.s3aToS3;
/**
* A zookeeper based lock. This {@link LockProvider} implementation allows to
lock table operations
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 9edc1729dc0..0d78d03b029 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -738,6 +738,11 @@ public class FSUtils {
? new Path(null, path.toUri().getAuthority(), path.toUri().getPath())
: path;
}
+ // Converts s3a to s3a
+ public static String s3aToS3(String s3aUrl) {
+ return s3aUrl.replaceFirst("(?i)^s3a://", "s3://");
+ }
+
/**
* Serializable function interface.
*
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
index 71509d7a9b7..d7d04903576 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
@@ -86,7 +86,7 @@ public class MetricsReporterFactory {
reporter = new ConsoleMetricsReporter(registry);
break;
case CLOUDWATCH:
- reporter = (MetricsReporter)
ReflectionUtils.loadClass("org.apache.hudi.metrics.cloudwatch.CloudWatchMetricsReporter",
+ reporter = (MetricsReporter)
ReflectionUtils.loadClass("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter",
new Class[]{HoodieMetricsConfig.class, MetricRegistry.class},
metricsConfig, registry);
break;
case M3:
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
similarity index 75%
rename from
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
rename to
hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
index 5afd27fbea3..d456748d976 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
+++
b/hudi-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
@@ -20,6 +20,7 @@
package org.apache.hudi.metrics;
import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metrics.custom.CustomizableMetricsReporter;
@@ -32,17 +33,23 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
-public class TestMetricsReporterFactory {
+class TestMetricsReporterFactory {
@Mock
HoodieMetricsConfig metricsConfig;
@@ -68,7 +75,27 @@ public class TestMetricsReporterFactory {
}
@Test
- public void metricsReporterFactoryShouldReturnUserDefinedReporter() {
+ void metricsReporterFactoryShouldReturnCloudWatchReporter() {
+
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.CLOUDWATCH);
+
+ MetricsReporter reporterMock = mock(MetricsReporter.class);
+ try (MockedStatic<ReflectionUtils> mockedStatic =
Mockito.mockStatic(ReflectionUtils.class)) {
+ mockedStatic.when(() ->
+ ReflectionUtils.loadClass(
+
eq("org.apache.hudi.aws.metrics.cloudwatch.CloudWatchMetricsReporter"),
+ any(Class[].class),
+ eq(metricsConfig),
+ eq(registry)
+ )
+ ).thenReturn(reporterMock);
+
+ MetricsReporter actualReporter =
MetricsReporterFactory.createReporter(metricsConfig, registry).get();
+ assertSame(reporterMock, actualReporter);
+ }
+ }
+
+ @Test
+ void metricsReporterFactoryShouldReturnUserDefinedReporter() {
when(metricsConfig.getMetricReporterClassName()).thenReturn(DummyMetricsReporter.class.getName());
TypedProperties props = new TypedProperties();
@@ -82,7 +109,7 @@ public class TestMetricsReporterFactory {
}
@Test
- public void
metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() {
+ void
metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() {
when(metricsConfig.getMetricReporterClassName()).thenReturn(IllegalTestMetricsReporter.class.getName());
when(metricsConfig.getProps()).thenReturn(new TypedProperties());
assertThrows(HoodieException.class, () ->
MetricsReporterFactory.createReporter(metricsConfig, registry));
@@ -96,14 +123,17 @@ public class TestMetricsReporterFactory {
@Override
public void start() {
+ // no-op
}
@Override
public void report() {
+ // no-op
}
@Override
public void stop() {
+ // no-op
}
}
@@ -113,5 +143,3 @@ public class TestMetricsReporterFactory {
}
}
}
-
-
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
index 19b63a3a5c6..c7d8a28bf76 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
@@ -47,6 +47,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.nio.file.Files;
@@ -658,6 +659,37 @@ public class TestFSUtils extends HoodieCommonTestHarness {
assertEquals(FSUtils.getPathWithoutScheme(new
Path(path1)).toUri().toString(), "//test_bucket_one/table/base/path", "should
return false since bucket names dont match");
}
+ @Test
+ void testS3aToS3_AWS() {
+ // Test cases for AWS S3 URLs
+ assertEquals("s3://my-bucket/path/to/object",
FSUtils.s3aToS3("s3a://my-bucket/path/to/object"));
+ assertEquals("s3://my-bucket", FSUtils.s3aToS3("s3a://my-bucket"));
+ assertEquals("s3://MY-BUCKET/PATH/TO/OBJECT",
FSUtils.s3aToS3("s3a://MY-BUCKET/PATH/TO/OBJECT"));
+ assertEquals("s3://my-bucket/path/to/object",
FSUtils.s3aToS3("S3a://my-bucket/path/to/object"));
+ assertEquals("s3://my-bucket/path/to/object",
FSUtils.s3aToS3("s3A://my-bucket/path/to/object"));
+ assertEquals("s3://my-bucket/path/to/object",
FSUtils.s3aToS3("S3A://my-bucket/path/to/object"));
+ assertEquals("s3://my-bucket/s3a://another-bucket/another/path",
FSUtils.s3aToS3("s3a://my-bucket/s3a://another-bucket/another/path"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "gs://my-bucket/path/to/object",
+ "gs://my-bucket",
+ "gs://MY-BUCKET/PATH/TO/OBJECT",
+ "https://myaccount.blob.core.windows.net/mycontainer/path/to/blob";,
+ "https://myaccount.blob.core.windows.net/MYCONTAINER/PATH/TO/BLOB";,
+ "https://example.com/path/to/resource";,
+ "http://example.com";,
+ "ftp://example.com/resource";,
+ "",
+ "gs://my-bucket/path/to/s3a://object",
+ "gs://my-bucket s3a://my-object",
+ })
+
+ void testUriDoesNotChange(String uri) {
+ assertEquals(uri, FSUtils.s3aToS3(uri));
+ }
+
private StoragePath getHoodieTempDir() {
return new StoragePath(baseUri.toString(), ".hoodie/.temp");
}
