[hudi] branch master updated: [HUDI-5545] Extending support to other special characters (#7585)

[yihua]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 70d450e8389 [HUDI-5545] Extending support to other special characters 
(#7585)
70d450e8389 is described below

commit 70d450e8389173700f7fc6ed4c50027ae4b40f95
Author: srikanth.jaggari <89932129+srikanthjagg...@users.noreply.github.com>
AuthorDate: Fri Jan 13 02:02:51 2023 +0000

    [HUDI-5545] Extending support to other special characters (#7585)
    
    Co-authored-by: sivabalan <n.siv...@gmail.com>
---
 .../apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java  | 4 +++-
 .../hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java     | 5 +++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java
 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java
index 13de7155404..5b084495136 100644
--- 
a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java
+++ 
b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java
@@ -151,7 +151,9 @@ public class S3EventsMetaSelector extends 
CloudObjectsSelector {
               
.getTime()).max().orElse(lastCheckpointStr.map(Long::parseLong).orElse(0L));
 
       for (Map<String, Object> eventRecord : eventRecords) {
-        filteredEventRecords.add(new 
ObjectMapper().writeValueAsString(eventRecord).replace("%3D", "="));
+        filteredEventRecords.add(new 
ObjectMapper().writeValueAsString(eventRecord).replace("%3D", "=")
+            .replace("%24", "$").replace("%A3", "£").replace("%23", 
"#").replace("%26", "&").replace("%3F", "?")
+            .replace("%7E", "~").replace("%25", "%").replace("%2B", "+"));
       }
       // Return the old checkpoint if no messages to consume from queue.
       String newCheckpoint = newCheckpointTime == 0 ? 
lastCheckpointStr.orElse(null) : String.valueOf(newCheckpointTime);
diff --git 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java
 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java
index f38e89b217b..7a0abe722f3 100644
--- 
a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java
+++ 
b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestS3EventsMetaSelector.java
@@ -89,7 +89,8 @@ public class TestS3EventsMetaSelector extends 
HoodieClientTestHarness {
     S3EventsMetaSelector selector = (S3EventsMetaSelector) 
ReflectionUtils.loadClass(clazz.getName(), props);
     // setup s3 record
     String bucket = "test-bucket";
-    String key = "part-foo-bar.snappy.parquet";
+    String key = 
"part%3Dpart%2Bpart%24part%A3part%23part%26part%3Fpart%7Epart%25.snappy.parquet";
+    String keyRes = 
"part=part+part$part£part#part&part?part~part%.snappy.parquet";
     Path path = new Path(bucket, key);
     CloudObjectTestUtils.setMessagesInQueue(sqs, path);
 
@@ -102,7 +103,7 @@ public class TestS3EventsMetaSelector extends 
HoodieClientTestHarness {
     assertEquals(1, eventFromQueue.getLeft().size());
     assertEquals(1, processed.size());
     assertEquals(
-        key,
+        keyRes,
         new JSONObject(eventFromQueue.getLeft().get(0))
             .getJSONObject("s3")
             .getJSONObject("object")