This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a6c77eb3d7 Move Gson ByteArrayToBase64TypeAdapter to its own utility
class (#3285)
a6c77eb3d7 is described below
commit a6c77eb3d7b5f8f0bfae5ba4639776a97785dbc3
Author: Christopher L. Shannon <[email protected]>
AuthorDate: Fri Apr 7 13:58:53 2023 -0400
Move Gson ByteArrayToBase64TypeAdapter to its own utility class (#3285)
BulkSerialize had a Gson adapter to serialize byte arrays to Base64
encoding that can be re-used by other areas in the code. Also update the
uses of the base 64 Gson object to be static and re-used as the objects
are thread-safe so can be created just one time.
---
.../core/clientImpl/bulk/BulkSerialize.java | 38 +----------
.../core/clientImpl/bulk/LoadMappingIterator.java | 8 +--
.../core/util/ByteArrayToBase64TypeAdapter.java | 78 ++++++++++++++++++++++
.../util/ByteArrayToBase64TypeAdapterTest.java | 64 ++++++++++++++++++
4 files changed, 148 insertions(+), 40 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
index 0f8d46c58a..42f958b22d 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
@@ -27,10 +27,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
-import java.lang.reflect.Type;
-import java.util.Base64;
-import java.util.Base64.Decoder;
-import java.util.Base64.Encoder;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -41,17 +37,10 @@ import org.apache.accumulo.core.clientImpl.bulk.Bulk.Files;
import org.apache.accumulo.core.clientImpl.bulk.Bulk.Mapping;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.util.ByteArrayToBase64TypeAdapter;
import org.apache.hadoop.fs.Path;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonWriter;
@@ -60,28 +49,7 @@ import com.google.gson.stream.JsonWriter;
*/
public class BulkSerialize {
- private static class ByteArrayToBase64TypeAdapter
- implements JsonSerializer<byte[]>, JsonDeserializer<byte[]> {
-
- Decoder decoder = Base64.getUrlDecoder();
- Encoder encoder = Base64.getUrlEncoder();
-
- @Override
- public byte[] deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context)
- throws JsonParseException {
- return decoder.decode(json.getAsString());
- }
-
- @Override
- public JsonElement serialize(byte[] src, Type typeOfSrc,
JsonSerializationContext context) {
- return new JsonPrimitive(encoder.encodeToString(src));
- }
- }
-
- static Gson createGson() {
- return new GsonBuilder()
- .registerTypeHierarchyAdapter(byte[].class, new
ByteArrayToBase64TypeAdapter()).create();
- }
+ private static final Gson gson =
ByteArrayToBase64TypeAdapter.createBase64Gson();
public interface Output {
OutputStream create(Path path) throws IOException;
@@ -100,7 +68,6 @@ public class BulkSerialize {
try (OutputStream fsOut = output.create(lmFile); JsonWriter writer =
new JsonWriter(new BufferedWriter(new OutputStreamWriter(fsOut,
UTF_8)))) {
- Gson gson = createGson();
writer.setIndent(" ");
writer.beginArray();
Set<Entry<KeyExtent,Files>> es = loadMapping.entrySet();
@@ -141,7 +108,6 @@ public class BulkSerialize {
public static Map<String,String> readRenameMap(String bulkDir, Input input)
throws IOException {
final Path renamingFile = new Path(bulkDir, Constants.BULK_RENAME_FILE);
Map<String,String> oldToNewNameMap;
- Gson gson = createGson();
try (InputStream fis = input.open(renamingFile);
BufferedReader reader = new BufferedReader(new
InputStreamReader(fis))) {
oldToNewNameMap = gson.fromJson(reader, new
TypeToken<Map<String,String>>() {}.getType());
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/LoadMappingIterator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/LoadMappingIterator.java
index 30e104196c..653b320874 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/LoadMappingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/LoadMappingIterator.java
@@ -19,7 +19,6 @@
package org.apache.accumulo.core.clientImpl.bulk;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.accumulo.core.clientImpl.bulk.BulkSerialize.createGson;
import java.io.BufferedReader;
import java.io.IOException;
@@ -32,6 +31,7 @@ import java.util.Map;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.util.ByteArrayToBase64TypeAdapter;
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
@@ -41,9 +41,9 @@ import com.google.gson.stream.JsonReader;
*/
public class LoadMappingIterator
implements Iterator<Map.Entry<KeyExtent,Bulk.Files>>, AutoCloseable {
- private TableId tableId;
- private JsonReader reader;
- private Gson gson = createGson();
+ private final TableId tableId;
+ private final JsonReader reader;
+ private static final Gson gson =
ByteArrayToBase64TypeAdapter.createBase64Gson();
private Map<String,String> renameMap;
LoadMappingIterator(TableId tableId, InputStream loadMapFile) throws
IOException {
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapter.java
b/core/src/main/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapter.java
new file mode 100644
index 0000000000..f48c206b26
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapter.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import java.lang.reflect.Type;
+import java.util.Base64;
+import java.util.Base64.Decoder;
+import java.util.Base64.Encoder;
+import java.util.Objects;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * Gson adapter to handle serializing and deserializing byte arrays using
Base64 encoding.
+ */
+public class ByteArrayToBase64TypeAdapter
+ implements JsonSerializer<byte[]>, JsonDeserializer<byte[]> {
+
+ private static final Decoder decoder = Base64.getUrlDecoder();
+ private static final Encoder encoder = Base64.getUrlEncoder();
+
+ @Override
+ public byte[] deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context)
+ throws JsonParseException {
+ return decoder.decode(json.getAsString());
+ }
+
+ @Override
+ public JsonElement serialize(byte[] src, Type typeOfSrc,
JsonSerializationContext context) {
+ return new JsonPrimitive(encoder.encodeToString(src));
+ }
+
+ /**
+ * Creates a new Gson instance that registers {@link
ByteArrayToBase64TypeAdapter} for handling
+ * serializing/deserializing byte[] types as Base64 encoded
+ *
+ * @return Gson instance
+ */
+ public static Gson createBase64Gson() {
+ return registerBase64TypeAdapter(new GsonBuilder()).create();
+ }
+
+ /**
+ * Register {@link ByteArrayToBase64TypeAdapter} for handling byte[] types
on an existing
+ * GsonBuilder
+ *
+ * @param gsonBuilder existing GsonBuilder
+ * @return GsonBuilder
+ */
+ public static GsonBuilder registerBase64TypeAdapter(final GsonBuilder
gsonBuilder) {
+ return
Objects.requireNonNull(gsonBuilder).registerTypeHierarchyAdapter(byte[].class,
+ new ByteArrayToBase64TypeAdapter());
+ }
+}
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapterTest.java
b/core/src/test/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapterTest.java
new file mode 100644
index 0000000000..92176a4aa1
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/util/ByteArrayToBase64TypeAdapterTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.Test;
+
+import com.google.gson.Gson;
+
+public class ByteArrayToBase64TypeAdapterTest {
+
+ private static final Gson gson =
ByteArrayToBase64TypeAdapter.createBase64Gson();
+
+ @Test
+ public void testSerializeText() throws IOException {
+ final Text original = new Text("This is a test");
+
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ original.write(new DataOutputStream(dos));
+
+ final String encoded = gson.toJson(baos.toByteArray());
+ final Text decoded = new Text();
+ decoded.readFields(
+ new DataInputStream(new ByteArrayInputStream(gson.fromJson(encoded,
byte[].class))));
+ assertEquals(original.toString(), decoded.toString());
+ }
+ }
+
+ @Test
+ public void testByteArray() {
+ final byte[] original = new byte[] {0x01, 0x06, 0x34, 0x09, 0x12, 0x34,
0x57, 0x56, 0x30, 0x74};
+
+ final String encoded = gson.toJson(original);
+ final byte[] decoded = gson.fromJson(encoded, byte[].class);
+
+ assertArrayEquals(original, decoded);
+ }
+}
