[
https://issues.apache.org/jira/browse/BEAM-3533?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16339643#comment-16339643
]
ASF GitHub Bot commented on BEAM-3533:
--------------------------------------
jbonofre closed pull request #4486: BEAM-3533 - Replace hard-coded UTF-8
Strings.
URL: https://github.com/apache/beam/pull/4486
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
index 42788b52f57..6099f5849c1 100644
---
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
+++
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
@@ -28,6 +28,7 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -195,9 +196,9 @@ public static String getJobMonitoringPageURL(String
projectName, String regionId
// that has the project name replaced with project id.
return String.format(
"https://console.cloud.google.com/dataflow/jobsDetail/locations/%s/jobs/%s?project=%s";,
- URLEncoder.encode(regionId, "UTF-8"),
- URLEncoder.encode(jobId, "UTF-8"),
- URLEncoder.encode(projectName, "UTF-8"));
+ URLEncoder.encode(regionId, StandardCharsets.UTF_8.name()),
+ URLEncoder.encode(jobId, StandardCharsets.UTF_8.name()),
+ URLEncoder.encode(projectName, StandardCharsets.UTF_8.name()));
} catch (UnsupportedEncodingException e) {
// Should never happen.
throw new AssertionError("UTF-8 encoding is not supported by the
environment", e);
diff --git
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
index 654f5b4e224..7a6e2883aa7 100644
---
a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
+++
b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/PackageUtilTest.java
@@ -258,7 +258,8 @@ public void testPackageUploadWithFileSucceeds() throws
Exception {
assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN +
".txt"));
assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName()));
- assertThat(new LineReader(Channels.newReader(pipe.source(),
"UTF-8")).readLine(),
+ assertThat(new LineReader(Channels.newReader(pipe.source(),
+ StandardCharsets.UTF_8.name())).readLine(),
equalTo(contents));
}
diff --git
a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
index 640cd4143b6..3efdb34cc96 100644
---
a/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
+++
b/runners/spark/src/test/java/org/apache/beam/runners/spark/io/NumShardsTest.java
@@ -21,11 +21,11 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import com.google.common.base.Charsets;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
@@ -79,7 +79,7 @@ public void testText() throws Exception {
for (File f :
tmpDir.getRoot().listFiles(pathname ->
pathname.getName().matches("out-.*\\.txt"))) {
count++;
- for (String line : Files.readLines(f, Charsets.UTF_8)) {
+ for (String line : Files.readLines(f, StandardCharsets.UTF_8)) {
assertTrue(line + " not found", expected.remove(line));
}
}
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
index d7c4dd1d414..4e113483c11 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroSource.java
@@ -36,6 +36,7 @@
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SeekableByteChannel;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Map;
import java.util.WeakHashMap;
@@ -419,9 +420,9 @@ static AvroMetadata readMetadataFromFile(ResourceId
fileResource) throws IOExcep
byte[] bytes = new byte[valueBuffer.remaining()];
valueBuffer.get(bytes);
if (key.equals(DataFileConstants.CODEC)) {
- codec = new String(bytes, "UTF-8");
+ codec = new String(bytes, StandardCharsets.UTF_8);
} else if (key.equals(DataFileConstants.SCHEMA)) {
- schemaString = new String(bytes, "UTF-8");
+ schemaString = new String(bytes, StandardCharsets.UTF_8);
}
}
numRecords = decoder.mapNext();
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
index 97d94c937f0..1d88f1de876 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java
@@ -420,7 +420,7 @@ public void testCompressionBZIP2() throws
FileNotFoundException, IOException {
new BufferedReader(
new InputStreamReader(
new BZip2CompressorInputStream(new FileInputStream(file)),
- StandardCharsets.UTF_8.name())),
+ StandardCharsets.UTF_8)),
"abc",
"123");
}
@@ -433,7 +433,7 @@ public void testCompressionGZIP() throws
FileNotFoundException, IOException {
assertReadValues(
new BufferedReader(
new InputStreamReader(
- new GZIPInputStream(new FileInputStream(file)),
StandardCharsets.UTF_8.name())),
+ new GZIPInputStream(new FileInputStream(file)),
StandardCharsets.UTF_8)),
"abc",
"123");
}
@@ -448,7 +448,7 @@ public void testCompressionDEFLATE() throws
FileNotFoundException, IOException {
new BufferedReader(
new InputStreamReader(
new DeflateCompressorInputStream(new FileInputStream(file)),
- StandardCharsets.UTF_8.name())),
+ StandardCharsets.UTF_8)),
"abc",
"123");
}
@@ -461,7 +461,7 @@ public void testCompressionUNCOMPRESSED() throws
FileNotFoundException, IOExcept
// Read uncompressed data back in using standard API.
assertReadValues(
new BufferedReader(
- new InputStreamReader(new FileInputStream(file),
StandardCharsets.UTF_8.name())),
+ new InputStreamReader(new FileInputStream(file),
StandardCharsets.UTF_8)),
"abc",
"123");
}
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
index 0d92ab8353f..ff16eaad747 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/SimpleSink.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -112,7 +113,7 @@ public SimpleWriter(SimpleWriteOperation writeOperation) {
}
private static ByteBuffer wrap(String value) throws Exception {
- return ByteBuffer.wrap((value + "\n").getBytes("UTF-8"));
+ return ByteBuffer.wrap((value + "\n").getBytes(StandardCharsets.UTF_8));
}
@Override
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
index 2271786fe31..a142cef0ba0 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubUnboundedSource.java
@@ -24,12 +24,12 @@
import com.google.api.client.util.Clock;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Charsets;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -989,7 +989,7 @@ public Instant getCurrentTimestamp() throws
NoSuchElementException {
if (current == null) {
throw new NoSuchElementException();
}
- return current.recordId.getBytes(Charsets.UTF_8);
+ return current.recordId.getBytes(StandardCharsets.UTF_8);
}
/**
diff --git
a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
index 7d2fd280240..bd2faba22e7 100644
---
a/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
+++
b/sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/SerializableScanTest.java
@@ -20,6 +20,8 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import java.nio.charset.StandardCharsets;
+
import org.apache.commons.lang3.SerializationUtils;
import org.apache.hadoop.hbase.client.Scan;
import org.junit.Rule;
@@ -37,11 +39,11 @@
@Test
public void testSerializationDeserialization() throws Exception {
- Scan scan = new Scan().setStartRow("1".getBytes("UTF-8"));
+ Scan scan = new Scan().setStartRow("1".getBytes(StandardCharsets.UTF_8));
byte[] object = SerializationUtils.serialize(new SerializableScan(scan));
SerializableScan serScan = SerializationUtils.deserialize(object);
assertNotNull(serScan);
- assertEquals(new String(serScan.get().getStartRow(), "UTF-8"), "1");
+ assertEquals(new String(serScan.get().getStartRow(),
StandardCharsets.UTF_8), "1");
}
@Test
diff --git
a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
index 057b7bb5ff7..e980c7597ee 100644
---
a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
+++
b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
@@ -21,9 +21,9 @@
import
com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
-import com.google.common.base.Charsets;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.joda.time.Instant;
@@ -72,7 +72,7 @@ public ExtendedSequenceNumber getExtendedSequenceNumber() {
* @return unique id of the record based on its position in the stream
*/
public byte[] getUniqueId() {
- return getExtendedSequenceNumber().toString().getBytes(Charsets.UTF_8);
+ return
getExtendedSequenceNumber().toString().getBytes(StandardCharsets.UTF_8);
}
public Instant getReadTime() {
diff --git
a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
index 8fbafd31b0a..41bade64702 100644
---
a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
+++
b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/KinesisUploader.java
@@ -25,9 +25,9 @@
import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry;
import com.amazonaws.services.kinesis.model.PutRecordsResult;
import com.amazonaws.services.kinesis.model.PutRecordsResultEntry;
-import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -52,7 +52,7 @@ public static void uploadAll(List<String> data,
KinesisTestOptions options) {
List<PutRecordsRequestEntry> allRecords = new ArrayList<>();
for (String row : partition) {
allRecords.add(new PutRecordsRequestEntry().
- withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8))).
+ withData(ByteBuffer.wrap(row.getBytes(StandardCharsets.UTF_8))).
withPartitionKey(Integer.toString(row.hashCode()))
);
diff --git
a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
index ebd5e743653..3f08d9dac18 100644
---
a/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
+++
b/sdks/java/io/mongodb/src/main/java/org/apache/beam/sdk/io/mongodb/MongoDbGridFSIO.java
@@ -35,6 +35,7 @@
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -175,7 +176,7 @@
.setConnectionConfiguration(ConnectionConfiguration.create())
.setWriteFn(
(output, outStream) -> {
- outStream.write(output.getBytes("utf-8"));
+ outStream.write(output.getBytes(StandardCharsets.UTF_8));
outStream.write('\n');
})
.build();
diff --git
a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
index 250e2ddb737..56b79237aa0 100644
---
a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
+++
b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBGridFSIOTest.java
@@ -47,6 +47,7 @@
import java.io.OutputStreamWriter;
import java.io.Serializable;
import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -324,7 +325,7 @@ public void testWriteMessage() throws Exception {
DataInputStream dis = new DataInputStream(ins);
byte b[] = new byte[l];
dis.readFully(b);
- results.append(new String(b, "utf-8"));
+ results.append(new String(b, StandardCharsets.UTF_8));
}
}
String dataString = results.toString();
diff --git
a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
index 776834ab7de..c55f65276d6 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlIO.java
@@ -105,7 +105,8 @@
public static <T> Read<T> read() {
return new AutoValue_XmlIO_Read.Builder<T>()
.setConfiguration(
- new
AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset("UTF-8").build())
+ new AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset(
+ StandardCharsets.UTF_8.name()).build())
.setMinBundleSize(1L)
.setCompression(Compression.AUTO)
.build();
@@ -133,7 +134,8 @@
public static <T> ReadFiles<T> readFiles() {
return new AutoValue_XmlIO_ReadFiles.Builder<T>()
.setConfiguration(
- new
AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset("UTF-8").build())
+ new AutoValue_XmlIO_MappingConfiguration.Builder<T>().setCharset(
+ StandardCharsets.UTF_8.name()).build())
.build();
}
@@ -144,7 +146,7 @@
* {@link FileIO#writeDynamic}.
*/
public static <T> Write<T> write() {
- return new AutoValue_XmlIO_Write.Builder<T>().setCharset("UTF-8").build();
+ return new
AutoValue_XmlIO_Write.Builder<T>().setCharset(StandardCharsets.UTF_8.name()).build();
}
@AutoValue
@@ -630,7 +632,7 @@ public void open(WritableByteChannel channel) throws
IOException {
}
this.outputStream = Channels.newOutputStream(channel);
- outputStream.write(("<" + getRootElement() +
">\n").getBytes(getCharset()));
+ outputStream.write(("<" + getRootElement() +
">\n").getBytes(Charset.forName(getCharset())));
}
@Override
@@ -644,7 +646,7 @@ public void write(T element) throws IOException {
@Override
public void flush() throws IOException {
- outputStream.write(("\n</" + getRootElement() +
">").getBytes(getCharset()));
+ outputStream.write(("\n</" + getRootElement() +
">").getBytes(Charset.forName(getCharset())));
}
}
}
diff --git
a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
index 5cbe07deb22..caaac32c224 100644
--- a/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
+++ b/sdks/java/io/xml/src/main/java/org/apache/beam/sdk/io/xml/XmlSource.java
@@ -27,6 +27,7 @@
import java.nio.CharBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.NoSuchElementException;
import javax.xml.bind.JAXBContext;
@@ -180,7 +181,7 @@ protected void startReading(ReadableByteChannel channel)
throws IOException {
+ getCurrentSource().configuration.getCharset()
+ "\"?><%s>",
XML_VERSION,
getCurrentSource().configuration.getRootElement()))
- .getBytes(getCurrentSource().configuration.getCharset());
+
.getBytes(Charset.forName(getCurrentSource().configuration.getCharset()));
preambleByteBuffer.write(dummyStartDocumentBytes);
// Gets the byte offset (in the input file) of the first record in
ReadableByteChannel. This
// method returns the offset and stores any bytes that should be used
when creating the XML
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
index ca1aab50f39..8d13a20bac5 100644
---
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java
@@ -22,6 +22,7 @@
import java.io.Serializable;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
import java.util.concurrent.ThreadLocalRandom;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
@@ -359,7 +360,7 @@ public void processElement(ProcessContext c, BoundedWindow
window)
try (OutputStream output =
Channels.newOutputStream(openWritableGcsFile(options, filename))) {
for (OutputFile outputFile : c.element().getValue())
{
-
output.write(outputFile.toString().getBytes("UTF-8"));
+
output.write(outputFile.toString().getBytes(StandardCharsets.UTF_8));
n++;
}
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Replace hard-coded UTF-8 Strings
> --------------------------------
>
> Key: BEAM-3533
> URL: https://issues.apache.org/jira/browse/BEAM-3533
> Project: Beam
> Issue Type: Improvement
> Components: beam-model
> Reporter: Colm O hEigeartaigh
> Assignee: Jean-Baptiste Onofré
> Priority: Trivial
> Fix For: 2.3.0
>
>
> This task is to replace hard-coded UTF-8 Strings in the code with
> StandardCharsets.UTF_8 instead.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
