This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8f524d093e HDDS-10811. Reduce UTF8 string encoding by caching encoding
result (#6656)
8f524d093e is described below
commit 8f524d093e8b0c73e2b76fe79f15fa23a48e149e
Author: XiChen <[email protected]>
AuthorDate: Mon May 13 01:19:55 2024 +0800
HDDS-10811. Reduce UTF8 string encoding by caching encoding result (#6656)
---
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 124 +++++++++++++++------
.../apache/hadoop/hdds/scm/net/NetConstants.java | 3 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 55 +++++++--
.../apache/hadoop/util/StringWithByteString.java | 54 +++++++++
5 files changed, 192 insertions(+), 46 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index ff0bdd65f1..c39f16053e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -803,7 +803,7 @@ public final class HddsUtils {
}
@Nonnull
- public static String threadNamePrefix(@Nullable String id) {
+ public static String threadNamePrefix(@Nullable Object id) {
return id != null && !"".equals(id)
? id + "-"
: "";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index b455daba52..224f961158 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -26,8 +26,11 @@ import java.util.UUID;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.ImmutableSet;
+import com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.DatanodeVersion;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.scm.net.NetUtils;
+import org.apache.hadoop.util.StringWithByteString;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -81,11 +84,10 @@ public class DatanodeDetails extends NodeImpl implements
* DataNode's unique identifier in the cluster.
*/
private final UUID uuid;
- private final String uuidString;
+ private final StringWithByteString uuidString;
private final String threadNamePrefix;
-
- private String ipAddress;
- private String hostName;
+ private StringWithByteString ipAddress;
+ private StringWithByteString hostName;
private final List<Port> ports;
private String certSerialId;
private String version;
@@ -100,7 +102,7 @@ public class DatanodeDetails extends NodeImpl implements
private DatanodeDetails(Builder b) {
super(b.hostName, b.networkLocation, NetConstants.NODE_COST_DEFAULT);
uuid = b.id;
- uuidString = uuid.toString();
+ uuidString = StringWithByteString.valueOf(uuid.toString());
threadNamePrefix = HddsUtils.threadNamePrefix(uuidString);
ipAddress = b.ipAddress;
hostName = b.hostName;
@@ -123,17 +125,17 @@ public class DatanodeDetails extends NodeImpl implements
}
public DatanodeDetails(DatanodeDetails datanodeDetails) {
- super(datanodeDetails.getHostName(), datanodeDetails.getNetworkLocation(),
+ super(datanodeDetails.getHostNameAsByteString(),
datanodeDetails.getNetworkLocationAsByteString(),
datanodeDetails.getParent(), datanodeDetails.getLevel(),
datanodeDetails.getCost());
this.uuid = datanodeDetails.uuid;
- this.uuidString = uuid.toString();
+ this.uuidString = datanodeDetails.uuidString;
threadNamePrefix = HddsUtils.threadNamePrefix(uuidString);
this.ipAddress = datanodeDetails.ipAddress;
this.hostName = datanodeDetails.hostName;
this.ports = datanodeDetails.ports;
this.certSerialId = datanodeDetails.certSerialId;
- this.setNetworkName(datanodeDetails.getNetworkName());
+ this.setNetworkName(datanodeDetails.getNetworkNameAsByteString());
this.setParent(datanodeDetails.getParent());
this.version = datanodeDetails.version;
this.setupTime = datanodeDetails.setupTime;
@@ -161,7 +163,7 @@ public class DatanodeDetails extends NodeImpl implements
* @return UUID of DataNode
*/
public String getUuidString() {
- return uuidString;
+ return uuidString.getString();
}
/**
@@ -170,7 +172,7 @@ public class DatanodeDetails extends NodeImpl implements
* @param ip IP Address
*/
public void setIpAddress(String ip) {
- this.ipAddress = ip;
+ this.ipAddress = StringWithByteString.valueOf(ip);
}
/**
@@ -179,6 +181,15 @@ public class DatanodeDetails extends NodeImpl implements
* @return IP address
*/
public String getIpAddress() {
+ return ipAddress.getString();
+ }
+
+ /**
+ * Returns IP address of DataNode as a StringWithByteString object.
+ *
+ * @return IP address as ByteString
+ */
+ public StringWithByteString getIpAddressAsByteString() {
return ipAddress;
}
@@ -188,7 +199,7 @@ public class DatanodeDetails extends NodeImpl implements
* @param host hostname
*/
public void setHostName(String host) {
- this.hostName = host;
+ this.hostName = StringWithByteString.valueOf(host);
}
/**
@@ -197,6 +208,15 @@ public class DatanodeDetails extends NodeImpl implements
* @return Hostname
*/
public String getHostName() {
+ return hostName.getString();
+ }
+
+ /**
+ * Returns IP address of DataNode as a StringWithByteString object.
+ *
+ * @return Hostname
+ */
+ public StringWithByteString getHostNameAsByteString() {
return hostName;
}
@@ -327,10 +347,10 @@ public class DatanodeDetails extends NodeImpl implements
}
if (datanodeDetailsProto.hasIpAddress()) {
- builder.setIpAddress(datanodeDetailsProto.getIpAddress());
+ builder.setIpAddress(datanodeDetailsProto.getIpAddress(),
datanodeDetailsProto.getIpAddressBytes());
}
if (datanodeDetailsProto.hasHostName()) {
- builder.setHostName(datanodeDetailsProto.getHostName());
+ builder.setHostName(datanodeDetailsProto.getHostName(),
datanodeDetailsProto.getHostNameBytes());
}
if (datanodeDetailsProto.hasCertSerialId()) {
builder.setCertSerialId(datanodeDetailsProto.getCertSerialId());
@@ -343,10 +363,12 @@ public class DatanodeDetails extends NodeImpl implements
}
}
if (datanodeDetailsProto.hasNetworkName()) {
- builder.setNetworkName(datanodeDetailsProto.getNetworkName());
+ builder.setNetworkName(
+ datanodeDetailsProto.getNetworkName(),
datanodeDetailsProto.getNetworkNameBytes());
}
if (datanodeDetailsProto.hasNetworkLocation()) {
- builder.setNetworkLocation(datanodeDetailsProto.getNetworkLocation());
+ builder.setNetworkLocation(
+ datanodeDetailsProto.getNetworkLocation(),
datanodeDetailsProto.getNetworkLocationBytes());
}
if (datanodeDetailsProto.hasLevel()) {
builder.setLevel(datanodeDetailsProto.getLevel());
@@ -426,22 +448,22 @@ public class DatanodeDetails extends NodeImpl implements
HddsProtos.DatanodeDetailsProto.newBuilder()
.setUuid128(uuid128);
- builder.setUuid(getUuidString());
+ builder.setUuidBytes(uuidString.getBytes());
if (ipAddress != null) {
- builder.setIpAddress(ipAddress);
+ builder.setIpAddressBytes(ipAddress.getBytes());
}
if (hostName != null) {
- builder.setHostName(hostName);
+ builder.setHostNameBytes(hostName.getBytes());
}
if (certSerialId != null) {
builder.setCertSerialId(certSerialId);
}
if (!Strings.isNullOrEmpty(getNetworkName())) {
- builder.setNetworkName(getNetworkName());
+ builder.setNetworkNameBytes(getNetworkNameAsByteString().getBytes());
}
if (!Strings.isNullOrEmpty(getNetworkLocation())) {
- builder.setNetworkLocation(getNetworkLocation());
+
builder.setNetworkLocationBytes(getNetworkLocationAsByteString().getBytes());
}
if (getLevel() > 0) {
builder.setLevel(getLevel());
@@ -571,10 +593,10 @@ public class DatanodeDetails extends NodeImpl implements
*/
public static final class Builder {
private UUID id;
- private String ipAddress;
- private String hostName;
- private String networkName;
- private String networkLocation;
+ private StringWithByteString ipAddress;
+ private StringWithByteString hostName;
+ private StringWithByteString networkName;
+ private StringWithByteString networkLocation;
private int level;
private List<Port> ports;
private String certSerialId;
@@ -603,10 +625,10 @@ public class DatanodeDetails extends NodeImpl implements
*/
public Builder setDatanodeDetails(DatanodeDetails details) {
this.id = details.getUuid();
- this.ipAddress = details.getIpAddress();
- this.hostName = details.getHostName();
- this.networkName = details.getNetworkName();
- this.networkLocation = details.getNetworkLocation();
+ this.ipAddress = details.getIpAddressAsByteString();
+ this.hostName = details.getHostNameAsByteString();
+ this.networkName = details.getHostNameAsByteString();
+ this.networkLocation = details.getNetworkLocationAsByteString();
this.level = details.getLevel();
this.ports = details.getPorts();
this.certSerialId = details.getCertSerialId();
@@ -638,7 +660,19 @@ public class DatanodeDetails extends NodeImpl implements
* @return DatanodeDetails.Builder
*/
public Builder setIpAddress(String ip) {
- this.ipAddress = ip;
+ this.ipAddress = StringWithByteString.valueOf(ip);
+ return this;
+ }
+
+ /**
+ * Sets the IP address of DataNode.
+ *
+ * @param ip address
+ * @param ipBytes address in Bytes
+ * @return DatanodeDetails.Builder
+ */
+ public Builder setIpAddress(String ip, ByteString ipBytes) {
+ this.ipAddress = new StringWithByteString(ip, ipBytes);
return this;
}
@@ -649,7 +683,19 @@ public class DatanodeDetails extends NodeImpl implements
* @return DatanodeDetails.Builder
*/
public Builder setHostName(String host) {
- this.hostName = host;
+ this.hostName = StringWithByteString.valueOf(host);
+ return this;
+ }
+
+ /**
+ * Sets the hostname of DataNode.
+ *
+ * @param host hostname
+ * @param hostBytes hostname
+ * @return DatanodeDetails.Builder
+ */
+ public Builder setHostName(String host, ByteString hostBytes) {
+ this.hostName = new StringWithByteString(host, hostBytes);
return this;
}
@@ -657,10 +703,11 @@ public class DatanodeDetails extends NodeImpl implements
* Sets the network name of DataNode.
*
* @param name network name
+ * @param nameBytes network name
* @return DatanodeDetails.Builder
*/
- public Builder setNetworkName(String name) {
- this.networkName = name;
+ public Builder setNetworkName(String name, ByteString nameBytes) {
+ this.networkName = new StringWithByteString(name, nameBytes);
return this;
}
@@ -671,7 +718,14 @@ public class DatanodeDetails extends NodeImpl implements
* @return DatanodeDetails.Builder
*/
public Builder setNetworkLocation(String loc) {
- this.networkLocation = loc;
+ return setNetworkLocation(loc, null);
+ }
+
+ public Builder setNetworkLocation(String loc, ByteString locBytes) {
+ final String normalized = NetUtils.normalize(loc);
+ this.networkLocation = normalized.equals(loc) && locBytes != null
+ ? new StringWithByteString(normalized, locBytes)
+ : StringWithByteString.valueOf(normalized);
return this;
}
@@ -794,8 +848,8 @@ public class DatanodeDetails extends NodeImpl implements
*/
public DatanodeDetails build() {
Preconditions.checkNotNull(id);
- if (networkLocation == null) {
- networkLocation = NetConstants.DEFAULT_RACK;
+ if (networkLocation == null || networkLocation.getString().isEmpty()) {
+ networkLocation = NetConstants.BYTE_STRING_DEFAULT_RACK;
}
return new DatanodeDetails(this);
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
index 8ee6decc9c..bd1aa71ebd 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.net;
import org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
+import org.apache.hadoop.util.StringWithByteString;
/**
* Class to hold network topology related constants and configurations.
@@ -32,11 +33,13 @@ public final class NetConstants {
public static final String SCOPE_REVERSE_STR = "~";
/** string representation of root. */
public static final String ROOT = "";
+ public static final StringWithByteString BYTE_STRING_ROOT =
StringWithByteString.valueOf(ROOT);
public static final int INNER_NODE_COST_DEFAULT = 1;
public static final int NODE_COST_DEFAULT = 0;
public static final int ANCESTOR_GENERATION_DEFAULT = 0;
public static final int ROOT_LEVEL = 1;
public static final String DEFAULT_RACK = "/default-rack";
+ public static final StringWithByteString BYTE_STRING_DEFAULT_RACK =
StringWithByteString.valueOf(DEFAULT_RACK);
public static final String DEFAULT_NODEGROUP = "/default-nodegroup";
public static final String DEFAULT_DATACENTER = "/default-datacenter";
public static final String DEFAULT_REGION = "/default-dataregion";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java
index e4d76cd3db..f5f6cec099 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hdds.scm.net;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.util.StringWithByteString;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.BYTE_STRING_ROOT;
import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
/**
@@ -28,9 +29,9 @@ import static
org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
*/
public class NodeImpl implements Node {
// host:port#
- private String name;
+ private StringWithByteString name;
// string representation of this node's location, such as /dc1/rack1
- private String location;
+ private StringWithByteString location;
// location + "/" + name
private String path;
// which level of the tree the node resides, start from 1 for root
@@ -46,18 +47,22 @@ public class NodeImpl implements Node {
* {@link NetConstants#PATH_SEPARATOR})
* @param location this node's location
*/
- public NodeImpl(String name, String location, int cost) {
- if (name != null && name.contains(PATH_SEPARATOR_STR)) {
+ public NodeImpl(StringWithByteString name, StringWithByteString location,
int cost) {
+ if (name != null && name.getString().contains(PATH_SEPARATOR_STR)) {
throw new IllegalArgumentException(
"Network location name:" + name + " should not contain " +
PATH_SEPARATOR_STR);
}
- this.name = (name == null) ? ROOT : name;
- this.location = NetUtils.normalize(location);
+ this.name = name == null ? BYTE_STRING_ROOT : name;
+ this.location = location;
this.path = getPath();
this.cost = cost;
}
+ public NodeImpl(String name, String location, int cost) {
+ this(StringWithByteString.valueOf(name),
StringWithByteString.valueOf(NetUtils.normalize(location)), cost);
+ }
+
/**
* Construct a node from its name and its location.
*
@@ -75,11 +80,25 @@ public class NodeImpl implements Node {
this.level = level;
}
+ public NodeImpl(StringWithByteString name, StringWithByteString location,
InnerNode parent, int level,
+ int cost) {
+ this(name, location, cost);
+ this.parent = parent;
+ this.level = level;
+ }
+
/**
* @return this node's name
*/
@Override
public String getNetworkName() {
+ return name.getString();
+ }
+
+ /**
+ * @return this node's name
+ */
+ public StringWithByteString getNetworkNameAsByteString() {
return name;
}
@@ -89,6 +108,15 @@ public class NodeImpl implements Node {
*/
@Override
public void setNetworkName(String networkName) {
+ this.name = StringWithByteString.valueOf(networkName);
+ this.path = getPath();
+ }
+
+ /**
+ * Set this node's name, can be hostname or Ipaddress.
+ * @param networkName it's network name
+ */
+ public void setNetworkName(StringWithByteString networkName) {
this.name = networkName;
this.path = getPath();
}
@@ -98,6 +126,13 @@ public class NodeImpl implements Node {
*/
@Override
public String getNetworkLocation() {
+ return location.getString();
+ }
+
+ /**
+ * @return this node's network location
+ */
+ public StringWithByteString getNetworkLocationAsByteString() {
return location;
}
@@ -107,7 +142,7 @@ public class NodeImpl implements Node {
*/
@Override
public void setNetworkLocation(String networkLocation) {
- this.location = networkLocation;
+ this.location = StringWithByteString.valueOf(networkLocation);
this.path = getPath();
}
@@ -269,8 +304,8 @@ public class NodeImpl implements Node {
}
private String getPath() {
- return this.location.equals(PATH_SEPARATOR_STR) ?
- this.location + this.name :
+ return this.location.getString().equals(PATH_SEPARATOR_STR) ?
+ this.location + this.name.getString() :
this.location + PATH_SEPARATOR_STR + this.name;
}
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/StringWithByteString.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/StringWithByteString.java
new file mode 100644
index 0000000000..0e99acba3f
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/StringWithByteString.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import com.google.protobuf.ByteString;
+import net.jcip.annotations.Immutable;
+
+import java.util.Objects;
+
+/**
+ * Class to encapsulate and cache the conversion of a Java String to a
ByteString.
+ */
+@Immutable
+public final class StringWithByteString {
+ public static StringWithByteString valueOf(String string) {
+ return string != null ? new StringWithByteString(string,
ByteString.copyFromUtf8(string)) : null;
+ }
+
+ private final String string;
+ private final ByteString bytes;
+
+ public StringWithByteString(String string, ByteString bytes) {
+ this.string = Objects.requireNonNull(string, "string == null");
+ this.bytes = Objects.requireNonNull(bytes, "bytes == null");
+ }
+
+ public String getString() {
+ return string;
+ }
+
+ public ByteString getBytes() {
+ return bytes;
+ }
+
+ @Override
+ public String toString() {
+ return getString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
