This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
The following commit(s) were added to refs/heads/master by this push:
new df921e915 refact(server): enlarge bytes write limit & remove param
`big` when encode/decode string id length (#2622)
df921e915 is described below
commit df921e915df6d44e753cb0d501b6a2cc835fdf2d
Author: V_Galaxy <[email protected]>
AuthorDate: Sat Aug 10 17:02:30 2024 +0800
refact(server): enlarge bytes write limit & remove param `big` when
encode/decode string id length (#2622)
As title, change limit:
- vid max to 16KB
- eid max to 64kb (128k as backup)
- property max to 10MB (keep consistent)
fix #1593 #2291
---------
Co-authored-by: imbajin <[email protected]>
---
.asf.yaml | 4 +-
.../hugegraph/api/filter/AccessLogFilter.java | 3 +-
.../org/apache/hugegraph/core/GraphManager.java | 6 +-
.../hugegraph/backend/cache/OffheapCache.java | 32 +++---
.../java/org/apache/hugegraph/backend/id/Id.java | 3 +-
.../backend/serializer/BinarySerializer.java | 4 +-
.../hugegraph/backend/serializer/BytesBuffer.java | 110 +++++++++------------
.../org/apache/hugegraph/structure/HugeEdge.java | 4 +-
.../apache/hugegraph/structure/HugeElement.java | 3 +-
.../java/org/apache/hugegraph/task/HugeTask.java | 2 +-
.../org/apache/hugegraph/core/EdgeCoreTest.java | 6 +-
.../org/apache/hugegraph/core/VertexCoreTest.java | 7 +-
.../org/apache/hugegraph/unit/cache/CacheTest.java | 7 +-
.../org/apache/hugegraph/unit/id/IdUtilTest.java | 18 ++++
.../hugegraph/unit/serializer/BytesBufferTest.java | 55 ++++++-----
15 files changed, 133 insertions(+), 131 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 6c7853042..93063dc7d 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -34,8 +34,8 @@ github:
protected_branches:
master:
required_status_checks:
- # strict means "Require branches to be up-to-date before merging".
(TODO: turnoff when branch is stable)
- strict: true
+ # strict means "Require PR to be up-to-date before merging". (enable
when branch unstable)
+ strict: false
# contexts are the names of checks that must pass (now only enable the
basic check)
contexts:
- Analyze (java)
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
index d429db4d9..ae5368646 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/filter/AccessLogFilter.java
@@ -26,7 +26,6 @@ import static
org.apache.hugegraph.metrics.MetricsUtil.METRICS_PATH_TOTAL_COUNTE
import java.io.IOException;
import java.net.URI;
-import org.apache.hugegraph.auth.HugeAuthenticator;
import org.apache.hugegraph.config.HugeConfig;
import org.apache.hugegraph.config.ServerOptions;
import org.apache.hugegraph.core.GraphManager;
@@ -124,7 +123,7 @@ public class AccessLogFilter implements
ContainerResponseFilter {
GraphManager manager = managerProvider.get();
// TODO: transfer Authorizer if we need after.
if (manager.requireAuthentication()) {
- manager.unauthorize(requestContext.getSecurityContext());
+ manager.unauthorized(requestContext.getSecurityContext());
}
}
diff --git
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
index 37939c201..5cba0a0f0 100644
---
a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
+++
b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
@@ -265,7 +265,7 @@ public final class GraphManager {
return this.authenticator().authenticate(credentials);
}
- public void unauthorize(SecurityContext context) {
+ public void unauthorized(SecurityContext context) {
this.authenticator().unauthorize(context);
}
@@ -515,7 +515,7 @@ public final class GraphManager {
private void addMetrics(HugeConfig config) {
final MetricManager metric = MetricManager.INSTANCE;
- // Force to add server reporter
+ // Force to add a server reporter
ServerReporter reporter =
ServerReporter.instance(metric.getRegistry());
reporter.start(60L, TimeUnit.SECONDS);
@@ -610,7 +610,7 @@ public final class GraphManager {
/*
* Will fill graph instance into HugeFactory.graphs after
- * GraphFactory.open() succeed, remove it when graph drop
+ * GraphFactory.open() succeed, remove it when the graph drops
*/
HugeFactory.remove(graph);
}
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java
index d641c8276..7ed4efcd6 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/OffheapCache.java
@@ -55,14 +55,18 @@ public class OffheapCache extends AbstractCache<Id, Object>
{
private final AbstractSerializer serializer;
public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes) {
- // NOTE: capacity unit is bytes, the super capacity expect elements
size
+ this(graph, capacity, avgEntryBytes,
Runtime.getRuntime().availableProcessors() * 2);
+ }
+
+ public OffheapCache(HugeGraph graph, long capacity, long avgEntryBytes,
int segments) {
+ // NOTE: capacity unit is bytes, the super capacity expects elements'
size
super(capacity);
- long capacityInBytes = capacity * (avgEntryBytes + 64L);
+ long capacityInBytes = Math.max(capacity, segments) * (avgEntryBytes +
64L);
if (capacityInBytes <= 0L) {
capacityInBytes = 1L;
}
this.graph = graph;
- this.cache = this.builder().capacity(capacityInBytes).build();
+ this.cache =
this.builder().capacity(capacityInBytes).segmentCount(segments).build();
this.serializer = new BinarySerializer();
}
@@ -162,19 +166,18 @@ public class OffheapCache extends AbstractCache<Id,
Object> {
@Override
public Id deserialize(ByteBuffer input) {
- return BytesBuffer.wrap(input).readId(true);
+ return BytesBuffer.wrap(input).readId();
}
@Override
public void serialize(Id id, ByteBuffer output) {
- BytesBuffer.wrap(output).writeId(id, true);
+ BytesBuffer.wrap(output).writeId(id);
}
@Override
public int serializedSize(Id id) {
// NOTE: return size must be == actual bytes to write
- return BytesBuffer.allocate(id.length() + 2)
- .writeId(id, true).position();
+ return BytesBuffer.allocate(id.length() +
2).writeId(id).position();
}
}
@@ -229,7 +232,6 @@ public class OffheapCache extends AbstractCache<Id, Object>
{
}
BytesBuffer buffer = BytesBuffer.allocate(64 * listSize);
-
// May fail to serialize and throw exception here
this.serialize(this.value, buffer);
@@ -276,8 +278,7 @@ public class OffheapCache extends AbstractCache<Id, Object>
{
}
}
- private void serializeList(BytesBuffer buffer,
- Collection<Object> list) {
+ private void serializeList(BytesBuffer buffer, Collection<Object>
list) {
// Write list
buffer.writeVInt(list.size());
for (Object i : list) {
@@ -295,8 +296,7 @@ public class OffheapCache extends AbstractCache<Id, Object>
{
return list;
}
- private void serializeElement(BytesBuffer buffer,
- ValueType type, Object value) {
+ private void serializeElement(BytesBuffer buffer, ValueType type,
Object value) {
E.checkNotNull(value, "serialize value");
BackendEntry entry;
if (type == ValueType.VERTEX) {
@@ -332,14 +332,12 @@ public class OffheapCache extends AbstractCache<Id,
Object> {
}
private HugeException unsupported(ValueType type) {
- throw new HugeException(
- "Unsupported deserialize type: %s", type);
+ throw new HugeException("Unsupported deserialize type: %s", type);
}
private HugeException unsupported(Object value) {
- throw new HugeException(
- "Unsupported type of serialize value: '%s'(%s)",
- value, value.getClass());
+ throw new HugeException("Unsupported type of serialize value:
'%s'(%s)",
+ value, value.getClass());
}
}
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java
index b0e561349..077aa43db 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/Id.java
@@ -67,8 +67,7 @@ public interface Id extends Comparable<Id> {
}
public static IdType valueOfPrefix(String id) {
- E.checkArgument(id != null && !id.isEmpty(),
- "Invalid id '%s'", id);
+ E.checkArgument(id != null && !id.isEmpty(), "Invalid id '%s'",
id);
switch (id.charAt(0)) {
case 'L':
return IdType.LONG;
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
index 37a7e9a9c..0dd4c9d3e 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinarySerializer.java
@@ -1312,9 +1312,9 @@ public class BinarySerializer extends AbstractSerializer {
}
private byte[] writeIds(Collection<Id> ids) {
- E.checkState(ids.size() <= BytesBuffer.UINT16_MAX,
+ E.checkState(ids.size() <= BytesBuffer.MAX_PROPERTIES,
"The number of properties of vertex/edge label " +
- "can't exceed '%s'", BytesBuffer.UINT16_MAX);
+ "can't exceed '%s'", BytesBuffer.MAX_PROPERTIES);
int size = 2;
for (Id id : ids) {
size += (1 + id.length());
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
index b99c2c8b5..c13d61108 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BytesBuffer.java
@@ -51,22 +51,27 @@ public final class BytesBuffer extends OutputStream {
public static final int CHAR_LEN = Character.BYTES;
public static final int FLOAT_LEN = Float.BYTES;
public static final int DOUBLE_LEN = Double.BYTES;
- public static final int BLOB_LEN = 4;
+ public static final int BYTES_LEN = 4;
+ public static final int BLOB_LEN = 5;
public static final int UINT8_MAX = ((byte) -1) & 0xff;
public static final int UINT16_MAX = ((short) -1) & 0xffff;
public static final long UINT32_MAX = (-1) & 0xffffffffL;
+ // TODO: support user-defined configuration
// NOTE: +1 to let code 0 represent length 1
- public static final int ID_LEN_MASK = 0x7f;
- public static final int ID_LEN_MAX = 0x7f + 1; // 128
- public static final int BIG_ID_LEN_MAX = 0x7fff + 1; // 32768
+ public static final int ID_LEN_MAX = 0x3fff + 1; // 16KB
+ public static final int EID_LEN_MAX = 64 * 1024;
public static final byte STRING_ENDING_BYTE = (byte) 0x00;
public static final byte STRING_ENDING_BYTE_FF = (byte) 0xff;
- public static final int STRING_LEN_MAX = UINT16_MAX;
+
+ // TODO: support user-defined configuration
+ public static final long BYTES_LEN_MAX = 10 * Bytes.MB;
public static final long BLOB_LEN_MAX = 1 * Bytes.GB;
+ public static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;
+
// The value must be in range [8, ID_LEN_MAX]
public static final int INDEX_HASH_ID_THRESHOLD = 32;
@@ -87,8 +92,7 @@ public final class BytesBuffer extends OutputStream {
public BytesBuffer(int capacity) {
E.checkArgument(capacity <= MAX_BUFFER_CAPACITY,
- "Capacity exceeds max buffer capacity: %s",
- MAX_BUFFER_CAPACITY);
+ "Capacity exceeds max buffer capacity: %s",
MAX_BUFFER_CAPACITY);
this.buffer = ByteBuffer.allocate(capacity);
this.resize = true;
}
@@ -167,8 +171,7 @@ public final class BytesBuffer extends OutputStream {
// Extra capacity as buffer
int newCapacity = size + this.buffer.limit() + DEFAULT_CAPACITY;
E.checkArgument(newCapacity <= MAX_BUFFER_CAPACITY,
- "Capacity exceeds max buffer capacity: %s",
- MAX_BUFFER_CAPACITY);
+ "Capacity exceeds max buffer capacity: %s",
MAX_BUFFER_CAPACITY);
ByteBuffer newBuffer = ByteBuffer.allocate(newCapacity);
this.buffer.flip();
newBuffer.put(this.buffer);
@@ -288,10 +291,9 @@ public final class BytesBuffer extends OutputStream {
}
public BytesBuffer writeBytes(byte[] bytes) {
- E.checkArgument(bytes.length <= UINT16_MAX,
- "The max length of bytes is %s, but got %s",
- UINT16_MAX, bytes.length);
- require(SHORT_LEN + bytes.length);
+ E.checkArgument(bytes.length <= BYTES_LEN_MAX, "The max length of
bytes is %s, but got %s",
+ BYTES_LEN_MAX, bytes.length);
+ require(BYTES_LEN + bytes.length);
this.writeVInt(bytes.length);
this.write(bytes);
return this;
@@ -304,8 +306,8 @@ public final class BytesBuffer extends OutputStream {
}
public BytesBuffer writeBigBytes(byte[] bytes) {
- E.checkArgument(bytes.length <= BLOB_LEN_MAX,
- "The max length of bytes is %s, but got %s",
+ // TODO: note the max blob size should be 128MB (due to
MAX_BUFFER_CAPACITY)
+ E.checkArgument(bytes.length <= BLOB_LEN_MAX, "The max length of bytes
is %s, but got %s",
BLOB_LEN_MAX, bytes.length);
require(BLOB_LEN + bytes.length);
this.writeVInt(bytes.length);
@@ -346,9 +348,7 @@ public final class BytesBuffer extends OutputStream {
assert !Bytes.contains(bytes, STRING_ENDING_BYTE_FF) :
"Invalid UTF8 bytes: " + value;
if (Bytes.contains(bytes, STRING_ENDING_BYTE)) {
- E.checkArgument(false,
- "Can't contains byte '0x00' in string: '%s'",
- value);
+ E.checkArgument(false, "Can't contains byte '0x00' in string:
'%s'", value);
}
this.write(bytes);
}
@@ -430,8 +430,7 @@ public final class BytesBuffer extends OutputStream {
public int readVInt() {
byte leading = this.read();
- E.checkArgument(leading != 0x80,
- "Unexpected varint with leading byte '0x%s'",
+ E.checkArgument(leading != 0x80, "Unexpected varint with leading byte
'0x%s'",
Bytes.toHex(leading));
int value = leading & 0x7f;
if (leading >= 0) {
@@ -450,8 +449,7 @@ public final class BytesBuffer extends OutputStream {
}
}
- E.checkArgument(i < 5,
- "Unexpected varint %s with too many bytes(%s)",
+ E.checkArgument(i < 5, "Unexpected varint %s with too many bytes(%s)",
value, i + 1);
E.checkArgument(i < 4 || (leading & 0x70) == 0,
"Unexpected varint %s with leading byte '0x%s'",
@@ -494,8 +492,7 @@ public final class BytesBuffer extends OutputStream {
public long readVLong() {
byte leading = this.read();
- E.checkArgument(leading != 0x80,
- "Unexpected varlong with leading byte '0x%s'",
+ E.checkArgument(leading != 0x80, "Unexpected varlong with leading byte
'0x%s'",
Bytes.toHex(leading));
long value = leading & 0x7fL;
if (leading >= 0) {
@@ -514,8 +511,7 @@ public final class BytesBuffer extends OutputStream {
}
}
- E.checkArgument(i < 10,
- "Unexpected varlong %s with too many bytes(%s)",
+ E.checkArgument(i < 10, "Unexpected varlong %s with too many
bytes(%s)",
value, i + 1);
E.checkArgument(i < 9 || (leading & 0x7e) == 0,
"Unexpected varlong %s with leading byte '0x%s'",
@@ -581,8 +577,7 @@ public final class BytesBuffer extends OutputStream {
this.writeString((String) value);
break;
case BLOB:
- byte[] bytes = value instanceof byte[] ?
- (byte[]) value : ((Blob) value).bytes();
+ byte[] bytes = value instanceof byte[] ? (byte[]) value :
((Blob) value).bytes();
this.writeBigBytes(bytes);
break;
case UUID:
@@ -592,6 +587,7 @@ public final class BytesBuffer extends OutputStream {
this.writeLong(uuid.getLeastSignificantBits());
break;
default:
+ // TODO: replace Kryo with Fury
(https://github.com/apache/fury)
this.writeBytes(KryoUtil.toKryoWithType(value));
break;
}
@@ -624,15 +620,12 @@ public final class BytesBuffer extends OutputStream {
case UUID:
return new UUID(this.readLong(), this.readLong());
default:
+ // TODO: replace Kryo with Fury
(https://github.com/apache/fury)
return KryoUtil.fromKryoWithType(this.readBytes());
}
}
public BytesBuffer writeId(Id id) {
- return this.writeId(id, false);
- }
-
- public BytesBuffer writeId(Id id, boolean big) {
switch (id.type()) {
case LONG:
// Number Id
@@ -640,36 +633,33 @@ public final class BytesBuffer extends OutputStream {
this.writeNumber(value);
break;
case UUID:
- // UUID Id
+ // UUID ID
byte[] bytes = id.asBytes();
assert bytes.length == Id.UUID_LENGTH;
this.writeUInt8(0x7f); // 0b01111111 means UUID
this.write(bytes);
break;
case EDGE:
- // Edge Id
+ // Edge ID
this.writeUInt8(0x7e); // 0b01111110 means EdgeId
this.writeEdgeId(id);
break;
default:
- // String Id
+ // String Id (VertexID)
bytes = id.asBytes();
int len = bytes.length;
E.checkArgument(len > 0, "Can't write empty id");
- if (!big) {
- E.checkArgument(len <= ID_LEN_MAX,
- "Id max length is %s, but got %s {%s}",
- ID_LEN_MAX, len, id);
- len -= 1; // mapping [1, 128] to [0, 127]
+ E.checkArgument(len <= ID_LEN_MAX, "Big id max length is %s,
but got %s {%s}",
+ ID_LEN_MAX, len, id);
+ len -= 1; // mapping [1, 16384] to [0, 16383]
+ if (len <= 0x3f) {
+ // If length is <= 63, use a single byte with the highest
bit set to 1
this.writeUInt8(len | 0x80);
} else {
- E.checkArgument(len <= BIG_ID_LEN_MAX,
- "Big id max length is %s, but got %s {%s}",
- BIG_ID_LEN_MAX, len, id);
- len -= 1;
int high = len >> 8;
int low = len & 0xff;
- this.writeUInt8(high | 0x80);
+ // Write high 8 bits with highest two bits set to 11
+ this.writeUInt8(high | 0xc0);
this.writeUInt8(low);
}
this.write(bytes);
@@ -679,10 +669,6 @@ public final class BytesBuffer extends OutputStream {
}
public Id readId() {
- return this.readId(false);
- }
-
- public Id readId(boolean big) {
byte b = this.read();
boolean number = (b & 0x80) == 0;
if (number) {
@@ -698,13 +684,13 @@ public final class BytesBuffer extends OutputStream {
}
} else {
// String Id
- int len = b & ID_LEN_MASK;
- if (big) {
+ int len = b & 0x3f; // Take the lowest 6 bits as part of the length
+ if ((b & 0x40) != 0) { // If the 7th bit is set, length
information spans 2 bytes
int high = len << 8;
int low = this.readUInt8();
len = high + low;
}
- len += 1; // restore [0, 127] to [1, 128]
+ len += 1; // restore [0, 16383] to [1, 16384]
byte[] id = this.read(len);
return IdGenerator.of(id, IdType.STRING);
}
@@ -739,12 +725,11 @@ public final class BytesBuffer extends OutputStream {
this.write(bytes);
if (type.isStringIndex()) {
if (Bytes.contains(bytes, STRING_ENDING_BYTE)) {
- // Not allow STRING_ENDING_BYTE exist in string index id
+ // Not allow STRING_ENDING_BYTE to exist in string index id
E.checkArgument(false,
"The %s type index id can't contains " +
- "byte '0x%s', but got: 0x%s", type,
- Bytes.toHex(STRING_ENDING_BYTE),
- Bytes.toHex(bytes));
+ "byte '0x%s', but got: 0x%s",
+ type, Bytes.toHex(STRING_ENDING_BYTE),
Bytes.toHex(bytes));
}
if (withEnding) {
this.writeStringWithEnding("");
@@ -806,12 +791,9 @@ public final class BytesBuffer extends OutputStream {
}
// Parse id from bytes
int start = this.buffer.position();
- /**
- * OLAP
- * {PropertyKey}{VertexId}
- */
+ // OLAP {PropertyKey}{VertexId}
if (isOlap) {
- // First read olap property id
+ // Read olap property id first
Id pkId = this.readId();
}
Id id = this.readId();
@@ -890,8 +872,7 @@ public final class BytesBuffer extends OutputStream {
}
private long readNumber(byte b) {
- E.checkArgument((b & 0x80) == 0,
- "Not a number type with prefix byte '0x%s'",
+ E.checkArgument((b & 0x80) == 0, "Not a number type with prefix byte
'0x%s'",
Bytes.toHex(b));
// Parse the kind from byte 0kkksxxx
int kind = b >>> 4;
@@ -948,8 +929,7 @@ public final class BytesBuffer extends OutputStream {
break;
}
}
- E.checkArgument(foundEnding, "Not found ending '0x%s'",
- Bytes.toHex(STRING_ENDING_BYTE));
+ E.checkArgument(foundEnding, "Not found ending '0x%s'",
Bytes.toHex(STRING_ENDING_BYTE));
int end = this.buffer.position() - 1;
int len = end - start;
if (len <= 0) {
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java
index 53b8a5493..f38c2b067 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeEdge.java
@@ -147,9 +147,9 @@ public class HugeEdge extends HugeElement implements Edge,
Cloneable {
if (this.fresh()) {
int len = this.id.length();
- E.checkArgument(len <= BytesBuffer.BIG_ID_LEN_MAX,
+ E.checkArgument(len <= BytesBuffer.EID_LEN_MAX,
"The max length of edge id is %s, but got %s {%s}",
- BytesBuffer.BIG_ID_LEN_MAX, len, this.id);
+ BytesBuffer.EID_LEN_MAX, len, this.id);
}
}
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java
index 137e623d8..adb12f31b 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeElement.java
@@ -55,7 +55,6 @@ public abstract class HugeElement implements Element,
GraphType, Idfiable, Compa
private static final MutableIntObjectMap<HugeProperty<?>> EMPTY_MAP =
CollectionFactory.newIntObjectMap();
- private static final int MAX_PROPERTIES = BytesBuffer.UINT16_MAX;
private final HugeGraph graph;
private MutableIntObjectMap<HugeProperty<?>> properties;
@@ -279,7 +278,7 @@ public abstract class HugeElement implements Element,
GraphType, Idfiable, Compa
PropertyKey pkey = prop.propertyKey();
E.checkArgument(this.properties.containsKey(intFromId(pkey.id())) ||
- this.properties.size() < MAX_PROPERTIES,
+ this.properties.size() < BytesBuffer.MAX_PROPERTIES,
"Exceeded the maximum number of properties");
return this.properties.put(intFromId(pkey.id()), prop);
}
diff --git
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java
index c07b9e8d4..f9e4f120f 100644
---
a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java
+++
b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/HugeTask.java
@@ -725,7 +725,7 @@ public class HugeTask<V> extends FutureTask<V> {
}
private void checkPropertySize(int propertyLength, String propertyName) {
- long propertyLimit = BytesBuffer.STRING_LEN_MAX;
+ long propertyLimit = BytesBuffer.MAX_PROPERTIES;
HugeGraph graph = this.scheduler().graph();
if (propertyName.equals(P.INPUT)) {
propertyLimit = graph.option(CoreOptions.TASK_INPUT_SIZE_LIMIT);
diff --git
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java
index b79cae196..265d40874 100644
---
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java
+++
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/EdgeCoreTest.java
@@ -75,7 +75,6 @@ import
org.apache.tinkerpop.gremlin.structure.util.CloseableIterator;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.junit.Assume;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
@@ -470,12 +469,13 @@ public class EdgeCoreTest extends BaseCoreTest {
Vertex book = graph.addVertex(T.label, "book", "name", "Test-Book-1");
Assert.assertThrows(IllegalArgumentException.class, () -> {
- final int LEN = BytesBuffer.BIG_ID_LEN_MAX;
+ final int LEN = BytesBuffer.EID_LEN_MAX;
String largeTime = "{large-time}" + new String(new byte[LEN]);
james.addEdge("write", book, "time", largeTime);
graph.tx().commit();
}, e -> {
- Assert.assertContains("The max length of edge id is 32768",
+ Assert.assertContains(String.format("The max length of edge id is
%s",
+ BytesBuffer.EID_LEN_MAX),
e.getMessage());
});
}
diff --git
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java
index c9a83ddc1..a329de3af 100644
---
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java
+++
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java
@@ -44,6 +44,7 @@ import org.apache.hugegraph.backend.page.PageInfo;
import org.apache.hugegraph.backend.query.Condition;
import org.apache.hugegraph.backend.query.ConditionQuery;
import org.apache.hugegraph.backend.query.Query;
+import org.apache.hugegraph.backend.serializer.BytesBuffer;
import org.apache.hugegraph.backend.store.BackendTable;
import org.apache.hugegraph.backend.store.Shard;
import org.apache.hugegraph.backend.tx.GraphTransaction;
@@ -1039,10 +1040,10 @@ public class VertexCoreTest extends BaseCoreTest {
"name", "marko", "age", 18, "city", "Beijing");
});
- // Expect id length <= 128
+ // Expect id length <= BytesBuffer.ID_LEN_MAX
Assert.assertThrows(IllegalArgumentException.class, () -> {
- String largeId = new String(new byte[128]) + ".";
- assert largeId.length() == 129;
+ String largeId = new String(new byte[BytesBuffer.ID_LEN_MAX]) +
".";
+ assert largeId.length() == BytesBuffer.ID_LEN_MAX + 1;
graph.addVertex(T.label, "programmer", T.id, largeId,
"name", "marko", "age", 18, "city", "Beijing");
});
diff --git
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java
index 63837c8aa..d9bd4cf95 100644
---
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java
+++
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CacheTest.java
@@ -139,6 +139,7 @@ public abstract class CacheTest extends BaseUnitTest {
public static class OffheapCacheTest extends CacheTest {
private static final long ENTRY_SIZE = 40L;
+ private static final int SEGMENTS = 4;
private final HugeGraph graph = Mockito.mock(HugeGraph.class);
@Override
@@ -148,7 +149,7 @@ public abstract class CacheTest extends BaseUnitTest {
@Override
protected Cache<Id, Object> newCache(long capacity) {
- return new OffheapCache(this.graph(), capacity, ENTRY_SIZE);
+ return new OffheapCache(this.graph(), capacity, ENTRY_SIZE,
SEGMENTS);
}
@Override
@@ -325,7 +326,7 @@ public abstract class CacheTest extends BaseUnitTest {
@Test
public void testUpdateAndGetWithSameSizeAndCapacity() {
- int limit = 40;
+ int limit = 1000;
Cache<Id, Object> cache = newCache(limit);
Map<Id, Object> map = new LimitMap(limit);
@@ -451,7 +452,7 @@ public abstract class CacheTest extends BaseUnitTest {
@Test
public void testSizeWithReachCapacity() {
- int limit = 20;
+ int limit = 1000;
Cache<Id, Object> cache = newCache(limit);
Map<Id, Object> map = new LimitMap(limit);
diff --git
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java
index fa582322a..fe9925b2b 100644
---
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java
+++
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/id/IdUtilTest.java
@@ -154,4 +154,22 @@ public class IdUtilTest {
}
return bytes;
}
+
+ /**
+ * Converts a byte array to a hexadecimal string.
+ *
+ * @param bytes the byte array to convert
+ * @return the hexadecimal string representation of the byte array
+ */
+ private String bytesToHex(byte[] bytes) {
+ StringBuilder hexString = new StringBuilder();
+ for (byte b : bytes) {
+ String hex = Integer.toHexString(0xFF & b);
+ if (hex.length() == 1) {
+ hexString.append('0'); // pad with leading zero if needed
+ }
+ hexString.append(hex);
+ }
+ return hexString.toString();
+ }
}
diff --git
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java
index 907265290..fb25a5b25 100644
---
a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java
+++
b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/serializer/BytesBufferTest.java
@@ -96,29 +96,35 @@ public class BytesBufferTest extends BaseUnitTest {
@Test
public void testStringIdWithBigSize() {
Id id = IdGenerator.of(genString(127));
- byte[] bytes = genBytes(128);
- bytes[0] = (byte) 0xfe;
+ byte[] bytes = genBytes(129);
+ bytes[0] = (byte) 0xc0;
+ bytes[1] = (byte) 0x7e;
Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0)
.writeId(id).bytes());
Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId());
id = IdGenerator.of(genString(128));
- bytes = genBytes(129);
- bytes[0] = (byte) 0xff;
+ bytes = genBytes(130);
+ bytes[0] = (byte) 0xc0;
+ bytes[1] = (byte) 0x7f;
Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0)
.writeId(id).bytes());
Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId());
Assert.assertThrows(IllegalArgumentException.class, () -> {
- BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(129)));
+
BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX
+ 1)));
}, e -> {
- Assert.assertContains("Id max length is 128, but got 129",
+ Assert.assertContains(String.format("Big id max length is %s, but
got %s",
+ BytesBuffer.ID_LEN_MAX,
+ BytesBuffer.ID_LEN_MAX + 1),
e.getMessage());
});
Assert.assertThrows(IllegalArgumentException.class, () -> {
- BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(130)));
+
BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX
+ 2)));
}, e -> {
- Assert.assertContains("Id max length is 128, but got 130",
+ Assert.assertContains(String.format("Big id max length is %s, but
got %s",
+ BytesBuffer.ID_LEN_MAX,
+ BytesBuffer.ID_LEN_MAX + 2),
e.getMessage());
});
}
@@ -127,33 +133,34 @@ public class BytesBufferTest extends BaseUnitTest {
public void testStringBigId() {
Id id = IdGenerator.of(genString(128));
byte[] bytes = genBytes(130);
- bytes[0] = (byte) 0x80;
+ bytes[0] = (byte) 0xc0;
bytes[1] = (byte) 0x7f;
Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0)
- .writeId(id, true).bytes());
- Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true));
+ .writeId(id).bytes());
+ Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId());
- id = IdGenerator.of(genString(32512));
- bytes = genBytes(32514);
- bytes[0] = (byte) 0xfe;
- bytes[1] = (byte) 0xff;
+ id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX - 1));
+ bytes = genBytes(BytesBuffer.ID_LEN_MAX + 1);
+ bytes[0] = (byte) 0xff;
+ bytes[1] = (byte) 0xfe;
Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0)
- .writeId(id, true).bytes());
- Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true));
+ .writeId(id).bytes());
+ Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId());
- id = IdGenerator.of(genString(32768));
- bytes = genBytes(32770);
+ id = IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX));
+ bytes = genBytes(BytesBuffer.ID_LEN_MAX + 2);
bytes[0] = (byte) 0xff;
bytes[1] = (byte) 0xff;
Assert.assertArrayEquals(bytes, BytesBuffer.allocate(0)
- .writeId(id, true).bytes());
- Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId(true));
+ .writeId(id).bytes());
+ Assert.assertEquals(id, BytesBuffer.wrap(bytes).readId());
Assert.assertThrows(IllegalArgumentException.class, () -> {
- BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(32769)),
- true);
+
BytesBuffer.allocate(0).writeId(IdGenerator.of(genString(BytesBuffer.ID_LEN_MAX
+ 1)));
}, e -> {
- Assert.assertContains("Big id max length is 32768, but got 32769",
+ Assert.assertContains(String.format("Big id max length is %s, but
got %s",
+ BytesBuffer.ID_LEN_MAX,
+ BytesBuffer.ID_LEN_MAX + 1),
e.getMessage());
});
}
