This is an automated email from the ASF dual-hosted git repository. dcromberge pushed a commit to branch Jdk17_Panama_David in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git
commit 1711e8992dc7c2ee913be0b92656c4edd6563b1b Author: David Cromberge <[email protected]> AuthorDate: Mon Dec 20 19:47:06 2021 +0000 Dummy implementation --- .../memory/internal/BaseStateImpl.java | 497 +++++++++++++++++++++ .../apache/datasketches/memory/internal/Dummy.java | 27 -- .../datasketches/memory/internal/UnsafeUtil.java | 228 ++++++++++ .../apache/datasketches/memory/internal/Util.java | 356 +++++++++++++++ 4 files changed, 1081 insertions(+), 27 deletions(-) diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java new file mode 100644 index 0000000..bb03e55 --- /dev/null +++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.internal; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.datasketches.memory.BaseState; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.ReadOnlyException; +import org.apache.datasketches.memory.internal.UnsafeUtil; + +import static org.apache.datasketches.memory.internal.UnsafeUtil.checkBounds; +import static org.apache.datasketches.memory.internal.UnsafeUtil.assertBounds; +import static org.apache.datasketches.memory.internal.UnsafeUtil.unsafe; + +/** + * Keeps key configuration state for MemoryImpl and BufferImpl plus some common static variables + * and check methods. + * + * @author Lee Rhodes + */ +@SuppressWarnings("restriction") +public abstract class BaseStateImpl implements BaseState { + + //Monitoring + static final AtomicLong currentDirectMemoryAllocations_ = new AtomicLong(); + static final AtomicLong currentDirectMemoryAllocated_ = new AtomicLong(); + static final AtomicLong currentDirectMemoryMapAllocations_ = new AtomicLong(); + static final AtomicLong currentDirectMemoryMapAllocated_ = new AtomicLong(); + static final String LS = System.getProperty("line.separator"); + + //class type IDs. Do not change the bit orders + //The first 3 bits are set dynamically + // 0000 0XXX + static final int READONLY = 1; + static final int REGION = 2; + static final int DUPLICATE = 4; + + //The following 4 bits are set by the 16 leaf nodes + // 000X X000 + static final int HEAP = 0; + static final int DIRECT = 1 << 3; + static final int MAP = 2 << 3; + static final int BYTEBUF = 3 << 3; + + // 00X0 0000 + static final int NATIVE = 0; + static final int NONNATIVE = 1 << 5; + + // 0X00 0000 + static final int MEMORY = 0; + static final int BUFFER = 1 << 6; + + private final long capacityBytes_; + + /** + * This becomes the base offset used by all Unsafe calls. It is cumulative in that in includes + * all offsets from regions, user-defined offsets when creating MemoryImpl, and the array object + * header offset when creating MemoryImpl from primitive arrays. + */ + private final long cumBaseOffset_; + + /** + * + * @param unsafeObj The primitive backing array. It may be null. Used by Unsafe calls. + * @param nativeBaseOffset The off-heap memory address including DirectByteBuffer split offsets. + * @param regionOffset This offset defines address zero of this object (usually a region) + * relative to address zero of the backing resource. It is used to compute cumBaseOffset. + * This will be loaded from heap ByteBuffers, which have a similar field used for slices. + * It is used by region() and writableRegion(). + * This offset does not include the size of an object array header, if there is one. + * @param capacityBytes the capacity of this object. Used by all methods when checking bounds. + */ + BaseStateImpl(final Object unsafeObj, final long nativeBaseOffset, final long regionOffset, + final long capacityBytes) { + capacityBytes_ = capacityBytes; + cumBaseOffset_ = regionOffset + (unsafeObj == null + ? nativeBaseOffset + : UnsafeUtil.getArrayBaseOffset(unsafeObj.getClass())); + } + + //Byte Order Related + + @Override + public final ByteOrder getTypeByteOrder() { + return isNonNativeType() ? Util.NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder(); + } + + /** + * Returns true if the given byteOrder is the same as the native byte order. + * @param byteOrder the given byte order + * @return true if the given byteOrder is the same as the native byte order. + */ + public static boolean isNativeByteOrder(final ByteOrder byteOrder) { + if (byteOrder == null) { + throw new IllegalArgumentException("ByteOrder parameter cannot be null."); + } + return ByteOrder.nativeOrder() == byteOrder; + } + + @Override + public final boolean isByteOrderCompatible(final ByteOrder byteOrder) { + final ByteOrder typeBO = getTypeByteOrder(); + return typeBO == ByteOrder.nativeOrder() && typeBO == byteOrder; + } + + @Override + public final boolean equals(final Object that) { + if (this == that) { return true; } + return that instanceof BaseStateImpl + ? CompareAndCopy.equals(this, (BaseStateImpl) that) + : false; + } + + @Override + public final boolean equalTo(final long thisOffsetBytes, final Object that, + final long thatOffsetBytes, final long lengthBytes) { + return that instanceof BaseStateImpl + ? CompareAndCopy.equals(this, thisOffsetBytes, (BaseStateImpl) that, thatOffsetBytes, lengthBytes) + : false; + } + + //Overridden by ByteBuffer Leafs + @Override + public ByteBuffer getByteBuffer() { + return null; + } + + @Override + public final long getCapacity() { + assertValid(); + return capacityBytes_; + } + + @Override + public final long getCumulativeOffset() { + assertValid(); + return cumBaseOffset_; + } + + @Override + public final long getCumulativeOffset(final long offsetBytes) { + assertValid(); + return cumBaseOffset_ + offsetBytes; + } + + //Documented in WritableMemory and WritableBuffer interfaces. + //Implemented in the Leaf nodes; Required here by toHex(...). + abstract MemoryRequestServer getMemoryRequestServer(); + + //Overridden by ByteBuffer, Direct and Map leafs + long getNativeBaseOffset() { + return 0; + } + + @Override + public final long getRegionOffset() { + final Object unsafeObj = getUnsafeObject(); + return unsafeObj == null + ? cumBaseOffset_ - getNativeBaseOffset() + : cumBaseOffset_ - UnsafeUtil.getArrayBaseOffset(unsafeObj.getClass()); + } + + @Override + public final long getRegionOffset(final long offsetBytes) { + return getRegionOffset() + offsetBytes; + } + + //Overridden by all leafs + abstract int getTypeId(); + + //Overridden by Heap and ByteBuffer Leafs. Made public as getArray() in WritableMemoryImpl and + // WritableBufferImpl + Object getUnsafeObject() { + return null; + } + + @Override + public final boolean hasArray() { + assertValid(); + return getUnsafeObject() != null; + } + + @Override + public final int hashCode() { + return (int) xxHash64(0, capacityBytes_, 0); //xxHash64() calls checkValid() + } + + @Override + public final long xxHash64(final long offsetBytes, final long lengthBytes, final long seed) { + checkValid(); + return XxHash64.hash(getUnsafeObject(), cumBaseOffset_ + offsetBytes, lengthBytes, seed); + } + + @Override + public final long xxHash64(final long in, final long seed) { + return XxHash64.hash(in, seed); + } + + @Override + public final boolean hasByteBuffer() { + assertValid(); + return getByteBuffer() != null; + } + + @Override + public final boolean isDirect() { + return getUnsafeObject() == null; + } + + @Override + public final boolean isReadOnly() { + assertValid(); + return isReadOnlyType(); + } + + @Override + public final boolean isSameResource(final Object that) { + checkValid(); + if (that == null) { return false; } + final BaseStateImpl that1 = (BaseStateImpl) that; + that1.checkValid(); + if (this == that1) { return true; } + + return cumBaseOffset_ == that1.cumBaseOffset_ + && capacityBytes_ == that1.capacityBytes_ + && getUnsafeObject() == that1.getUnsafeObject() + && getByteBuffer() == that1.getByteBuffer(); + } + + //Overridden by Direct and Map leafs + @Override + public boolean isValid() { + return true; + } + + //ASSERTS AND CHECKS + final void assertValid() { + assert isValid() : "MemoryImpl not valid."; + } + + void checkValid() { + if (!isValid()) { + throw new IllegalStateException("MemoryImpl not valid."); + } + } + + final void assertValidAndBoundsForRead(final long offsetBytes, final long lengthBytes) { + assertValid(); + // capacityBytes_ is intentionally read directly instead of calling getCapacity() + // because the later can make JVM to not inline the assert code path (and entirely remove it) + // even though it does nothing in production code path. + assertBounds(offsetBytes, lengthBytes, capacityBytes_); + } + + final void assertValidAndBoundsForWrite(final long offsetBytes, final long lengthBytes) { + assertValid(); + // capacityBytes_ is intentionally read directly instead of calling getCapacity() + // because the later can make JVM to not inline the assert code path (and entirely remove it) + // even though it does nothing in production code path. + assertBounds(offsetBytes, lengthBytes, capacityBytes_); + assert !isReadOnly() : "MemoryImpl is read-only."; + } + + @Override + public final void checkValidAndBounds(final long offsetBytes, final long lengthBytes) { + checkValid(); + //read capacityBytes_ directly to eliminate extra checkValid() call + checkBounds(offsetBytes, lengthBytes, capacityBytes_); + } + + final void checkValidAndBoundsForWrite(final long offsetBytes, final long lengthBytes) { + checkValid(); + //read capacityBytes_ directly to eliminate extra checkValid() call + checkBounds(offsetBytes, lengthBytes, capacityBytes_); + if (isReadOnly()) { + throw new ReadOnlyException("MemoryImpl is read-only."); + } + } + + //TYPE ID Management + final boolean isReadOnlyType() { + return (getTypeId() & READONLY) > 0; + } + + final static byte setReadOnlyType(final byte type, final boolean readOnly) { + return (byte)((type & ~1) | (readOnly ? READONLY : 0)); + } + + final boolean isRegionType() { + return (getTypeId() & REGION) > 0; + } + + final boolean isDuplicateType() { + return (getTypeId() & DUPLICATE) > 0; + } + + //The following are set by the leaf nodes + final boolean isBufferType() { + return (getTypeId() & BUFFER) > 0; + } + + final boolean isNonNativeType() { + return (getTypeId() & NONNATIVE) > 0; + } + + final boolean isHeapType() { + return (getTypeId() >>> 3 & 3) == 0; + } + + final boolean isDirectType() { + return (getTypeId() >>> 3 & 3) == 1; + } + + final boolean isMapType() { + return (getTypeId() >>> 3 & 3) == 2; + } + + final boolean isBBType() { + return (getTypeId() >>> 3 & 3) == 3; + } + + + //TO STRING + /** + * Decodes the resource type. This is primarily for debugging. + * @param typeId the given typeId + * @return a human readable string. + */ + public static final String typeDecode(final int typeId) { + final StringBuilder sb = new StringBuilder(); + final int group1 = typeId & 0x7; + switch (group1) { + case 1 : sb.append("ReadOnly, "); break; + case 2 : sb.append("Region, "); break; + case 3 : sb.append("ReadOnly Region, "); break; + case 4 : sb.append("Duplicate, "); break; + case 5 : sb.append("ReadOnly Duplicate, "); break; + case 6 : sb.append("Region Duplicate, "); break; + case 7 : sb.append("ReadOnly Region Duplicate, "); break; + default: break; + } + final int group2 = (typeId >>> 3) & 0x3; + switch (group2) { + case 0 : sb.append("Heap, "); break; + case 1 : sb.append("Direct, "); break; + case 2 : sb.append("Map, "); break; + case 3 : sb.append("ByteBuffer, "); break; + default: break; + } + final int group3 = (typeId >>> 5) & 0x1; + switch (group3) { + case 0 : sb.append("Native, "); break; + case 1 : sb.append("NonNative, "); break; + default: break; + } + final int group4 = (typeId >>> 6) & 0x1; + switch (group4) { + case 0 : sb.append("Memory"); break; + case 1 : sb.append("Buffer"); break; + default: break; + } + return sb.toString(); + } + + @Override + public final String toHexString(final String header, final long offsetBytes, + final int lengthBytes) { + checkValid(); + final String klass = this.getClass().getSimpleName(); + final String s1 = String.format("(..., %d, %d)", offsetBytes, lengthBytes); + final long hcode = hashCode() & 0XFFFFFFFFL; + final String call = ".toHexString" + s1 + ", hashCode: " + hcode; + final StringBuilder sb = new StringBuilder(); + sb.append("### ").append(klass).append(" SUMMARY ###").append(LS); + sb.append("Header Comment : ").append(header).append(LS); + sb.append("Call Parameters : ").append(call); + return toHex(this, sb.toString(), offsetBytes, lengthBytes); + } + + /** + * Returns a formatted hex string of an area of this object. + * Used primarily for testing. + * @param state the BaseStateImpl + * @param preamble a descriptive header + * @param offsetBytes offset bytes relative to the MemoryImpl start + * @param lengthBytes number of bytes to convert to a hex string + * @return a formatted hex string in a human readable array + */ + static final String toHex(final BaseStateImpl state, final String preamble, final long offsetBytes, + final int lengthBytes) { + final long capacity = state.getCapacity(); + checkBounds(offsetBytes, lengthBytes, capacity); + final StringBuilder sb = new StringBuilder(); + final Object uObj = state.getUnsafeObject(); + final String uObjStr; + final long uObjHeader; + if (uObj == null) { + uObjStr = "null"; + uObjHeader = 0; + } else { + uObjStr = uObj.getClass().getSimpleName() + ", " + (uObj.hashCode() & 0XFFFFFFFFL); + uObjHeader = UnsafeUtil.getArrayBaseOffset(uObj.getClass()); + } + final ByteBuffer bb = state.getByteBuffer(); + final String bbStr = bb == null ? "null" + : bb.getClass().getSimpleName() + ", " + (bb.hashCode() & 0XFFFFFFFFL); + final MemoryRequestServer memReqSvr = state.getMemoryRequestServer(); + final String memReqStr = memReqSvr != null + ? memReqSvr.getClass().getSimpleName() + ", " + (memReqSvr.hashCode() & 0XFFFFFFFFL) + : "null"; + final long cumBaseOffset = state.getCumulativeOffset(); + sb.append(preamble).append(LS); + sb.append("UnsafeObj, hashCode : ").append(uObjStr).append(LS); + sb.append("UnsafeObjHeader : ").append(uObjHeader).append(LS); + sb.append("ByteBuf, hashCode : ").append(bbStr).append(LS); + sb.append("RegionOffset : ").append(state.getRegionOffset()).append(LS); + sb.append("Capacity : ").append(capacity).append(LS); + sb.append("CumBaseOffset : ").append(cumBaseOffset).append(LS); + sb.append("MemReq, hashCode : ").append(memReqStr).append(LS); + sb.append("Valid : ").append(state.isValid()).append(LS); + sb.append("Read Only : ").append(state.isReadOnly()).append(LS); + sb.append("Type Byte Order : ").append(state.getTypeByteOrder().toString()).append(LS); + sb.append("Native Byte Order : ").append(ByteOrder.nativeOrder().toString()).append(LS); + sb.append("JDK Runtime Version : ").append(UnsafeUtil.JDK).append(LS); + //Data detail + sb.append("Data, littleEndian : 0 1 2 3 4 5 6 7"); + + for (long i = 0; i < lengthBytes; i++) { + final int b = unsafe.getByte(uObj, cumBaseOffset + offsetBytes + i) & 0XFF; + if (i % 8 == 0) { //row header + sb.append(String.format("%n%20s: ", offsetBytes + i)); + } + sb.append(String.format("%02x ", b)); + } + sb.append(LS); + + return sb.toString(); + } + + //MONITORING + + /** + * Gets the current number of active direct memory allocations. + * @return the current number of active direct memory allocations. + */ + public static final long getCurrentDirectMemoryAllocations() { + return BaseStateImpl.currentDirectMemoryAllocations_.get(); + } + + /** + * Gets the current size of active direct memory allocated. + * @return the current size of active direct memory allocated. + */ + public static final long getCurrentDirectMemoryAllocated() { + return BaseStateImpl.currentDirectMemoryAllocated_.get(); + } + + /** + * Gets the current number of active direct memory map allocations. + * @return the current number of active direct memory map allocations. + */ + public static final long getCurrentDirectMemoryMapAllocations() { + return BaseStateImpl.currentDirectMemoryMapAllocations_.get(); + } + + /** + * Gets the current size of active direct memory map allocated. + * @return the current size of active direct memory map allocated. + */ + public static final long getCurrentDirectMemoryMapAllocated() { + return BaseStateImpl.currentDirectMemoryMapAllocated_.get(); + } + + //REACHABILITY FENCE + static void reachabilityFence(@SuppressWarnings("unused") final Object obj) { } + +} \ No newline at end of file diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java deleted file mode 100644 index aefdcba..0000000 --- a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.memory.internal; - -/** - * Temporary class & placeholder. It may not be needed. - */ -public class Dummy { - -} diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java new file mode 100644 index 0000000..9165b0d --- /dev/null +++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.internal; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import sun.misc.Unsafe; + +/** + * Provides access to the sun.misc.Unsafe class and its key static fields. + * + * @author Lee Rhodes + */ +@SuppressWarnings({"restriction", "javadoc"}) +public final class UnsafeUtil { + public static final Unsafe unsafe; + public static final String JDK; //must be at least "1.8" + public static final int JDK_MAJOR; //8, 9, 10, 11, 12, etc + + //not an indicator of whether compressed references are used. + public static final int ADDRESS_SIZE; + public static final int PAGE_SIZE; + + //For 64-bit JVMs: these offsets vary depending on coop: 16 for JVM <= 32GB; 24 for JVM > 32GB. + // Making this constant long-typed, rather than int, to exclude possibility of accidental overflow + // in expressions like arrayLength * ARRAY_BYTE_BASE_OFFSET, where arrayLength is int-typed. + // The same consideration for constants below: ARRAY_*_INDEX_SCALE, ARRAY_*_INDEX_SHIFT. + public static final long ARRAY_BOOLEAN_BASE_OFFSET; + public static final long ARRAY_BYTE_BASE_OFFSET; + public static final long ARRAY_SHORT_BASE_OFFSET; + public static final long ARRAY_CHAR_BASE_OFFSET; + public static final long ARRAY_INT_BASE_OFFSET; + public static final long ARRAY_LONG_BASE_OFFSET; + public static final long ARRAY_FLOAT_BASE_OFFSET; + public static final long ARRAY_DOUBLE_BASE_OFFSET; + public static final long ARRAY_OBJECT_BASE_OFFSET; + + //@formatter:off + + // Setting those values directly instead of using unsafe.arrayIndexScale(), because it may be + // beneficial for runtime execution, those values are backed into generated machine code as + // constants. E. g. see https://shipilev.net/jvm-anatomy-park/14-constant-variables/ + public static final int ARRAY_BOOLEAN_INDEX_SCALE = 1; + public static final int ARRAY_BYTE_INDEX_SCALE = 1; + public static final long ARRAY_SHORT_INDEX_SCALE = 2; + public static final long ARRAY_CHAR_INDEX_SCALE = 2; + public static final long ARRAY_INT_INDEX_SCALE = 4; + public static final long ARRAY_LONG_INDEX_SCALE = 8; + public static final long ARRAY_FLOAT_INDEX_SCALE = 4; + public static final long ARRAY_DOUBLE_INDEX_SCALE = 8; + public static final long ARRAY_OBJECT_INDEX_SCALE; // varies, 4 or 8 depending on coop + + //Used to convert "type" to bytes: bytes = longs << LONG_SHIFT + public static final int BOOLEAN_SHIFT = 0; + public static final int BYTE_SHIFT = 0; + public static final long SHORT_SHIFT = 1; + public static final long CHAR_SHIFT = 1; + public static final long INT_SHIFT = 2; + public static final long LONG_SHIFT = 3; + public static final long FLOAT_SHIFT = 2; + public static final long DOUBLE_SHIFT = 3; + public static final long OBJECT_SHIFT; // varies, 2 or 3 depending on coop + + public static final String LS = System.getProperty("line.separator"); + + //@formatter:on + + static { + try { + final Constructor<Unsafe> unsafeConstructor = Unsafe.class.getDeclaredConstructor(); + unsafeConstructor.setAccessible(true); + unsafe = unsafeConstructor.newInstance(); + + // Alternative, but may not work across different JVMs. + // Field field = Unsafe.class.getDeclaredField("theUnsafe"); + // field.setAccessible(true); + // unsafe = (Unsafe) field.get(null); + + } catch (final InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException e) { + e.printStackTrace(); + throw new RuntimeException("Unable to acquire Unsafe. " + e); + } + + //4 on 32-bit systems. 4 on 64-bit systems < 32GB, otherwise 8. + //This alone is not an indicator of compressed ref (coop) + ADDRESS_SIZE = unsafe.addressSize(); + PAGE_SIZE = unsafe.pageSize(); + + ARRAY_BOOLEAN_BASE_OFFSET = unsafe.arrayBaseOffset(boolean[].class); + ARRAY_BYTE_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class); + ARRAY_SHORT_BASE_OFFSET = unsafe.arrayBaseOffset(short[].class); + ARRAY_CHAR_BASE_OFFSET = unsafe.arrayBaseOffset(char[].class); + ARRAY_INT_BASE_OFFSET = unsafe.arrayBaseOffset(int[].class); + ARRAY_LONG_BASE_OFFSET = unsafe.arrayBaseOffset(long[].class); + ARRAY_FLOAT_BASE_OFFSET = unsafe.arrayBaseOffset(float[].class); + ARRAY_DOUBLE_BASE_OFFSET = unsafe.arrayBaseOffset(double[].class); + ARRAY_OBJECT_BASE_OFFSET = unsafe.arrayBaseOffset(Object[].class); + + ARRAY_OBJECT_INDEX_SCALE = unsafe.arrayIndexScale(Object[].class); + OBJECT_SHIFT = ARRAY_OBJECT_INDEX_SCALE == 4 ? 2 : 3; + + final String jdkVer = System.getProperty("java.version"); + final int[] p = parseJavaVersion(jdkVer); + JDK = p[0] + "." + p[1]; + JDK_MAJOR = (p[0] == 1) ? p[1] : p[0]; + } + + private UnsafeUtil() {} + + /** + * Returns first two number groups of the java version string. + * @param jdkVer the java version string from System.getProperty("java.version"). + * @return first two number groups of the java version string. + */ + public static int[] parseJavaVersion(final String jdkVer) { + final int p0, p1; + try { + String[] parts = jdkVer.trim().split("[^0-9\\.]");//grab only number groups and "." + parts = parts[0].split("\\."); //split out the number groups + p0 = Integer.parseInt(parts[0]); //the first number group + p1 = (parts.length > 1) ? Integer.parseInt(parts[1]) : 0; //2nd number group, or 0 + } catch (final NumberFormatException | ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException("Improper Java -version string: " + jdkVer + "\n" + e); + } + //checkJavaVersion(jdkVer, p0, p1); //TODO Optional to omit this. + return new int[] {p0, p1}; + } + + public static void checkJavaVersion(final String jdkVer, final int p0, final int p1) { + if ( (p0 < 1) || ((p0 == 1) && (p1 < 8)) || (p0 > 13) ) { + throw new IllegalArgumentException( + "Unsupported JDK Major Version, must be one of 1.8, 8, 9, 10, 11, 12, 13: " + jdkVer); + } + } + + public static long getFieldOffset(final Class<?> c, final String fieldName) { + try { + return unsafe.objectFieldOffset(c.getDeclaredField(fieldName)); + } catch (final NoSuchFieldException e) { + throw new IllegalStateException(e); + } + } + + /** + * Like {@link Unsafe#arrayBaseOffset(Class)}, but caches return values for common array types. + * Useful because calling {@link Unsafe#arrayBaseOffset(Class)} directly incurs more overhead. + * @param c The given Class<?>. + * @return the base-offset + */ + public static long getArrayBaseOffset(final Class<?> c) { + // Ordering here is roughly in order of what we expect to be most popular. + if (c == byte[].class) { + return ARRAY_BYTE_BASE_OFFSET; + } else if (c == int[].class) { + return ARRAY_INT_BASE_OFFSET; + } else if (c == long[].class) { + return ARRAY_LONG_BASE_OFFSET; + } else if (c == float[].class) { + return ARRAY_FLOAT_BASE_OFFSET; + } else if (c == double[].class) { + return ARRAY_DOUBLE_BASE_OFFSET; + } else if (c == boolean[].class) { + return ARRAY_BOOLEAN_BASE_OFFSET; + } else if (c == short[].class) { + return ARRAY_SHORT_BASE_OFFSET; + } else if (c == char[].class) { + return ARRAY_CHAR_BASE_OFFSET; + } else if (c == Object[].class) { + return ARRAY_OBJECT_BASE_OFFSET; + } else { + return unsafe.arrayBaseOffset(c); + } + } + + public static long pageCount(final long bytes) { + return (int)((bytes + PAGE_SIZE) - 1L) / PAGE_SIZE; + } + + /** + * Assert the requested offset and length against the allocated size. + * The invariants equation is: {@code 0 <= reqOff <= reqLen <= reqOff + reqLen <= allocSize}. + * If this equation is violated and assertions are enabled, an {@link AssertionError} will + * be thrown. + * @param reqOff the requested offset + * @param reqLen the requested length + * @param allocSize the allocated size. + */ + public static void assertBounds(final long reqOff, final long reqLen, final long allocSize) { + assert ((reqOff | reqLen | (reqOff + reqLen) | (allocSize - (reqOff + reqLen))) >= 0) : + "reqOffset: " + reqOff + ", reqLength: " + reqLen + + ", (reqOff + reqLen): " + (reqOff + reqLen) + ", allocSize: " + allocSize; + } + + /** + * Check the requested offset and length against the allocated size. + * The invariants equation is: {@code 0 <= reqOff <= reqLen <= reqOff + reqLen <= allocSize}. + * If this equation is violated an {@link IllegalArgumentException} will be thrown. + * @param reqOff the requested offset + * @param reqLen the requested length + * @param allocSize the allocated size. + */ + public static void checkBounds(final long reqOff, final long reqLen, final long allocSize) { + if ((reqOff | reqLen | (reqOff + reqLen) | (allocSize - (reqOff + reqLen))) < 0) { + throw new IllegalArgumentException( + "reqOffset: " + reqOff + ", reqLength: " + reqLen + + ", (reqOff + reqLen): " + (reqOff + reqLen) + ", allocSize: " + allocSize); + } + } +} \ No newline at end of file diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java new file mode 100644 index 0000000..d7e0343 --- /dev/null +++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.internal; + +import static jdk.incubator.foreign.MemoryAccess.getLongAtIndex; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Random; + +import jdk.incubator.foreign.MemorySegment; + +/** + * @author Lee Rhodes + */ +@SuppressWarnings("javadoc") +public final class Util { + public static final String LS = System.getProperty("line.separator"); + + //Byte Order related + public static final ByteOrder NON_NATIVE_BYTE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN + ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + + public static ByteOrder otherByteOrder(final ByteOrder order) { + return (order == ByteOrder.nativeOrder()) ? NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder(); + } + + + /** + * Don't use sun.misc.Unsafe#copyMemory to copy blocks of memory larger than this + * threshold, because internally it doesn't have safepoint polls, that may cause long + * "Time To Safe Point" pauses in the application. This has been fixed in JDK 9 (see + * https://bugs.openjdk.java.net/browse/JDK-8149596 and + * https://bugs.openjdk.java.net/browse/JDK-8141491), but not in JDK 8, so the Memory library + * should keep having this boilerplate as long as it supports Java 8. + * + * <p>A reference to this can be found in java.nio.Bits.</p> + */ + public static final int UNSAFE_COPY_THRESHOLD_BYTES = 1024 * 1024; + + private Util() { } + + //Byte Order Related + + /** + * Returns true if the given byteOrder is the same as the native byte order. + * @param byteOrder the given byte order + * @return true if the given byteOrder is the same as the native byte order. + */ + public static boolean isNativeByteOrder(final ByteOrder byteOrder) { + if (byteOrder == null) { + throw new IllegalArgumentException("ByteOrder parameter cannot be null."); + } + return ByteOrder.nativeOrder() == byteOrder; + } + + + /** + * Searches a range of the specified array of longs for the specified value using the binary + * search algorithm. The range must be sorted method) prior to making this call. + * If it is not sorted, the results are undefined. If the range contains + * multiple elements with the specified value, there is no guarantee which one will be found. + * @param seg the MemorySegment to be searched + * @param fromLongIndex the index of the first long element (inclusive) to be searched + * @param toLongIndex the index of the last long element (exclusive) to be searched + * @param key the value to be searched for + * @return index of the search key, if it is contained in the array within the specified range; + * otherwise, (-(insertion point) - 1). The insertion point is defined as the point at which + * the key would be inserted into the array: the index of the first element in the range greater + * than the key, or toIndex if all elements in the range are less than the specified key. + * Note that this guarantees that the return value will be ≥ 0 if and only if the key is found. + */ + public static long binarySearchLongs(final MemorySegment seg, final long fromLongIndex, + final long toLongIndex, final long key) { + UnsafeUtil.checkBounds(fromLongIndex << 3, (toLongIndex - fromLongIndex) << 3, seg.byteSize()); + long low = fromLongIndex; + long high = toLongIndex - 1L; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midVal = getLongAtIndex(seg, mid); + + if (midVal < key) { low = mid + 1; } + else if (midVal > key) { high = mid - 1; } + else { return mid; } // key found + } + return -(low + 1); // key not found. + } + + /** + * Prepend the given string with zeros. If the given string is equal or greater than the given + * field length, it will be returned without modification. + * @param s the given string + * @param fieldLength desired total field length including the given string + * @return the given string prepended with zeros. + */ + public static final String zeroPad(final String s, final int fieldLength) { + return characterPad(s, fieldLength, '0', false); + } + + /** + * Prepend or postpend the given string with the given character to fill the given field length. + * If the given string is equal or greater than the given field length, it will be returned + * without modification. + * @param s the given string + * @param fieldLength the desired field length + * @param padChar the desired pad character + * @param postpend if true append the pacCharacters to the end of the string. + * @return prepended or postpended given string with the given character to fill the given field + * length. + */ + public static final String characterPad(final String s, final int fieldLength, + final char padChar, final boolean postpend) { + final char[] chArr = s.toCharArray(); + final int sLen = chArr.length; + if (sLen < fieldLength) { + final char[] out = new char[fieldLength]; + final int blanks = fieldLength - sLen; + + if (postpend) { + for (int i = 0; i < sLen; i++) { + out[i] = chArr[i]; + } + for (int i = sLen; i < fieldLength; i++) { + out[i] = padChar; + } + } else { //prepend + for (int i = 0; i < blanks; i++) { + out[i] = padChar; + } + for (int i = blanks; i < fieldLength; i++) { + out[i] = chArr[i - blanks]; + } + } + + return String.valueOf(out); + } + return s; + } + + /** + * Return true if all the masked bits of value are zero + * @param value the value to be tested + * @param bitMask defines the bits of interest + * @return true if all the masked bits of value are zero + */ + public static final boolean isAllBitsClear(final long value, final long bitMask) { + return (~value & bitMask) == bitMask; + } + + /** + * Return true if all the masked bits of value are one + * @param value the value to be tested + * @param bitMask defines the bits of interest + * @return true if all the masked bits of value are one + */ + public static final boolean isAllBitsSet(final long value, final long bitMask) { + return (value & bitMask) == bitMask; + } + + /** + * Return true if any the masked bits of value are zero + * @param value the value to be tested + * @param bitMask defines the bits of interest + * @return true if any the masked bits of value are zero + */ + public static final boolean isAnyBitsClear(final long value, final long bitMask) { + return (~value & bitMask) != 0; + } + + /** + * Return true if any the masked bits of value are one + * @param value the value to be tested + * @param bitMask defines the bits of interest + * @return true if any the masked bits of value are one + */ + public static final boolean isAnyBitsSet(final long value, final long bitMask) { + return (value & bitMask) != 0; + } + + /** + * Creates random valid Character Code Points (as integers). By definition, valid CodePoints + * are integers in the range 0 to Character.MAX_CODE_POINT, and exclude the surrogate values. + * This is used in unit testing and characterization testing of the UTF8 class. Because the + * characterization tools are in a separate package, this must remain public. + * + * @author Lee Rhodes + */ + public static class RandomCodePoints { + private Random rand; // + private static final int ALL_CP = Character.MAX_CODE_POINT + 1; + private static final int MIN_SUR = Character.MIN_SURROGATE; + private static final int MAX_SUR = Character.MAX_SURROGATE; + + /** + * @param deterministic if true, configure java.util.Random with a fixed seed. + */ + public RandomCodePoints(final boolean deterministic) { + rand = deterministic ? new Random(0) : new Random(); + } + + /** + * Fills the given array with random valid Code Points from 0, inclusive, to + * <i>Character.MAX_CODE_POINT</i>, inclusive. + * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to + * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>. + * @param cpArr the array to fill + */ + public final void fillCodePointArray(final int[] cpArr) { + fillCodePointArray(cpArr, 0, ALL_CP); + } + + /** + * Fills the given array with random valid Code Points from <i>startCP</i>, inclusive, to + * <i>endCP</i>, exclusive. + * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to + * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>. + * @param cpArr the array to fill + * @param startCP the starting Code Point, included. + * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000. + */ + public final void fillCodePointArray(final int[] cpArr, final int startCP, final int endCP) { + final int arrLen = cpArr.length; + final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP); + int idx = 0; + while (idx < arrLen) { + final int cp = startCP + rand.nextInt(numCP); + if ((cp >= MIN_SUR) && (cp <= MAX_SUR)) { + continue; + } + cpArr[idx++] = cp; + } + } + + /** + * Return a single valid random Code Point from 0, inclusive, to + * <i>Character.MAX_CODE_POINT</i>, inclusive. + * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to + * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>. + * @return a single valid random CodePoint. + */ + public final int getCodePoint() { + return getCodePoint(0, ALL_CP); + } + + /** + * Return a single valid random Code Point from <i>startCP</i>, inclusive, to + * <i>endCP</i>, exclusive. + * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to + * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>. + * @param startCP the starting Code Point, included. + * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000. + * @return a single valid random CodePoint. + */ + public final int getCodePoint(final int startCP, final int endCP) { + final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP); + while (true) { + final int cp = startCP + rand.nextInt(numCP); + if ((cp < MIN_SUR) || (cp > MAX_SUR)) { + return cp; + } + } + } + } //End class RandomCodePoints + + public static final void zeroCheck(final long value, final String arg) { + if (value <= 0) { + throw new IllegalArgumentException("The argument '" + arg + "' may not be negative or zero."); + } + } + + public static final void negativeCheck(final long value, final String arg) { + if (value < 0) { + throw new IllegalArgumentException("The argument '" + arg + "' may not be negative."); + } + } + + public static final void nullCheck(final Object obj, final String arg) { + if (obj == null) { + throw new IllegalArgumentException("The argument '" + arg + "' may not be null."); + } + } + + //Resources + + /** + * Gets the absolute path of the given resource file's shortName. + * + * <p>Note that the ClassLoader.getResource(shortName) returns a URL, + * which can have special characters, e.g., "%20" for spaces. This method + * obtains the URL, converts it to a URI, then does a uri.getPath(), which + * decodes any special characters in the URI path. This is required to make + * obtaining resources operating-system independent.</p> + * + * @param shortFileName the last name in the pathname's name sequence. + * @return the absolute path of the given resource file's shortName. + */ + public static String getResourcePath(final String shortFileName) { + try { + final URL url = Util.class.getClassLoader().getResource(shortFileName); + final URI uri = url.toURI(); + final String path = uri.getPath(); //decodes any special characters + return path; + } catch (final NullPointerException | URISyntaxException e) { + throw new IllegalArgumentException("Cannot find resource: " + shortFileName + LS + e); + } + } + + /** + * Gets the file defined by the given resource file's shortFileName. + * @param shortFileName the last name in the pathname's name sequence. + * @return the file defined by the given resource file's shortFileName. + */ + public static File getResourceFile(final String shortFileName) { + return new File(getResourcePath(shortFileName)); + } + + /** + * Returns a byte array of the contents of the file defined by the given resource file's + * shortFileName. + * @param shortFileName the last name in the pathname's name sequence. + * @return a byte array of the contents of the file defined by the given resource file's + * shortFileName. + */ + public static byte[] getResourceBytes(final String shortFileName) { + try { + return Files.readAllBytes(Paths.get(getResourcePath(shortFileName))); + } catch (final IOException e) { + throw new IllegalArgumentException("Cannot read resource: " + shortFileName + LS + e); + } + } + +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
