This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b826c53 [HUDI-1373] Add Support for OpenJ9 JVM (#2231)
b826c53 is described below
commit b826c53e331fa5a80b7f9505dcdee0cbf16740e2
Author: Guy Khazma <[email protected]>
AuthorDate: Tue Dec 1 23:19:40 2020 +0200
[HUDI-1373] Add Support for OpenJ9 JVM (#2231)
* add supoort for OpenJ9 VM
* add 32bit openJ9
* Pulled the memory layout specs into their own classes.
---
.../hudi/common/util/ObjectSizeCalculator.java | 186 ++++++---------------
.../jvm/HotSpotMemoryLayoutSpecification32bit.java | 46 +++++
.../jvm/HotSpotMemoryLayoutSpecification64bit.java | 46 +++++
...otMemoryLayoutSpecification64bitCompressed.java | 46 +++++
.../common/util/jvm/MemoryLayoutSpecification.java | 60 +++++++
.../jvm/OpenJ9MemoryLayoutSpecification32bit.java | 46 +++++
.../jvm/OpenJ9MemoryLayoutSpecification64bit.java | 46 +++++
...J9MemoryLayoutSpecification64bitCompressed.java | 46 +++++
8 files changed, 387 insertions(+), 135 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java
index 88587c7..f394415 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java
@@ -18,6 +18,14 @@
package org.apache.hudi.common.util;
+import org.apache.hudi.common.util.jvm.MemoryLayoutSpecification;
+import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification32bit;
+import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bit;
+import
org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bitCompressed;
+import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification32bit;
+import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bit;
+import
org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bitCompressed;
+
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.lang.reflect.Array;
@@ -35,7 +43,7 @@ import java.util.Objects;
import java.util.Set;
/**
- * Contains utility methods for calculating the memory usage of objects. It
only works on the HotSpot JVM, and infers
+ * Contains utility methods for calculating the memory usage of objects. It
only works on the HotSpot and OpenJ9 JVMs, and infers
* the actual memory layout (32 bit vs. 64 bit word size, compressed object
pointers vs. uncompressed) from best
* available indicators. It can reliably detect a 32 bit vs. 64 bit JVM. It
can only make an educated guess at whether
* compressed OOPs are used, though; specifically, it knows what the JVM's
default choice of OOP compression would be
@@ -46,48 +54,6 @@ import java.util.Set;
* @author Attila Szegedi
*/
public class ObjectSizeCalculator {
-
- /**
- * Describes constant memory overheads for various constructs in a JVM
implementation.
- */
- public interface MemoryLayoutSpecification {
-
- /**
- * Returns the fixed overhead of an array of any type or length in this
JVM.
- *
- * @return the fixed overhead of an array.
- */
- int getArrayHeaderSize();
-
- /**
- * Returns the fixed overhead of for any {@link Object} subclass in this
JVM.
- *
- * @return the fixed overhead of any object.
- */
- int getObjectHeaderSize();
-
- /**
- * Returns the quantum field size for a field owned by an object in this
JVM.
- *
- * @return the quantum field size for an object.
- */
- int getObjectPadding();
-
- /**
- * Returns the fixed size of an object reference in this JVM.
- *
- * @return the size of all object references.
- */
- int getReferenceSize();
-
- /**
- * Returns the quantum field size for a field owned by one of an object's
ancestor superclasses in this JVM.
- *
- * @return the quantum field size for a superclass field.
- */
- int getSuperclassFieldPadding();
- }
-
private static class CurrentLayout {
private static final MemoryLayoutSpecification SPEC =
getEffectiveMemoryLayoutSpecification();
@@ -328,109 +294,59 @@ public class ObjectSizeCalculator {
static MemoryLayoutSpecification getEffectiveMemoryLayoutSpecification() {
final String vmName = System.getProperty("java.vm.name");
if (vmName == null || !(vmName.startsWith("Java HotSpot(TM) ") ||
vmName.startsWith("OpenJDK")
- || vmName.startsWith("TwitterJDK"))) {
- throw new UnsupportedOperationException("ObjectSizeCalculator only
supported on HotSpot VM");
- }
-
- final String dataModel = System.getProperty("sun.arch.data.model");
- if ("32".equals(dataModel)) {
- // Running with 32-bit data model
- return new MemoryLayoutSpecification() {
- @Override
- public int getArrayHeaderSize() {
- return 12;
- }
-
- @Override
- public int getObjectHeaderSize() {
- return 8;
- }
-
- @Override
- public int getObjectPadding() {
- return 8;
- }
-
- @Override
- public int getReferenceSize() {
- return 4;
- }
-
- @Override
- public int getSuperclassFieldPadding() {
- return 4;
- }
- };
- } else if (!"64".equals(dataModel)) {
- throw new UnsupportedOperationException(
- "Unrecognized value '" + dataModel + "' of sun.arch.data.model
system property");
+ || vmName.startsWith("TwitterJDK") || vmName.startsWith("Eclipse
OpenJ9"))) {
+ throw new UnsupportedOperationException("ObjectSizeCalculator only
supported on HotSpot or Eclipse OpenJ9 VMs");
}
final String strVmVersion = System.getProperty("java.vm.version");
- final int vmVersion = Integer.parseInt(strVmVersion.substring(0,
strVmVersion.indexOf('.')));
- if (vmVersion >= 17) {
+ // Support for OpenJ9 JVM
+ if (strVmVersion.startsWith("openj9")) {
+ final String dataModel = System.getProperty("sun.arch.data.model");
+ if ("32".equals(dataModel)) {
+ // Running with 32-bit data model
+ return new OpenJ9MemoryLayoutSpecification32bit();
+ } else if (!"64".equals(dataModel)) {
+ throw new UnsupportedOperationException(
+ "Unrecognized value '" + dataModel + "' of sun.arch.data.model
system property");
+ }
+
long maxMemory = 0;
for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) {
maxMemory += mp.getUsage().getMax();
}
- if (maxMemory < 30L * 1024 * 1024 * 1024) {
- // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total
- // for all memory pools (yes, including code cache).
- return new MemoryLayoutSpecification() {
- @Override
- public int getArrayHeaderSize() {
- return 16;
- }
-
- @Override
- public int getObjectHeaderSize() {
- return 12;
- }
-
- @Override
- public int getObjectPadding() {
- return 8;
- }
-
- @Override
- public int getReferenceSize() {
- return 4;
- }
-
- @Override
- public int getSuperclassFieldPadding() {
- return 4;
- }
- };
- }
- }
-
- // In other cases, it's a 64-bit uncompressed OOPs object model
- return new MemoryLayoutSpecification() {
- @Override
- public int getArrayHeaderSize() {
- return 24;
- }
-
- @Override
- public int getObjectHeaderSize() {
- return 16;
+ if (maxMemory < 57L * 1024 * 1024 * 1024) {
+ // OpenJ9 use compressed references below 57GB of RAM total
+ return new OpenJ9MemoryLayoutSpecification64bitCompressed();
+ } else {
+ // it's a 64-bit uncompressed references object model
+ return new OpenJ9MemoryLayoutSpecification64bit();
}
-
- @Override
- public int getObjectPadding() {
- return 8;
+ } else {
+ // Support for HotSpot JVM
+ final String dataModel = System.getProperty("sun.arch.data.model");
+ if ("32".equals(dataModel)) {
+ // Running with 32-bit data model
+ return new HotSpotMemoryLayoutSpecification32bit();
+ } else if (!"64".equals(dataModel)) {
+ throw new UnsupportedOperationException(
+ "Unrecognized value '" + dataModel + "' of sun.arch.data.model
system property");
}
- @Override
- public int getReferenceSize() {
- return 8;
+ final int vmVersion = Integer.parseInt(strVmVersion.substring(0,
strVmVersion.indexOf('.')));
+ if (vmVersion >= 17) {
+ long maxMemory = 0;
+ for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) {
+ maxMemory += mp.getUsage().getMax();
+ }
+ if (maxMemory < 30L * 1024 * 1024 * 1024) {
+ // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total
+ // for all memory pools (yes, including code cache).
+ return new HotSpotMemoryLayoutSpecification64bitCompressed();
+ }
}
- @Override
- public int getSuperclassFieldPadding() {
- return 8;
- }
- };
+ // In other cases, it's a 64-bit uncompressed OOPs object model
+ return new HotSpotMemoryLayoutSpecification64bit();
+ }
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java
new file mode 100644
index 0000000..e76c67e
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification32bit implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 12;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 8;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 8;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 4;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 4;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java
new file mode 100644
index 0000000..4f4e2a4
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification64bit implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 24;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 8;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 8;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 8;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java
new file mode 100644
index 0000000..60ad8be
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification64bitCompressed implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 12;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 8;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 4;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 4;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java
new file mode 100644
index 0000000..fe6d421
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+/**
+ * Describes constant memory overheads for various constructs in a JVM
implementation.
+ */
+public interface MemoryLayoutSpecification {
+
+ /**
+ * Returns the fixed overhead of an array of any type or length in this JVM.
+ *
+ * @return the fixed overhead of an array.
+ */
+ int getArrayHeaderSize();
+
+ /**
+ * Returns the fixed overhead of for any {@link Object} subclass in this JVM.
+ *
+ * @return the fixed overhead of any object.
+ */
+ int getObjectHeaderSize();
+
+ /**
+ * Returns the quantum field size for a field owned by an object in this JVM.
+ *
+ * @return the quantum field size for an object.
+ */
+ int getObjectPadding();
+
+ /**
+ * Returns the fixed size of an object reference in this JVM.
+ *
+ * @return the size of all object references.
+ */
+ int getReferenceSize();
+
+ /**
+ * Returns the quantum field size for a field owned by one of an object's
ancestor superclasses in this JVM.
+ *
+ * @return the quantum field size for a superclass field.
+ */
+ int getSuperclassFieldPadding();
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java
new file mode 100644
index 0000000..1e66e0c
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification32bit implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 4;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 4;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 4;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 4;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java
new file mode 100644
index 0000000..d89d05a
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification64bit implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 8;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 8;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 8;
+ }
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java
new file mode 100644
index 0000000..05a4575
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification64bitCompressed implements
MemoryLayoutSpecification {
+ @Override
+ public int getArrayHeaderSize() {
+ return 16;
+ }
+
+ @Override
+ public int getObjectHeaderSize() {
+ return 4;
+ }
+
+ @Override
+ public int getObjectPadding() {
+ return 4;
+ }
+
+ @Override
+ public int getReferenceSize() {
+ return 4;
+ }
+
+ @Override
+ public int getSuperclassFieldPadding() {
+ return 4;
+ }
+}
