pitrou commented on a change in pull request #11067: URL: https://github.com/apache/arrow/pull/11067#discussion_r726133702
########## File path: java/ffi/src/main/cpp/jni_wrapper.cc ########## @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <jni.h> + +#include <cassert> +#include <memory> +#include <stdexcept> +#include <string> + +#include "./abi.h" +#include "org_apache_arrow_ffi_jni_JniWrapper.h" + +namespace { + +jclass CreateGlobalClassReference(JNIEnv* env, const char* class_name) { + jclass local_class = env->FindClass(class_name); + jclass global_class = (jclass)env->NewGlobalRef(local_class); + env->DeleteLocalRef(local_class); + return global_class; +} + +jclass illegal_access_exception_class; +jclass illegal_argument_exception_class; +jclass runtime_exception_class; +jclass private_data_class; + +jmethodID private_data_close_method; + +jint JNI_VERSION = JNI_VERSION_1_6; + +class JniPendingException : public std::runtime_error { + public: + explicit JniPendingException(const std::string& arg) : std::runtime_error(arg) {} +}; + +void ThrowPendingException(const std::string& message) { + throw JniPendingException(message); +} + +void JniThrow(std::string message) { ThrowPendingException(message); } + +jmethodID GetMethodID(JNIEnv* env, jclass this_class, const char* name, const char* sig) { + jmethodID ret = env->GetMethodID(this_class, name, sig); + if (ret == nullptr) { + std::string error_message = "Unable to find method " + std::string(name) + + " within signature " + std::string(sig); + ThrowPendingException(error_message); + } + return ret; +} + +class InnerPrivateData { + public: + InnerPrivateData(JavaVM* vm, jobject private_data) + : vm_(vm), j_private_data_(private_data) {} + + JavaVM* vm_; + jobject j_private_data_; +}; + +class JNIEnvGuard { + public: + explicit JNIEnvGuard(JavaVM* vm) : vm_(vm), should_deattach_(false) { + JNIEnv* env; + jint code = vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION); + if (code == JNI_EDETACHED) { + JavaVMAttachArgs args; + args.version = JNI_VERSION; + args.name = NULL; + args.group = NULL; + code = vm->AttachCurrentThread(reinterpret_cast<void**>(&env), &args); + should_deattach_ = (code == JNI_OK); + } + if (code != JNI_OK) { + ThrowPendingException("Failed to attach the current thread to a Java VM"); + } + env_ = env; + } + + JNIEnv* env() { return env_; } + + ~JNIEnvGuard() { + if (should_deattach_) { + vm_->DetachCurrentThread(); + should_deattach_ = false; + } + } + + private: + bool should_deattach_; Review comment: Nit: probably `should_detach_`? ########## File path: java/ffi/src/main/java/org/apache/arrow/ffi/ArrowArray.java ########## @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.ffi; + +import static org.apache.arrow.ffi.NativeUtil.NULL; +import static org.apache.arrow.util.Preconditions.checkNotNull; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import org.apache.arrow.ffi.jni.JniWrapper; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.ReferenceManager; +import org.apache.arrow.memory.util.MemoryUtil; + +/** + * C Data Interface ArrowArray. + * <p> + * Represents a wrapper for the following C structure: + * + * <pre> + * struct ArrowArray { + * // Array data description + * int64_t length; + * int64_t null_count; + * int64_t offset; + * int64_t n_buffers; + * int64_t n_children; + * const void** buffers; + * struct ArrowArray** children; + * struct ArrowArray* dictionary; + * + * // Release callback + * void (*release)(struct ArrowArray*); + * // Opaque producer-specific data + * void* private_data; + * }; + * </pre> + */ +public class ArrowArray implements BaseStruct { + private static final int SIZE_OF = 80; Review comment: Also, for sanity the import/export functions should error out on 32-bit systems (I assume this shouldn't be difficult to do?). ########## File path: java/ffi/src/main/cpp/jni_wrapper.cc ########## @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <jni.h> + +#include <cassert> +#include <memory> +#include <stdexcept> +#include <string> + +#include "./abi.h" +#include "org_apache_arrow_ffi_jni_JniWrapper.h" + +namespace { + +jclass CreateGlobalClassReference(JNIEnv* env, const char* class_name) { + jclass local_class = env->FindClass(class_name); + jclass global_class = (jclass)env->NewGlobalRef(local_class); + env->DeleteLocalRef(local_class); + return global_class; +} + +jclass illegal_access_exception_class; +jclass illegal_argument_exception_class; +jclass runtime_exception_class; +jclass private_data_class; + +jmethodID private_data_close_method; + +jint JNI_VERSION = JNI_VERSION_1_6; + +class JniPendingException : public std::runtime_error { + public: + explicit JniPendingException(const std::string& arg) : std::runtime_error(arg) {} +}; + +void ThrowPendingException(const std::string& message) { + throw JniPendingException(message); +} + +void JniThrow(std::string message) { ThrowPendingException(message); } + +jmethodID GetMethodID(JNIEnv* env, jclass this_class, const char* name, const char* sig) { + jmethodID ret = env->GetMethodID(this_class, name, sig); + if (ret == nullptr) { + std::string error_message = "Unable to find method " + std::string(name) + + " within signature " + std::string(sig); + ThrowPendingException(error_message); + } + return ret; +} + +class InnerPrivateData { + public: + InnerPrivateData(JavaVM* vm, jobject private_data) + : vm_(vm), j_private_data_(private_data) {} + + JavaVM* vm_; + jobject j_private_data_; +}; + +class JNIEnvGuard { + public: + explicit JNIEnvGuard(JavaVM* vm) : vm_(vm), should_deattach_(false) { + JNIEnv* env; + jint code = vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION); + if (code == JNI_EDETACHED) { + JavaVMAttachArgs args; + args.version = JNI_VERSION; + args.name = NULL; + args.group = NULL; + code = vm->AttachCurrentThread(reinterpret_cast<void**>(&env), &args); + should_deattach_ = (code == JNI_OK); + } + if (code != JNI_OK) { + ThrowPendingException("Failed to attach the current thread to a Java VM"); + } + env_ = env; + } + + JNIEnv* env() { return env_; } + + ~JNIEnvGuard() { + if (should_deattach_) { + vm_->DetachCurrentThread(); + should_deattach_ = false; + } + } + + private: + bool should_deattach_; + JavaVM* vm_; + JNIEnv* env_; +}; + +template <typename T> +void release_exported(T* base) { + // This should not be called on already released structure + assert(base->release != nullptr); + + // Release children + for (int64_t i = 0; i < base->n_children; ++i) { + T* child = base->children[i]; + if (child->release != nullptr) { + child->release(child); + assert(child->release == nullptr); + } + } + + // Release dictionary + T* dict = base->dictionary; + if (dict != nullptr && dict->release != nullptr) { + dict->release(dict); + assert(dict->release == nullptr); + } + + // Release all data directly owned by the struct + InnerPrivateData* private_data = + reinterpret_cast<InnerPrivateData*>(base->private_data); + + JNIEnvGuard guard(private_data->vm_); + JNIEnv* env = guard.env(); + + env->CallObjectMethod(private_data->j_private_data_, private_data_close_method); + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + ThrowPendingException("Error calling close of private data"); Review comment: Will this exception be caught? The release callback can be invoked at arbitrary points. ########## File path: java/ffi/src/test/java/org/apache/arrow/ffi/RoundtripTest.java ########## @@ -0,0 +1,794 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.ffi; + +import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.DurationVector; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.IntervalDayVector; +import org.apache.arrow.vector.IntervalYearVector; +import org.apache.arrow.vector.LargeVarBinaryVector; +import org.apache.arrow.vector.LargeVarCharVector; +import org.apache.arrow.vector.NullVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeMilliVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.arrow.vector.TimeStampSecVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt2Vector; +import org.apache.arrow.vector.UInt4Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ZeroVector; +import org.apache.arrow.vector.compare.VectorEqualsVisitor; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.LargeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.arrow.vector.holders.IntervalDayHolder; +import org.apache.arrow.vector.holders.NullableLargeVarBinaryHolder; +import org.apache.arrow.vector.holders.NullableUInt4Holder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; +import org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class RoundtripTest { Review comment: For the record, are these tests able to check that memory is released correctly? ########## File path: ci/docker/linux-apt-jni.dockerfile ########## @@ -79,6 +79,7 @@ ENV ARROW_BUILD_TESTS=OFF \ ARROW_PLASMA_JAVA_CLIENT=ON \ ARROW_PLASMA=ON \ ARROW_USE_CCACHE=ON \ + ARROW_JAVA_FFI=ON \ Review comment: Can you keep this in alphabetical order? ########## File path: java/ffi/src/main/java/org/apache/arrow/ffi/Metadata.java ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.ffi; + +import static org.apache.arrow.ffi.NativeUtil.NULL; +import static org.apache.arrow.util.Preconditions.checkState; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.util.MemoryUtil; + +/** + * Encode and decode metadata. + */ +final class Metadata { + + private Metadata() { + } + + static ArrowBuf encode(BufferAllocator allocator, Map<String, String> metadata) { + if (metadata == null || metadata.size() == 0) { + return null; + } + + List<byte[]> buffers = new ArrayList<>(metadata.size() * 2); + int totalSize = 4 + metadata.size() * 8; // number of key/value pairs + buffer length fields + for (Map.Entry<String, String> entry : metadata.entrySet()) { + byte[] keyBuffer = entry.getKey().getBytes(StandardCharsets.UTF_8); + byte[] valueBuffer = entry.getValue().getBytes(StandardCharsets.UTF_8); + totalSize += keyBuffer.length; + totalSize += valueBuffer.length; + buffers.add(keyBuffer); + buffers.add(valueBuffer); + } + + ArrowBuf result = allocator.buffer(totalSize); + ByteBuffer writer = MemoryUtil.directBuffer(result.memoryAddress(), totalSize).order(ByteOrder.nativeOrder()); + writer.putInt(metadata.size()); + for (byte[] buffer : buffers) { + writer.putInt(buffer.length); + writer.put(buffer); + } + return result.slice(0, totalSize); + } + + static Map<String, String> decode(long bufferAddress) { + if (bufferAddress == NULL) { + return null; + } + + ByteBuffer reader = MemoryUtil.directBuffer(bufferAddress, Integer.MAX_VALUE).order(ByteOrder.nativeOrder()); + + int size = reader.getInt(); + checkState(size >= 0, "Metadata size must not be negative"); + if (size == 0) { + return null; + } + + Map<String, String> result = new HashMap<>(size); + for (int i = 0; i < size; i++) { + String key = readString(reader); + String value = readString(reader); + result.put(key, value); + } + return result; + } + + private static String readString(ByteBuffer reader) { + int length = reader.getInt(); + checkState(length >= 0, "Metadata item length must not be negative"); + String result = ""; + if (length > 0) { + byte[] dst = new byte[length]; + reader.get(dst); + result = new String(dst, StandardCharsets.UTF_8); Review comment: Note that the spec doesn't mandate metadata keys or values to be valid utf-8 (but as a first approximation you can probably expect them to be): https://arrow.apache.org/docs/format/CDataInterface.html#c.ArrowSchema.metadata -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
