Repository: arrow Updated Branches: refs/heads/master 0dc6fe8f3 -> a68f31b0f
ARROW-860: [C++] Remove typed Tensor containers cc @kou for opinions -- this patch breaks glib for the moment. Since tensors are all fixed width types, there's less reason to have strongly-typed containers for them (unlike the `arrow::Array` subclasses, where ListArray is quite different from Int8Array). My view is that if the visitor pattern needs to be employed, we can do it using the `type()` member on the tensor (which also provides compile-time access to `TypeClass::c_type` if needed) Author: Wes McKinney <wes.mckin...@twosigma.com> Author: Kouhei Sutou <k...@clear-code.com> Closes #571 from wesm/ARROW-860 and squashes the following commits: fe0b4d8 [Kouhei Sutou] Remove typed Tensors from glib 357f441 [Wes McKinney] Remove typed Tensor containers Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a68f31b0 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a68f31b0 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a68f31b0 Branch: refs/heads/master Commit: a68f31b0f3f2c094c5d6660a2d936baa05da3103 Parents: 0dc6fe8 Author: Wes McKinney <wes.mckin...@twosigma.com> Authored: Thu Apr 20 09:36:21 2017 +0200 Committer: Uwe L. Korn <uw...@xhochy.com> Committed: Thu Apr 20 09:36:21 2017 +0200 ---------------------------------------------------------------------- c_glib/arrow-glib/Makefile.am | 5 -- c_glib/arrow-glib/arrow-glib.h | 2 - c_glib/arrow-glib/int8-tensor.cpp | 105 -------------------------- c_glib/arrow-glib/int8-tensor.h | 79 ------------------- c_glib/arrow-glib/numeric-tensor.hpp | 64 ---------------- c_glib/arrow-glib/tensor.cpp | 103 +++++++++++++------------ c_glib/arrow-glib/tensor.h | 8 ++ c_glib/arrow-glib/uint8-tensor.cpp | 105 -------------------------- c_glib/arrow-glib/uint8-tensor.h | 79 ------------------- c_glib/test/test-int8-tensor.rb | 43 ----------- c_glib/test/test-tensor.rb | 6 +- c_glib/test/test-uint8-tensor.rb | 43 ----------- cpp/src/arrow/compare.cc | 37 +++------ cpp/src/arrow/ipc/ipc-read-write-test.cc | 6 +- cpp/src/arrow/ipc/reader.cc | 3 +- cpp/src/arrow/python/numpy_convert.cc | 3 +- cpp/src/arrow/tensor-test.cc | 14 ++-- cpp/src/arrow/tensor.cc | 65 ---------------- cpp/src/arrow/tensor.h | 47 +----------- cpp/src/arrow/visitor_inline.h | 25 ------ 20 files changed, 94 insertions(+), 748 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/Makefile.am ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index fbfe3a4..11b6508 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -65,7 +65,6 @@ libarrow_glib_la_headers = \ int8-array.h \ int8-array-builder.h \ int8-data-type.h \ - int8-tensor.h \ int16-array.h \ int16-array-builder.h \ int16-data-type.h \ @@ -94,7 +93,6 @@ libarrow_glib_la_headers = \ uint8-array.h \ uint8-array-builder.h \ uint8-data-type.h \ - uint8-tensor.h \ uint16-array.h \ uint16-array-builder.h \ uint16-data-type.h \ @@ -155,7 +153,6 @@ libarrow_glib_la_sources = \ int8-array.cpp \ int8-array-builder.cpp \ int8-data-type.cpp \ - int8-tensor.cpp \ int16-array.cpp \ int16-array-builder.cpp \ int16-data-type.cpp \ @@ -184,7 +181,6 @@ libarrow_glib_la_sources = \ uint8-array.cpp \ uint8-array-builder.cpp \ uint8-data-type.cpp \ - uint8-tensor.cpp \ uint16-array.cpp \ uint16-array-builder.cpp \ uint16-data-type.cpp \ @@ -226,7 +222,6 @@ libarrow_glib_la_cpp_headers = \ data-type.hpp \ error.hpp \ field.hpp \ - numeric-tensor.hpp \ record-batch.hpp \ schema.hpp \ table.hpp \ http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/arrow-glib.h ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/arrow-glib.h b/c_glib/arrow-glib/arrow-glib.h index eec9e25..8d9bfe2 100644 --- a/c_glib/arrow-glib/arrow-glib.h +++ b/c_glib/arrow-glib/arrow-glib.h @@ -42,7 +42,6 @@ #include <arrow-glib/int8-array.h> #include <arrow-glib/int8-array-builder.h> #include <arrow-glib/int8-data-type.h> -#include <arrow-glib/int8-tensor.h> #include <arrow-glib/int16-array.h> #include <arrow-glib/int16-array-builder.h> #include <arrow-glib/int16-data-type.h> @@ -71,7 +70,6 @@ #include <arrow-glib/uint8-array.h> #include <arrow-glib/uint8-array-builder.h> #include <arrow-glib/uint8-data-type.h> -#include <arrow-glib/uint8-tensor.h> #include <arrow-glib/uint16-array.h> #include <arrow-glib/uint16-array-builder.h> #include <arrow-glib/uint16-data-type.h> http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/int8-tensor.cpp ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/int8-tensor.cpp b/c_glib/arrow-glib/int8-tensor.cpp deleted file mode 100644 index 06521a0..0000000 --- a/c_glib/arrow-glib/int8-tensor.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifdef HAVE_CONFIG_H -# include <config.h> -#endif - -#include <arrow-glib/buffer.hpp> -#include <arrow-glib/int8-tensor.h> -#include <arrow-glib/numeric-tensor.hpp> - -G_BEGIN_DECLS - -/** - * SECTION: int8-tensor - * @short_description: 8-bit integer tensor class - * - * #GArrowInt8Tensor is a class for 8-bit integer tensor. It can store - * zero or more 8-bit integer data. - */ - -G_DEFINE_TYPE(GArrowInt8Tensor, \ - garrow_int8_tensor, \ - GARROW_TYPE_TENSOR) - -static void -garrow_int8_tensor_init(GArrowInt8Tensor *object) -{ -} - -static void -garrow_int8_tensor_class_init(GArrowInt8TensorClass *klass) -{ -} - -/** - * garrow_int8_tensor_new: - * @data: A #GArrowBuffer that contains tensor data. - * @shape: (array length=n_dimensions): A list of dimension sizes. - * @n_dimensions: The number of dimensions. - * @strides: (array length=n_strides) (nullable): A list of the number of - * bytes in each dimension. - * @n_strides: The number of strides. - * @dimention_names: (array length=n_dimention_names) (nullable): A list of - * dimension names. - * @n_dimention_names: The number of dimension names - * - * Returns: The newly created #GArrowInt8Tensor. - * - * Since: 0.3.0 - */ -GArrowInt8Tensor * -garrow_int8_tensor_new(GArrowBuffer *data, - gint64 *shape, - gsize n_dimensions, - gint64 *strides, - gsize n_strides, - gchar **dimension_names, - gsize n_dimension_names) -{ - auto tensor = - garrow::numeric_tensor_new<arrow::Int8Tensor>(data, - shape, - n_dimensions, - strides, - n_strides, - dimension_names, - n_dimension_names); - return GARROW_INT8_TENSOR(tensor); -} - -/** - * garrow_int8_tensor_get_raw_data: - * @tensor: A #GArrowInt8Tensor. - * @n_data: (out): The number of data. - * - * Returns: (array length=n_data): The raw data in the tensor. - * - * Since: 0.3.0 - */ -const gint8 * -garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor, - gint64 *n_data) -{ - return garrow::numeric_tensor_get_raw_data<arrow::Int8Tensor, int8_t>(GARROW_TENSOR(tensor), - n_data); -} - -G_END_DECLS http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/int8-tensor.h ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/int8-tensor.h b/c_glib/arrow-glib/int8-tensor.h deleted file mode 100644 index 76ed3c8..0000000 --- a/c_glib/arrow-glib/int8-tensor.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include <arrow-glib/tensor.h> - -G_BEGIN_DECLS - -#define GARROW_TYPE_INT8_TENSOR \ - (garrow_int8_tensor_get_type()) -#define GARROW_INT8_TENSOR(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INT8_TENSOR, \ - GArrowInt8Tensor)) -#define GARROW_INT8_TENSOR_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INT8_TENSOR, \ - GArrowInt8TensorClass)) -#define GARROW_IS_INT8_TENSOR(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INT8_TENSOR)) -#define GARROW_IS_INT8_TENSOR_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INT8_TENSOR)) -#define GARROW_INT8_TENSOR_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INT8_TENSOR, \ - GArrowInt8TensorClass)) - -typedef struct _GArrowInt8Tensor GArrowInt8Tensor; -typedef struct _GArrowInt8TensorClass GArrowInt8TensorClass; - -/** - * GArrowInt8Tensor: - * - * It wraps `arrow::Int8Tensor`. - */ -struct _GArrowInt8Tensor -{ - /*< private >*/ - GArrowTensor parent_instance; -}; - -struct _GArrowInt8TensorClass -{ - GArrowTensorClass parent_class; -}; - -GType garrow_int8_tensor_get_type(void) G_GNUC_CONST; - -GArrowInt8Tensor *garrow_int8_tensor_new(GArrowBuffer *data, - gint64 *shape, - gsize n_dimensions, - gint64 *strides, - gsize n_strides, - gchar **dimention_names, - gsize n_dimention_names); - -const gint8 *garrow_int8_tensor_get_raw_data(GArrowInt8Tensor *tensor, - gint64 *n_data); - -G_END_DECLS http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/numeric-tensor.hpp ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/numeric-tensor.hpp b/c_glib/arrow-glib/numeric-tensor.hpp deleted file mode 100644 index 07cea62..0000000 --- a/c_glib/arrow-glib/numeric-tensor.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include <arrow-glib/tensor.hpp> - -namespace garrow { - template <typename T> - GArrowTensor *numeric_tensor_new(GArrowBuffer *data, - gint64 *shape, - gsize n_dimensions, - gint64 *strides, - gsize n_strides, - gchar **dimention_names, - gsize n_dimention_names) { - auto arrow_data = garrow_buffer_get_raw(data); - std::vector<int64_t> arrow_shape; - for (gsize i = 0; i < n_dimensions; ++i) { - arrow_shape.push_back(shape[i]); - } - std::vector<int64_t> arrow_strides; - for (gsize i = 0; i < n_strides; ++i) { - arrow_strides.push_back(strides[i]); - } - std::vector<std::string> arrow_dimention_names; - for (gsize i = 0; i < n_dimention_names; ++i) { - arrow_dimention_names.push_back(dimention_names[i]); - } - auto arrow_numeric_tensor = - std::make_shared<T>(arrow_data, - arrow_shape, - arrow_strides, - arrow_dimention_names); - std::shared_ptr<arrow::Tensor> arrow_tensor = arrow_numeric_tensor; - auto tensor = garrow_tensor_new_raw(&arrow_tensor); - return tensor; - } - - template <typename T, typename value_type> - const value_type *numeric_tensor_get_raw_data(GArrowTensor *tensor, - gint64 *n_data) { - auto arrow_tensor = garrow_tensor_get_raw(tensor); - auto arrow_numeric_tensor = static_cast<const T *>(arrow_tensor.get()); - *n_data = arrow_numeric_tensor->size(); - return arrow_numeric_tensor->raw_data(); - } -} http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/tensor.cpp ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/tensor.cpp b/c_glib/arrow-glib/tensor.cpp index cbc9d8e..468eb07 100644 --- a/c_glib/arrow-glib/tensor.cpp +++ b/c_glib/arrow-glib/tensor.cpp @@ -23,10 +23,8 @@ #include <arrow-glib/buffer.hpp> #include <arrow-glib/data-type.hpp> -#include <arrow-glib/int8-tensor.h> #include <arrow-glib/tensor.hpp> #include <arrow-glib/type.hpp> -#include <arrow-glib/uint8-tensor.h> G_BEGIN_DECLS @@ -122,6 +120,58 @@ garrow_tensor_class_init(GArrowTensorClass *klass) } /** + * garrow_tensor_new: + * @data_type: A #GArrowDataType that indicates each element type + * in the tensor. + * @data: A #GArrowBuffer that contains tensor data. + * @shape: (array length=n_dimensions): A list of dimension sizes. + * @n_dimensions: The number of dimensions. + * @strides: (array length=n_strides) (nullable): A list of the number of + * bytes in each dimension. + * @n_strides: The number of strides. + * @dimention_names: (array length=n_dimention_names) (nullable): A list of + * dimension names. + * @n_dimention_names: The number of dimension names + * + * Returns: The newly created #GArrowTensor. + * + * Since: 0.3.0 + */ +GArrowTensor * +garrow_tensor_new(GArrowDataType *data_type, + GArrowBuffer *data, + gint64 *shape, + gsize n_dimensions, + gint64 *strides, + gsize n_strides, + gchar **dimension_names, + gsize n_dimension_names) +{ + auto arrow_data_type = garrow_data_type_get_raw(data_type); + auto arrow_data = garrow_buffer_get_raw(data); + std::vector<int64_t> arrow_shape; + for (gsize i = 0; i < n_dimensions; ++i) { + arrow_shape.push_back(shape[i]); + } + std::vector<int64_t> arrow_strides; + for (gsize i = 0; i < n_strides; ++i) { + arrow_strides.push_back(strides[i]); + } + std::vector<std::string> arrow_dimension_names; + for (gsize i = 0; i < n_dimension_names; ++i) { + arrow_dimension_names.push_back(dimension_names[i]); + } + auto arrow_tensor = + std::make_shared<arrow::Tensor>(arrow_data_type, + arrow_data, + arrow_shape, + arrow_strides, + arrow_dimension_names); + auto tensor = garrow_tensor_new_raw(&arrow_tensor); + return tensor; +} + +/** * garrow_tensor_get_value_data_type: * @tensor: A #GArrowTensor. * @@ -333,52 +383,9 @@ G_END_DECLS GArrowTensor * garrow_tensor_new_raw(std::shared_ptr<arrow::Tensor> *arrow_tensor) { - GType type; - GArrowTensor *tensor; - - switch ((*arrow_tensor)->type_id()) { - case arrow::Type::type::UINT8: - type = GARROW_TYPE_UINT8_TENSOR; - break; - case arrow::Type::type::INT8: - type = GARROW_TYPE_INT8_TENSOR; - break; -/* - case arrow::Type::type::UINT16: - type = GARROW_TYPE_UINT16_TENSOR; - break; - case arrow::Type::type::INT16: - type = GARROW_TYPE_INT16_TENSOR; - break; - case arrow::Type::type::UINT32: - type = GARROW_TYPE_UINT32_TENSOR; - break; - case arrow::Type::type::INT32: - type = GARROW_TYPE_INT32_TENSOR; - break; - case arrow::Type::type::UINT64: - type = GARROW_TYPE_UINT64_TENSOR; - break; - case arrow::Type::type::INT64: - type = GARROW_TYPE_INT64_TENSOR; - break; - case arrow::Type::type::HALF_FLOAT: - type = GARROW_TYPE_HALF_FLOAT_TENSOR; - break; - case arrow::Type::type::FLOAT: - type = GARROW_TYPE_FLOAT_TENSOR; - break; - case arrow::Type::type::DOUBLE: - type = GARROW_TYPE_DOUBLE_TENSOR; - break; -*/ - default: - type = GARROW_TYPE_TENSOR; - break; - } - tensor = GARROW_TENSOR(g_object_new(type, - "tensor", arrow_tensor, - NULL)); + auto tensor = GARROW_TENSOR(g_object_new(GARROW_TYPE_TENSOR, + "tensor", arrow_tensor, + NULL)); return tensor; } http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/tensor.h ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/tensor.h b/c_glib/arrow-glib/tensor.h index bedc803..71c6b4e 100644 --- a/c_glib/arrow-glib/tensor.h +++ b/c_glib/arrow-glib/tensor.h @@ -58,6 +58,14 @@ struct _GArrowTensorClass GType garrow_tensor_get_type (void) G_GNUC_CONST; +GArrowTensor *garrow_tensor_new (GArrowDataType *data_type, + GArrowBuffer *data, + gint64 *shape, + gsize n_dimensions, + gint64 *strides, + gsize n_strides, + gchar **dimention_names, + gsize n_dimention_names); GArrowDataType *garrow_tensor_get_value_data_type(GArrowTensor *tensor); GArrowType garrow_tensor_get_value_type (GArrowTensor *tensor); GArrowBuffer *garrow_tensor_get_buffer (GArrowTensor *tensor); http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/uint8-tensor.cpp ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/uint8-tensor.cpp b/c_glib/arrow-glib/uint8-tensor.cpp deleted file mode 100644 index 69f0f69..0000000 --- a/c_glib/arrow-glib/uint8-tensor.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifdef HAVE_CONFIG_H -# include <config.h> -#endif - -#include <arrow-glib/buffer.hpp> -#include <arrow-glib/uint8-tensor.h> -#include <arrow-glib/numeric-tensor.hpp> - -G_BEGIN_DECLS - -/** - * SECTION: uint8-tensor - * @short_description: 8-bit unsigned integer tensor class - * - * #GArrowUint8Tensor is a class for 8-bit unsigned integer tensor. It - * can store zero or more 8-bit integer data. - */ - -G_DEFINE_TYPE(GArrowUInt8Tensor, \ - garrow_uint8_tensor, \ - GARROW_TYPE_TENSOR) - -static void -garrow_uint8_tensor_init(GArrowUInt8Tensor *object) -{ -} - -static void -garrow_uint8_tensor_class_init(GArrowUInt8TensorClass *klass) -{ -} - -/** - * garrow_uint8_tensor_new: - * @data: A #GArrowBuffer that contains tensor data. - * @shape: (array length=n_dimensions): A list of dimension sizes. - * @n_dimensions: The number of dimensions. - * @strides: (array length=n_strides) (nullable): A list of the number of - * bytes in each dimension. - * @n_strides: The number of strides. - * @dimention_names: (array length=n_dimention_names) (nullable): A list of - * dimension names. - * @n_dimention_names: The number of dimension names - * - * Returns: The newly created #GArrowUInt8Tensor. - * - * Since: 0.3.0 - */ -GArrowUInt8Tensor * -garrow_uint8_tensor_new(GArrowBuffer *data, - gint64 *shape, - gsize n_dimensions, - gint64 *strides, - gsize n_strides, - gchar **dimension_names, - gsize n_dimension_names) -{ - auto tensor = - garrow::numeric_tensor_new<arrow::UInt8Tensor>(data, - shape, - n_dimensions, - strides, - n_strides, - dimension_names, - n_dimension_names); - return GARROW_UINT8_TENSOR(tensor); -} - -/** - * garrow_uint8_tensor_get_raw_data: - * @tensor: A #GArrowUInt8Tensor. - * @n_data: (out): The number of data. - * - * Returns: (array length=n_data): The raw data in the tensor. - * - * Since: 0.3.0 - */ -const guint8 * -garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor, - gint64 *n_data) -{ - return garrow::numeric_tensor_get_raw_data<arrow::UInt8Tensor, uint8_t>(GARROW_TENSOR(tensor), - n_data); -} - -G_END_DECLS http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/arrow-glib/uint8-tensor.h ---------------------------------------------------------------------- diff --git a/c_glib/arrow-glib/uint8-tensor.h b/c_glib/arrow-glib/uint8-tensor.h deleted file mode 100644 index 248c507..0000000 --- a/c_glib/arrow-glib/uint8-tensor.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include <arrow-glib/tensor.h> - -G_BEGIN_DECLS - -#define GARROW_TYPE_UINT8_TENSOR \ - (garrow_uint8_tensor_get_type()) -#define GARROW_UINT8_TENSOR(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_UINT8_TENSOR, \ - GArrowUInt8Tensor)) -#define GARROW_UINT8_TENSOR_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_UINT8_TENSOR, \ - GArrowUInt8TensorClass)) -#define GARROW_IS_UINT8_TENSOR(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_UINT8_TENSOR)) -#define GARROW_IS_UINT8_TENSOR_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_UINT8_TENSOR)) -#define GARROW_UINT8_TENSOR_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_UINT8_TENSOR, \ - GArrowUInt8TensorClass)) - -typedef struct _GArrowUInt8Tensor GArrowUInt8Tensor; -typedef struct _GArrowUInt8TensorClass GArrowUInt8TensorClass; - -/** - * GArrowUInt8Tensor: - * - * It wraps `arrow::UInt8Tensor`. - */ -struct _GArrowUInt8Tensor -{ - /*< private >*/ - GArrowTensor parent_instance; -}; - -struct _GArrowUInt8TensorClass -{ - GArrowTensorClass parent_class; -}; - -GType garrow_uint8_tensor_get_type(void) G_GNUC_CONST; - -GArrowUInt8Tensor *garrow_uint8_tensor_new(GArrowBuffer *data, - gint64 *shape, - gsize n_dimensions, - gint64 *strides, - gsize n_strides, - gchar **dimention_names, - gsize n_dimention_names); - -const guint8 *garrow_uint8_tensor_get_raw_data(GArrowUInt8Tensor *tensor, - gint64 *n_data); - -G_END_DECLS http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/test/test-int8-tensor.rb ---------------------------------------------------------------------- diff --git a/c_glib/test/test-int8-tensor.rb b/c_glib/test/test-int8-tensor.rb deleted file mode 100644 index a96a407..0000000 --- a/c_glib/test/test-int8-tensor.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class TestInt8Tensor < Test::Unit::TestCase - include Helper::Omittable - - def setup - @raw_data = [ - 1, 2, - 3, 4, - - 5, 6, - 7, 8, - - 9, 10, - 11, 12, - ] - data = Arrow::Buffer.new(@raw_data.pack("c*")) - shape = [3, 2, 2] - strides = [] - names = [] - @tensor = Arrow::Int8Tensor.new(data, shape, strides, names) - end - - def test_raw_data - require_gi(3, 1, 2) - assert_equal(@raw_data, @tensor.raw_data) - end -end http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/test/test-tensor.rb ---------------------------------------------------------------------- diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb index 455b0d9..3e1f541 100644 --- a/c_glib/test/test-tensor.rb +++ b/c_glib/test/test-tensor.rb @@ -33,7 +33,11 @@ class TestTensor < Test::Unit::TestCase @shape = [3, 2, 2] strides = [] names = ["a", "b", "c"] - @tensor = Arrow::Int8Tensor.new(data, @shape, strides, names) + @tensor = Arrow::Tensor.new(Arrow::Int8DataType.new, + data, + @shape, + strides, + names) end def test_value_data_type http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/c_glib/test/test-uint8-tensor.rb ---------------------------------------------------------------------- diff --git a/c_glib/test/test-uint8-tensor.rb b/c_glib/test/test-uint8-tensor.rb deleted file mode 100644 index 0fe758b..0000000 --- a/c_glib/test/test-uint8-tensor.rb +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -class TestUInt8Tensor < Test::Unit::TestCase - include Helper::Omittable - - def setup - @raw_data = [ - 1, 2, - 3, 4, - - 5, 6, - 7, 8, - - 9, 10, - 11, 12, - ] - data = Arrow::Buffer.new(@raw_data.pack("c*")) - shape = [3, 2, 2] - strides = [] - names = [] - @tensor = Arrow::UInt8Tensor.new(data, shape, strides, names) - end - - def test_raw_data - require_gi(3, 1, 2) - assert_equal(@raw_data, @tensor.raw_data) - end -end http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/compare.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index ccb299e..562d4e1 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -580,31 +580,6 @@ Status ArrayRangeEquals(const Array& left, const Array& right, int64_t left_star // ---------------------------------------------------------------------- // Implement TensorEquals -class TensorEqualsVisitor { - public: - explicit TensorEqualsVisitor(const Tensor& right) : right_(right) {} - - template <typename TensorType> - Status Visit(const TensorType& left) { - const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type()); - const int byte_width = size_meta.bit_width() / 8; - DCHECK_GT(byte_width, 0); - - const uint8_t* left_data = left.data()->data(); - const uint8_t* right_data = right_.data()->data(); - - result_ = - memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0; - return Status::OK(); - } - - bool result() const { return result_; } - - protected: - const Tensor& right_; - bool result_; -}; - Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) { // The arrays are the same object if (&left == &right) { @@ -619,9 +594,15 @@ Status TensorEquals(const Tensor& left, const Tensor& right, bool* are_equal) { "Comparison not implemented for non-contiguous tensors"); } - TensorEqualsVisitor visitor(right); - RETURN_NOT_OK(VisitTensorInline(left, &visitor)); - *are_equal = visitor.result(); + const auto& size_meta = dynamic_cast<const FixedWidthType&>(*left.type()); + const int byte_width = size_meta.bit_width() / 8; + DCHECK_GT(byte_width, 0); + + const uint8_t* left_data = left.data()->data(); + const uint8_t* right_data = right.data()->data(); + + *are_equal = + memcmp(left_data, right_data, static_cast<size_t>(byte_width * left.size())) == 0; } return Status::OK(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/ipc/ipc-read-write-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc index cd793e0..b4a88b5 100644 --- a/cpp/src/arrow/ipc/ipc-read-write-test.cc +++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc @@ -704,8 +704,8 @@ TEST_F(TestTensorRoundTrip, BasicRoundtrip) { auto data = test::GetBufferFromVector(values); - Int64Tensor t0(data, shape, strides, dim_names); - Int64Tensor tzero(data, {}, {}, {}); + Tensor t0(int64(), data, shape, strides, dim_names); + Tensor tzero(int64(), data, {}, {}, {}); CheckTensorRoundTrip(t0); CheckTensorRoundTrip(tzero); @@ -724,7 +724,7 @@ TEST_F(TestTensorRoundTrip, NonContiguous) { test::randint<int64_t>(24, 0, 100, &values); auto data = test::GetBufferFromVector(values); - Int64Tensor tensor(data, {4, 3}, {48, 16}); + Tensor tensor(int64(), data, {4, 3}, {48, 16}); int32_t metadata_length; int64_t body_length; http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/ipc/reader.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 69fde17..aea4c9c 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -507,7 +507,8 @@ Status ReadTensor( std::vector<std::string> dim_names; RETURN_NOT_OK( GetTensorMetadata(message->header(), &type, &shape, &strides, &dim_names)); - return MakeTensor(type, data, shape, strides, dim_names, out); + *out = std::make_shared<Tensor>(type, data, shape, strides, dim_names); + return Status::OK(); } } // namespace ipc http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/python/numpy_convert.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc index 2c1a591..c391b5d 100644 --- a/cpp/src/arrow/python/numpy_convert.cc +++ b/cpp/src/arrow/python/numpy_convert.cc @@ -223,7 +223,8 @@ Status NdarrayToTensor(MemoryPool* pool, PyObject* ao, std::shared_ptr<Tensor>* std::shared_ptr<DataType> type; RETURN_NOT_OK( GetTensorType(reinterpret_cast<PyObject*>(PyArray_DESCR(ndarray)), &type)); - return MakeTensor(type, data, shape, strides, {}, out); + *out = std::make_shared<Tensor>(type, data, shape, strides); + return Status::OK(); } Status TensorToNdarray(const Tensor& tensor, PyObject* base, PyObject** out) { http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/tensor-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/tensor-test.cc b/cpp/src/arrow/tensor-test.cc index 336905c..c41683a 100644 --- a/cpp/src/arrow/tensor-test.cc +++ b/cpp/src/arrow/tensor-test.cc @@ -39,7 +39,7 @@ TEST(TestTensor, ZeroDim) { std::shared_ptr<MutableBuffer> buffer; ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer)); - Int64Tensor t0(buffer, shape); + Tensor t0(int64(), buffer, shape); ASSERT_EQ(1, t0.size()); } @@ -55,9 +55,9 @@ TEST(TestTensor, BasicCtors) { std::shared_ptr<MutableBuffer> buffer; ASSERT_OK(AllocateBuffer(default_memory_pool(), values * sizeof(T), &buffer)); - Int64Tensor t1(buffer, shape); - Int64Tensor t2(buffer, shape, strides); - Int64Tensor t3(buffer, shape, strides, dim_names); + Tensor t1(int64(), buffer, shape); + Tensor t2(int64(), buffer, shape, strides); + Tensor t3(int64(), buffer, shape, strides, dim_names); ASSERT_EQ(24, t1.size()); ASSERT_TRUE(t1.is_mutable()); @@ -84,9 +84,9 @@ TEST(TestTensor, IsContiguous) { std::vector<int64_t> c_strides = {48, 8}; std::vector<int64_t> f_strides = {8, 32}; std::vector<int64_t> noncontig_strides = {8, 8}; - Int64Tensor t1(buffer, shape, c_strides); - Int64Tensor t2(buffer, shape, f_strides); - Int64Tensor t3(buffer, shape, noncontig_strides); + Tensor t1(int64(), buffer, shape, c_strides); + Tensor t2(int64(), buffer, shape, f_strides); + Tensor t3(int64(), buffer, shape, noncontig_strides); ASSERT_TRUE(t1.is_contiguous()); ASSERT_TRUE(t2.is_contiguous()); http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/tensor.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index fa3e203..909b05e 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -118,69 +118,4 @@ bool Tensor::Equals(const Tensor& other) const { return are_equal; } -template <typename T> -NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data, - const std::vector<int64_t>& shape, const std::vector<int64_t>& strides, - const std::vector<std::string>& dim_names) - : Tensor(TypeTraits<T>::type_singleton(), data, shape, strides, dim_names), - raw_data_(nullptr), - mutable_raw_data_(nullptr) { - if (data_) { - raw_data_ = reinterpret_cast<const value_type*>(data_->data()); - if (data_->is_mutable()) { - auto mut_buf = static_cast<MutableBuffer*>(data_.get()); - mutable_raw_data_ = reinterpret_cast<value_type*>(mut_buf->mutable_data()); - } - } -} - -template <typename T> -NumericTensor<T>::NumericTensor( - const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape) - : NumericTensor(data, shape, {}, {}) {} - -template <typename T> -NumericTensor<T>::NumericTensor(const std::shared_ptr<Buffer>& data, - const std::vector<int64_t>& shape, const std::vector<int64_t>& strides) - : NumericTensor(data, shape, strides, {}) {} - -template class ARROW_TEMPLATE_EXPORT NumericTensor<Int8Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt8Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<Int16Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt16Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<Int32Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt32Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<Int64Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<UInt64Type>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<HalfFloatType>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<FloatType>; -template class ARROW_TEMPLATE_EXPORT NumericTensor<DoubleType>; - -#define TENSOR_CASE(TYPE, TENSOR_TYPE) \ - case Type::TYPE: \ - *tensor = std::make_shared<TENSOR_TYPE>(data, shape, strides, dim_names); \ - break; - -Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type, - const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape, - const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names, - std::shared_ptr<Tensor>* tensor) { - switch (type->id()) { - TENSOR_CASE(INT8, Int8Tensor); - TENSOR_CASE(INT16, Int16Tensor); - TENSOR_CASE(INT32, Int32Tensor); - TENSOR_CASE(INT64, Int64Tensor); - TENSOR_CASE(UINT8, UInt8Tensor); - TENSOR_CASE(UINT16, UInt16Tensor); - TENSOR_CASE(UINT32, UInt32Tensor); - TENSOR_CASE(UINT64, UInt64Tensor); - TENSOR_CASE(HALF_FLOAT, HalfFloatTensor); - TENSOR_CASE(FLOAT, FloatTensor); - TENSOR_CASE(DOUBLE, DoubleTensor); - default: - return Status::NotImplemented(type->ToString()); - } - return Status::OK(); -} - } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/tensor.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h index 7741c30..371f591 100644 --- a/cpp/src/arrow/tensor.h +++ b/cpp/src/arrow/tensor.h @@ -76,6 +76,9 @@ class ARROW_EXPORT Tensor { std::shared_ptr<DataType> type() const { return type_; } std::shared_ptr<Buffer> data() const { return data_; } + const uint8_t* raw_data() const { return data_->data(); } + uint8_t* raw_data() { return data_->mutable_data(); } + const std::vector<int64_t>& shape() const { return shape_; } const std::vector<int64_t>& strides() const { return strides_; } @@ -117,50 +120,6 @@ class ARROW_EXPORT Tensor { DISALLOW_COPY_AND_ASSIGN(Tensor); }; -template <typename T> -class ARROW_EXPORT NumericTensor : public Tensor { - public: - using value_type = typename T::c_type; - - NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape); - - /// Constructor with non-negative strides - NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape, - const std::vector<int64_t>& strides); - - /// Constructor with strides and dimension names - NumericTensor(const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape, - const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names); - - const value_type* raw_data() const { return raw_data_; } - value_type* raw_data() { return mutable_raw_data_; } - - private: - const value_type* raw_data_; - value_type* mutable_raw_data_; -}; - -Status ARROW_EXPORT MakeTensor(const std::shared_ptr<DataType>& type, - const std::shared_ptr<Buffer>& data, const std::vector<int64_t>& shape, - const std::vector<int64_t>& strides, const std::vector<std::string>& dim_names, - std::shared_ptr<Tensor>* tensor); - -// ---------------------------------------------------------------------- -// extern templates and other details - -// Only instantiate these templates once -ARROW_EXTERN_TEMPLATE NumericTensor<Int8Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<UInt8Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<Int16Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<UInt16Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<Int32Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<UInt32Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<Int64Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<UInt64Type>; -ARROW_EXTERN_TEMPLATE NumericTensor<HalfFloatType>; -ARROW_EXTERN_TEMPLATE NumericTensor<FloatType>; -ARROW_EXTERN_TEMPLATE NumericTensor<DoubleType>; - } // namespace arrow #endif // ARROW_TENSOR_H http://git-wip-us.apache.org/repos/asf/arrow/blob/a68f31b0/cpp/src/arrow/visitor_inline.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h index bc5f493..7478950 100644 --- a/cpp/src/arrow/visitor_inline.h +++ b/cpp/src/arrow/visitor_inline.h @@ -104,31 +104,6 @@ inline Status VisitArrayInline(const Array& array, VISITOR* visitor) { return Status::NotImplemented("Type not implemented"); } -#define TENSOR_VISIT_INLINE(TYPE_CLASS) \ - case TYPE_CLASS::type_id: \ - return visitor->Visit( \ - static_cast<const typename TypeTraits<TYPE_CLASS>::TensorType&>(array)); - -template <typename VISITOR> -inline Status VisitTensorInline(const Tensor& array, VISITOR* visitor) { - switch (array.type_id()) { - TENSOR_VISIT_INLINE(Int8Type); - TENSOR_VISIT_INLINE(UInt8Type); - TENSOR_VISIT_INLINE(Int16Type); - TENSOR_VISIT_INLINE(UInt16Type); - TENSOR_VISIT_INLINE(Int32Type); - TENSOR_VISIT_INLINE(UInt32Type); - TENSOR_VISIT_INLINE(Int64Type); - TENSOR_VISIT_INLINE(UInt64Type); - TENSOR_VISIT_INLINE(HalfFloatType); - TENSOR_VISIT_INLINE(FloatType); - TENSOR_VISIT_INLINE(DoubleType); - default: - break; - } - return Status::NotImplemented("Type not implemented"); -} - } // namespace arrow #endif // ARROW_VISITOR_INLINE_H
