This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 92419bd ARROW-11196: [GLib] Add support for mock, HDFS and S3 file
systems with factory function
92419bd is described below
commit 92419bd0cde428a2e99bdb58cb8a58a40473f966
Author: Sutou Kouhei <[email protected]>
AuthorDate: Mon Jan 11 05:59:41 2021 +0900
ARROW-11196: [GLib] Add support for mock, HDFS and S3 file systems with
factory function
Closes #9152 from kou/glib-file-system-factory
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
c_glib/arrow-glib/file-system.cpp | 104 +++++++++++++++++++++++++++++++++++
c_glib/arrow-glib/file-system.h | 43 +++++++++++++++
c_glib/arrow-glib/file-system.hpp | 4 ++
c_glib/test/test-mock-file-system.rb | 30 ++++++++++
4 files changed, 181 insertions(+)
diff --git a/c_glib/arrow-glib/file-system.cpp
b/c_glib/arrow-glib/file-system.cpp
index fa08f20..2c2c36e 100644
--- a/c_glib/arrow-glib/file-system.cpp
+++ b/c_glib/arrow-glib/file-system.cpp
@@ -22,6 +22,7 @@
#include <arrow-glib/error.hpp>
#include <arrow-glib/file-system.hpp>
#include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/local-file-system.h>
#include <arrow-glib/output-stream.hpp>
G_BEGIN_DECLS
@@ -44,6 +45,13 @@ G_BEGIN_DECLS
*
* #GArrowSlowFileSystem is a delegator to another file system.
* This inserts latencies at various points.
+ *
+ * #GArrowMockFileSystem is a class for mock file system that holds
+ * its contents in memory.
+ *
+ * #GArrowHDFSFileSystem is a class for HDFS-backed file system.
+ *
+ * #GArrowS3FileSystem is a class for S3-backed file system.
*/
/* arrow::fs::FileInfo */
@@ -617,6 +625,34 @@ garrow_file_system_class_init(GArrowFileSystemClass *klass)
}
/**
+ * garrow_file_system_create:
+ * @uri: An URI to specify file system with options. If you only have an
+ * absolute path, g_filename_to_uri() will help you.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * This is a factory function to create a specific #GArrowFileSystem
+ * object.
+ *
+ * Returns: (nullable) (transfer full): The newly created file system
+ * that is an object of a subclass of #GArrowFileSystem.
+ *
+ * Since: 3.0.0
+ */
+GArrowFileSystem *
+garrow_file_system_create(const gchar *uri, GError **error)
+{
+ auto arrow_file_system_result = arrow::fs::FileSystemFromUri(uri);
+ if (garrow::check(error,
+ arrow_file_system_result,
+ "[file-system][create]")) {
+ auto arrow_file_system = *arrow_file_system_result;
+ return garrow_file_system_new_raw(&arrow_file_system);
+ } else {
+ return NULL;
+ }
+}
+
+/**
* garrow_file_system_get_type_name:
* @file_system: A #GArrowFileSystem.
*
@@ -1297,6 +1333,52 @@
garrow_slow_file_system_new_average_latency_and_seed(GArrowFileSystem *base_file
base_file_system);
}
+
+G_DEFINE_TYPE(GArrowMockFileSystem,
+ garrow_mock_file_system,
+ GARROW_TYPE_FILE_SYSTEM)
+
+static void
+garrow_mock_file_system_init(GArrowMockFileSystem *file_system)
+{
+}
+
+static void
+garrow_mock_file_system_class_init(GArrowMockFileSystemClass *klass)
+{
+}
+
+
+G_DEFINE_TYPE(GArrowHDFSFileSystem,
+ garrow_hdfs_file_system,
+ GARROW_TYPE_FILE_SYSTEM)
+
+static void
+garrow_hdfs_file_system_init(GArrowHDFSFileSystem *file_system)
+{
+}
+
+static void
+garrow_hdfs_file_system_class_init(GArrowHDFSFileSystemClass *klass)
+{
+}
+
+
+G_DEFINE_TYPE(GArrowS3FileSystem,
+ garrow_s3_file_system,
+ GARROW_TYPE_FILE_SYSTEM)
+
+static void
+garrow_s3_file_system_init(GArrowS3FileSystem *file_system)
+{
+}
+
+static void
+garrow_s3_file_system_class_init(GArrowS3FileSystemClass *klass)
+{
+}
+
+
G_END_DECLS
GArrowFileInfo *
@@ -1314,6 +1396,28 @@ garrow_file_info_get_raw(GArrowFileInfo *file_info)
return &(priv->file_info);
}
+GArrowFileSystem *
+garrow_file_system_new_raw(
+ std::shared_ptr<arrow::fs::FileSystem> *arrow_file_system)
+{
+ const auto &type_name = (*arrow_file_system)->type_name();
+
+ GType file_system_type = GARROW_TYPE_FILE_SYSTEM;
+ if (type_name == "local") {
+ file_system_type = GARROW_TYPE_LOCAL_FILE_SYSTEM;
+ } else if (type_name == "hdfs") {
+ file_system_type = GARROW_TYPE_HDFS_FILE_SYSTEM;
+ } else if (type_name == "s3") {
+ file_system_type = GARROW_TYPE_S3_FILE_SYSTEM;
+ } else if (type_name == "mock") {
+ file_system_type = GARROW_TYPE_MOCK_FILE_SYSTEM;
+ }
+
+ return GARROW_FILE_SYSTEM(g_object_new(file_system_type,
+ "file-system", arrow_file_system,
+ NULL));
+}
+
std::shared_ptr<arrow::fs::FileSystem>
garrow_file_system_get_raw(GArrowFileSystem *file_system)
{
diff --git a/c_glib/arrow-glib/file-system.h b/c_glib/arrow-glib/file-system.h
index b0bbad7..dc9fba7 100644
--- a/c_glib/arrow-glib/file-system.h
+++ b/c_glib/arrow-glib/file-system.h
@@ -104,6 +104,11 @@ struct _GArrowFileSystemClass
GObjectClass parent_class;
};
+GARROW_AVAILABLE_IN_3_0
+GArrowFileSystem *
+garrow_file_system_create(const gchar *uri,
+ GError **error);
+
GARROW_AVAILABLE_IN_0_17
gchar *garrow_file_system_get_type_name(GArrowFileSystem *file_system);
@@ -237,4 +242,42 @@
garrow_slow_file_system_new_average_latency_and_seed(GArrowFileSystem *base_file
gdouble average_latency,
gint32 seed);
+
+
+#define GARROW_TYPE_MOCK_FILE_SYSTEM (garrow_mock_file_system_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowMockFileSystem,
+ garrow_mock_file_system,
+ GARROW,
+ MOCK_FILE_SYSTEM,
+ GArrowFileSystem)
+struct _GArrowMockFileSystemClass
+{
+ GArrowFileSystemClass parent_class;
+};
+
+
+#define GARROW_TYPE_HDFS_FILE_SYSTEM (garrow_hdfs_file_system_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowHDFSFileSystem,
+ garrow_hdfs_file_system,
+ GARROW,
+ HDFS_FILE_SYSTEM,
+ GArrowFileSystem)
+struct _GArrowHDFSFileSystemClass
+{
+ GArrowFileSystemClass parent_class;
+};
+
+
+#define GARROW_TYPE_S3_FILE_SYSTEM (garrow_s3_file_system_get_type())
+G_DECLARE_DERIVABLE_TYPE(GArrowS3FileSystem,
+ garrow_s3_file_system,
+ GARROW,
+ S3_FILE_SYSTEM,
+ GArrowFileSystem)
+struct _GArrowS3FileSystemClass
+{
+ GArrowFileSystemClass parent_class;
+};
+
+
G_END_DECLS
diff --git a/c_glib/arrow-glib/file-system.hpp
b/c_glib/arrow-glib/file-system.hpp
index 175ac09..6130d2d 100644
--- a/c_glib/arrow-glib/file-system.hpp
+++ b/c_glib/arrow-glib/file-system.hpp
@@ -29,6 +29,10 @@ garrow_file_info_new_raw(const arrow::fs::FileInfo
&arrow_file_info);
arrow::fs::FileInfo *
garrow_file_info_get_raw(GArrowFileInfo *file_info);
+GArrowFileSystem *
+garrow_file_system_new_raw(
+ std::shared_ptr<arrow::fs::FileSystem> *arrow_file_system);
+
std::shared_ptr<arrow::fs::FileSystem>
garrow_file_system_get_raw(GArrowFileSystem *file_system);
diff --git a/c_glib/test/test-mock-file-system.rb
b/c_glib/test/test-mock-file-system.rb
new file mode 100644
index 0000000..c6148d6
--- /dev/null
+++ b/c_glib/test/test-mock-file-system.rb
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative "file-system-tests"
+
+class TestMockFileSystem < Test::Unit::TestCase
+ include FileSystemTests
+
+ def setup
+ @fs = Arrow::FileSystem.create("mock://")
+ end
+
+ def test_type_name
+ assert_equal("mock", @fs.type_name)
+ end
+end
