This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 0b059b9ed [#5507] feat(python): Support Azure blob storage for GVFS
python client (#5538)
0b059b9ed is described below
commit 0b059b9ed4b7dcf9e5388e1920c7a2739d6ae32a
Author: Qi Yu <[email protected]>
AuthorDate: Tue Nov 26 19:42:48 2024 +0800
[#5507] feat(python): Support Azure blob storage for GVFS python client
(#5538)
### What changes were proposed in this pull request?
Support GVFS python client to access ADSL fileset.
### Why are the changes needed?
This is a subsequent PR for #5508
Fix: #5507
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
IT locally.
---------
Co-authored-by: Jerry Shao <[email protected]>
---
clients/client-python/gravitino/filesystem/gvfs.py | 81 +++--
.../gravitino/filesystem/gvfs_config.py | 3 +
clients/client-python/requirements.txt | 3 +-
.../tests/integration/test_gvfs_with_abs.py | 366 +++++++++++++++++++++
docs/hadoop-catalog.md | 6 +-
docs/how-to-use-gvfs.md | 8 +
docs/security/security.md | 2 +
7 files changed, 445 insertions(+), 24 deletions(-)
diff --git a/clients/client-python/gravitino/filesystem/gvfs.py
b/clients/client-python/gravitino/filesystem/gvfs.py
index 8176c325a..0bb85f64e 100644
--- a/clients/client-python/gravitino/filesystem/gvfs.py
+++ b/clients/client-python/gravitino/filesystem/gvfs.py
@@ -50,6 +50,7 @@ class StorageType(Enum):
GCS = "gs"
S3A = "s3a"
OSS = "oss"
+ ABS = "abfss"
class FilesetContextPair:
@@ -320,6 +321,7 @@ class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
StorageType.GCS,
StorageType.S3A,
StorageType.OSS,
+ StorageType.ABS,
]:
src_context_pair.filesystem().mv(
self._strip_storage_protocol(storage_type, src_actual_path),
@@ -577,6 +579,21 @@ class
GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
)
actual_prefix = ops["host"] + ops["path"]
+ elif storage_location.startswith(f"{StorageType.ABS.value}://"):
+ ops = infer_storage_options(storage_location)
+ if "username" not in ops or "host" not in ops or "path" not in ops:
+ raise GravitinoRuntimeException(
+ f"Storage location:{storage_location} doesn't support now,
the username,"
+ f"host and path are required in the storage location."
+ )
+ actual_prefix =
f"{StorageType.ABS.value}://{ops['username']}@{ops['host']}{ops['path']}"
+
+ # the actual path may be '{container}/{path}', we need to add the
host and username
+ # get the path from {container}/{path}
+ if not actual_path.startswith(f"{StorageType.ABS}"):
+ path_without_username = actual_path[actual_path.index("/") + 1
:]
+ actual_path =
f"{StorageType.ABS.value}://{ops['username']}@{ops['host']}/{path_without_username}"
+
elif storage_location.startswith(f"{StorageType.LOCAL.value}:/"):
actual_prefix =
storage_location[len(f"{StorageType.LOCAL.value}:") :]
else:
@@ -613,33 +630,22 @@ class
GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
entry["name"], storage_location, virtual_location
)
- # if entry contains 'mtime', then return the entry with 'mtime' else
- # if entry contains 'LastModified', then return the entry with
'LastModified'
-
+ last_modified = None
if "mtime" in entry:
# HDFS and GCS
- return {
- "name": path,
- "size": entry["size"],
- "type": entry["type"],
- "mtime": entry["mtime"],
- }
-
- if "LastModified" in entry:
+ last_modified = entry["mtime"]
+ elif "LastModified" in entry:
# S3 and OSS
- return {
- "name": path,
- "size": entry["size"],
- "type": entry["type"],
- "mtime": entry["LastModified"],
- }
-
- # Unknown
+ last_modified = entry["LastModified"]
+ elif "last_modified" in entry:
+ # Azure Blob Storage
+ last_modified = entry["last_modified"]
+
return {
"name": path,
"size": entry["size"],
"type": entry["type"],
- "mtime": None,
+ "mtime": last_modified,
}
def _get_fileset_context(self, virtual_path: str, operation:
FilesetDataOperation):
@@ -745,6 +751,8 @@ class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
return StorageType.S3A
if path.startswith(f"{StorageType.OSS.value}://"):
return StorageType.OSS
+ if path.startswith(f"{StorageType.ABS.value}://"):
+ return StorageType.ABS
raise GravitinoRuntimeException(
f"Storage type doesn't support now. Path:{path}"
)
@@ -801,6 +809,13 @@ class
GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
if storage_type == StorageType.LOCAL:
return path[len(f"{StorageType.LOCAL.value}:") :]
+ ## We need to remove the protocol and accout from the path, for
instance,
+ # the path can be converted from 'abfss://container@account/path' to
+ # 'container/path'.
+ if storage_type == StorageType.ABS:
+ ops = infer_storage_options(path)
+ return ops["username"] + ops["path"]
+
# OSS has different behavior than S3 and GCS, if we do not remove the
# protocol, it will always return an empty array.
if storage_type == StorageType.OSS:
@@ -883,6 +898,8 @@ class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
fs = self._get_s3_filesystem()
elif storage_type == StorageType.OSS:
fs = self._get_oss_filesystem()
+ elif storage_type == StorageType.ABS:
+ fs = self._get_abs_filesystem()
else:
raise GravitinoRuntimeException(
f"Storage type: `{storage_type}` doesn't support now."
@@ -965,5 +982,29 @@ class
GravitinoVirtualFileSystem(fsspec.AbstractFileSystem):
endpoint=oss_endpoint_url,
)
+ def _get_abs_filesystem(self):
+ # get 'abs_account_name' from abs options, if the key is not found,
throw an exception
+ abs_account_name = self._options.get(
+ GVFSConfig.GVFS_FILESYSTEM_AZURE_ACCOUNT_NAME
+ )
+ if abs_account_name is None:
+ raise GravitinoRuntimeException(
+ "ABS account name is not found in the options."
+ )
+
+ # get 'abs_account_key' from abs options, if the key is not found,
throw an exception
+ abs_account_key = self._options.get(
+ GVFSConfig.GVFS_FILESYSTEM_AZURE_ACCOUNT_KEY
+ )
+ if abs_account_key is None:
+ raise GravitinoRuntimeException(
+ "ABS account key is not found in the options."
+ )
+
+ return importlib.import_module("adlfs").AzureBlobFileSystem(
+ account_name=abs_account_name,
+ account_key=abs_account_key,
+ )
+
fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem)
diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py
b/clients/client-python/gravitino/filesystem/gvfs_config.py
index c8e22e95f..4261fb48f 100644
--- a/clients/client-python/gravitino/filesystem/gvfs_config.py
+++ b/clients/client-python/gravitino/filesystem/gvfs_config.py
@@ -41,3 +41,6 @@ class GVFSConfig:
GVFS_FILESYSTEM_OSS_ACCESS_KEY = "oss_access_key_id"
GVFS_FILESYSTEM_OSS_SECRET_KEY = "oss_secret_access_key"
GVFS_FILESYSTEM_OSS_ENDPOINT = "oss_endpoint"
+
+ GVFS_FILESYSTEM_AZURE_ACCOUNT_NAME = "abs_account_name"
+ GVFS_FILESYSTEM_AZURE_ACCOUNT_KEY = "abs_account_key"
diff --git a/clients/client-python/requirements.txt
b/clients/client-python/requirements.txt
index 8eebd5727..217687ea8 100644
--- a/clients/client-python/requirements.txt
+++ b/clients/client-python/requirements.txt
@@ -25,4 +25,5 @@ pyarrow==15.0.2
cachetools==5.3.3
gcsfs==2024.3.1
s3fs==2024.3.1
-ossfs==2023.12.0
\ No newline at end of file
+ossfs==2023.12.0
+adlfs==2023.12.0
\ No newline at end of file
diff --git a/clients/client-python/tests/integration/test_gvfs_with_abs.py
b/clients/client-python/tests/integration/test_gvfs_with_abs.py
new file mode 100644
index 000000000..3377c07cd
--- /dev/null
+++ b/clients/client-python/tests/integration/test_gvfs_with_abs.py
@@ -0,0 +1,366 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+from random import randint
+import unittest
+
+
+from adlfs import AzureBlobFileSystem
+
+
+from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS
+from gravitino import (
+ gvfs,
+ GravitinoClient,
+ Catalog,
+ Fileset,
+)
+from gravitino.exceptions.base import GravitinoRuntimeException
+from gravitino.filesystem.gvfs_config import GVFSConfig
+from gravitino.filesystem.gvfs import StorageType
+
+
+logger = logging.getLogger(__name__)
+
+
+def azure_abs_is_prepared():
+ return (
+ os.environ.get("ABS_ACCOUNT_NAME")
+ and os.environ.get("ABS_ACCOUNT_KEY")
+ and os.environ.get("ABS_CONTAINER_NAME")
+ )
+
+
[email protected](azure_abs_is_prepared(), "Azure Blob Storage is not
prepared.")
+class TestGvfsWithABS(TestGvfsWithHDFS):
+ # Before running this test, please set the make sure azure-bundle-xxx.jar
has been
+ # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory
+ azure_abs_account_key = os.environ.get("ABS_ACCOUNT_KEY")
+ azure_abs_account_name = os.environ.get("ABS_ACCOUNT_NAME")
+ azure_abs_container_name = os.environ.get("ABS_CONTAINER_NAME")
+
+ metalake_name: str = "TestGvfsWithABS_metalake" + str(randint(1, 10000))
+
+ def setUp(self):
+ self.options = {
+ GVFSConfig.GVFS_FILESYSTEM_AZURE_ACCOUNT_NAME:
self.azure_abs_account_name,
+ GVFSConfig.GVFS_FILESYSTEM_AZURE_ACCOUNT_KEY:
self.azure_abs_account_key,
+ }
+
+ def tearDown(self):
+ self.options = {}
+
+ @classmethod
+ def setUpClass(cls):
+ cls._get_gravitino_home()
+
+ cls.hadoop_conf_path =
f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf"
+ # restart the server
+ cls.restart_server()
+ # create entity
+ cls._init_test_entities()
+
+ @classmethod
+ def tearDownClass(cls):
+ cls._clean_test_data()
+ # reset server conf in case of other ITs like HDFS has changed it and
fail
+ # to reset it
+ cls._reset_conf(cls.config, cls.hadoop_conf_path)
+ # restart server
+ cls.restart_server()
+
+ # clear all config in the conf_path
+ @classmethod
+ def _reset_conf(cls, config, conf_path):
+ logger.info("Reset %s.", conf_path)
+ if not os.path.exists(conf_path):
+ raise GravitinoRuntimeException(f"Conf file is not found at
`{conf_path}`.")
+ filtered_lines = []
+ with open(conf_path, mode="r", encoding="utf-8") as file:
+ origin_lines = file.readlines()
+
+ for line in origin_lines:
+ line = line.strip()
+ if line.startswith("#"):
+ # append annotations directly
+ filtered_lines.append(line + "\n")
+
+ with open(conf_path, mode="w", encoding="utf-8") as file:
+ for line in filtered_lines:
+ file.write(line)
+
+ @classmethod
+ def _init_test_entities(cls):
+ cls.gravitino_admin_client.create_metalake(
+ name=cls.metalake_name, comment="", properties={}
+ )
+ cls.gravitino_client = GravitinoClient(
+ uri="http://localhost:8090";, metalake_name=cls.metalake_name
+ )
+
+ cls.config = {}
+ cls.conf = {}
+ catalog = cls.gravitino_client.create_catalog(
+ name=cls.catalog_name,
+ catalog_type=Catalog.Type.FILESET,
+ provider=cls.catalog_provider,
+ comment="",
+ properties={
+ "filesystem-providers": "abs",
+ "abs-account-name": cls.azure_abs_account_name,
+ "abs-account-key": cls.azure_abs_account_key,
+ },
+ )
+ catalog.as_schemas().create_schema(
+ schema_name=cls.schema_name, comment="", properties={}
+ )
+
+ cls.fileset_storage_location: str = (
+
f"{cls.azure_abs_container_name}/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
+ )
+ cls.fileset_gvfs_location = (
+
f"gvfs://fileset/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
+ )
+ catalog.as_fileset_catalog().create_fileset(
+ ident=cls.fileset_ident,
+ fileset_type=Fileset.Type.MANAGED,
+ comment=cls.fileset_comment,
+ storage_location=(
+
f"abfss://{cls.azure_abs_container_name}@{cls.azure_abs_account_name}.dfs.core.windows.net/"
+ f"{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
+ ),
+ properties=cls.fileset_properties,
+ )
+
+ cls.fs = AzureBlobFileSystem(
+ account_name=cls.azure_abs_account_name,
+ account_key=cls.azure_abs_account_key,
+ )
+
+ def test_cat_file(self):
+ cat_dir = self.fileset_gvfs_location + "/test_cat"
+ cat_actual_dir = self.fileset_storage_location + "/test_cat"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+
+ self.check_mkdir(cat_dir, cat_actual_dir, fs)
+
+ cat_file = self.fileset_gvfs_location + "/test_cat/test.file"
+ cat_actual_file = self.fileset_storage_location + "/test_cat/test.file"
+ self.fs.touch(cat_actual_file)
+ self.assertTrue(self.fs.exists(cat_actual_file))
+ self.assertTrue(fs.exists(cat_file))
+
+ # test open and write file
+ with fs.open(cat_file, mode="wb") as f:
+ f.write(b"test_cat_file")
+ self.assertTrue(fs.info(cat_file)["size"] > 0)
+
+ # test cat file
+ content = fs.cat_file(cat_file)
+ self.assertEqual(b"test_cat_file", content)
+
+ def check_mkdir(self, gvfs_dir, actual_dir, gvfs_instance):
+ self.fs.mkdir(actual_dir)
+ self.assertFalse(self.fs.exists(actual_dir))
+ self.assertFalse(gvfs_instance.exists(gvfs_dir))
+
+ def check_makedirs(self, gvfs_dir, actual_dir, gvfs_instance):
+ self.fs.makedirs(actual_dir)
+ self.assertFalse(self.fs.exists(actual_dir))
+ self.assertFalse(gvfs_instance.exists(gvfs_dir))
+
+ def test_modified(self):
+ modified_dir = self.fileset_gvfs_location + "/test_modified"
+ modified_actual_dir = self.fileset_storage_location + "/test_modified"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+
+ self.check_mkdir(modified_dir, modified_actual_dir, fs)
+
+ # create a file under the dir 'modified_dir'.
+ file_path = modified_dir + "/test.txt"
+ fs.touch(file_path)
+ self.assertTrue(fs.exists(file_path))
+ self.assertIsNotNone(fs.modified(file_path))
+
+ def test_rm(self):
+ rm_dir = self.fileset_gvfs_location + "/test_rm"
+ rm_actual_dir = self.fileset_storage_location + "/test_rm"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+ self.check_mkdir(rm_dir, rm_actual_dir, fs)
+
+ rm_file = self.fileset_gvfs_location + "/test_rm/test.file"
+ rm_actual_file = self.fileset_storage_location + "/test_rm/test.file"
+ fs.touch(rm_file)
+ self.assertTrue(self.fs.exists(rm_actual_file))
+ self.assertTrue(fs.exists(rm_file))
+
+ # test delete file
+ fs.rm(rm_file)
+ self.assertFalse(fs.exists(rm_file))
+
+ # test delete dir with recursive = false
+ rm_new_file = self.fileset_gvfs_location + "/test_rm/test_new.file"
+ rm_new_actual_file = self.fileset_storage_location +
"/test_rm/test_new.file"
+ self.fs.touch(rm_new_actual_file)
+ self.assertTrue(self.fs.exists(rm_new_actual_file))
+ self.assertTrue(fs.exists(rm_new_file))
+
+ def test_rmdir(self):
+ rmdir_dir = self.fileset_gvfs_location + "/test_rmdir"
+ rmdir_actual_dir = self.fileset_storage_location + "/test_rmdir"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+ self.check_mkdir(rmdir_dir, rmdir_actual_dir, fs)
+
+ rmdir_file = self.fileset_gvfs_location + "/test_rmdir/test.file"
+ rmdir_actual_file = self.fileset_storage_location +
"/test_rmdir/test.file"
+ self.fs.touch(rmdir_actual_file)
+ self.assertTrue(self.fs.exists(rmdir_actual_file))
+ self.assertTrue(fs.exists(rmdir_file))
+
+ # NOT IMPLEMENTED for ABS
+ # fs.rm_file(rmdir_file)
+
+ def test_mkdir(self):
+ mkdir_dir = self.fileset_gvfs_location + "/test_mkdir"
+ mkdir_actual_dir = self.fileset_storage_location + "/test_mkdir"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+
+ # it actually takes no effect.
+ self.check_mkdir(mkdir_dir, mkdir_actual_dir, fs)
+
+ # check whether it will automatically create the bucket if
'create_parents'
+ # is set to True.
+ new_bucket = self.azure_abs_container_name + "2"
+ mkdir_dir = mkdir_dir.replace(self.azure_abs_container_name,
new_bucket)
+ mkdir_actual_dir = mkdir_actual_dir.replace(
+ self.azure_abs_container_name, new_bucket
+ )
+ fs.mkdir(mkdir_dir, create_parents=True)
+
+ self.assertFalse(self.fs.exists(mkdir_actual_dir))
+ self.assertFalse(fs.exists(mkdir_dir))
+
+
self.assertFalse(self.fs.exists(f"{StorageType.ABS.value}://{new_bucket}"))
+
+ def test_makedirs(self):
+ mkdir_dir = self.fileset_gvfs_location + "/test_mkdir"
+ mkdir_actual_dir = self.fileset_storage_location + "/test_mkdir"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+
+ # it actually takes no effect.
+ self.check_mkdir(mkdir_dir, mkdir_actual_dir, fs)
+
+ # check whether it will automatically create the bucket if
'create_parents'
+ # is set to True.
+ new_bucket = self.azure_abs_container_name + "1"
+ mkdir_dir = mkdir_dir.replace(self.azure_abs_container_name,
new_bucket)
+ mkdir_actual_dir = mkdir_actual_dir.replace(
+ self.azure_abs_container_name, new_bucket
+ )
+
+ fs.makedirs(mkdir_dir)
+
+ self.assertFalse(self.fs.exists(mkdir_actual_dir))
+
+ self.assertFalse(fs.exists(mkdir_dir))
+
self.assertFalse(self.fs.exists(f"{StorageType.ABS.value}://{new_bucket}"))
+
+ def test_ls(self):
+ ls_dir = self.fileset_gvfs_location + "/test_ls"
+ ls_actual_dir = self.fileset_storage_location + "/test_ls"
+
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+
+ self.check_mkdir(ls_dir, ls_actual_dir, fs)
+
+ ls_file = self.fileset_gvfs_location + "/test_ls/test.file"
+ ls_actual_file = self.fileset_storage_location + "/test_ls/test.file"
+ self.fs.touch(ls_actual_file)
+ self.assertTrue(self.fs.exists(ls_actual_file))
+
+ # test detail = false
+ file_list_without_detail = fs.ls(ls_dir, detail=False)
+ self.assertEqual(1, len(file_list_without_detail))
+ self.assertEqual(file_list_without_detail[0], ls_file[len("gvfs://")
:])
+
+ # test detail = true
+ file_list_with_detail = fs.ls(ls_dir, detail=True)
+ self.assertEqual(1, len(file_list_with_detail))
+ self.assertEqual(file_list_with_detail[0]["name"],
ls_file[len("gvfs://") :])
+
+ def test_rm_file(self):
+ rm_file_dir = self.fileset_gvfs_location + "/test_rm_file"
+ rm_file_actual_dir = self.fileset_storage_location + "/test_rm_file"
+ fs = gvfs.GravitinoVirtualFileSystem(
+ server_uri="http://localhost:8090";,
+ metalake_name=self.metalake_name,
+ options=self.options,
+ **self.conf,
+ )
+ self.check_mkdir(rm_file_dir, rm_file_actual_dir, fs)
+
+ rm_file_file = self.fileset_gvfs_location + "/test_rm_file/test.file"
+ rm_file_actual_file = self.fileset_storage_location +
"/test_rm_file/test.file"
+ self.fs.touch(rm_file_actual_file)
+ self.assertTrue(self.fs.exists(rm_file_actual_file))
+ self.assertTrue(fs.exists(rm_file_file))
+
+ # test delete file
+ with self.assertRaises(NotImplementedError):
+ fs.rm_file(rm_file_file)
+ self.assertTrue(fs.exists(rm_file_file))
+
+ # test delete dir
+ with self.assertRaises(NotImplementedError):
+ fs.rm_file(rm_file_dir)
diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md
index 15f42883e..26d27dce8 100644
--- a/docs/hadoop-catalog.md
+++ b/docs/hadoop-catalog.md
@@ -83,14 +83,14 @@ In the meantime, you need to place the corresponding bundle
jar [`gravitino-aliy
|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|-------------------------------------------|------------------|
| `filesystem-providers` | The file system providers to add. Set it to
`abs` if it's a Azure Blob Storage fileset, or a comma separated string that
contains `abs` like `oss,abs,s3` to support multiple kinds of fileset including
`abs`. | (none) | Yes
| 0.8.0-incubating |
| `default-filesystem-provider` | The name default filesystem providers of
this Hadoop catalog if users do not specify the scheme in the URI. Default
value is `builtin-local`, for Azure Blob Storage, if we set this value, we can
omit the prefix 'abfss://' in the location. | `builtin-local` | No
| 0.8.0-incubating |
-| `abs-account-name` | The account name of Azure Blob storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
-| `abs-account-key` | The account key of Azure Blob storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
+| `abs-account-name` | The account name of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
+| `abs-account-key` | The account key of Azure Blob Storage.
| (none) | Yes if it's a Azure Blob
Storage fileset. | 0.8.0-incubating |
Similar to the above, you need to place the corresponding bundle jar
[`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/)
in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`.
:::note
- Gravitino contains builtin file system providers for local file
system(`builtin-local`) and HDFS(`builtin-hdfs`), that is to say if
`filesystem-providers` is not set, Gravitino will still support local file
system and HDFS. Apart from that, you can set the `filesystem-providers` to
support other file systems like S3, GCS, OSS or custom file system.
-- `default-filesystem-provider` is used to set the default file system
provider for the Hadoop catalog. If the user does not specify the scheme in the
URI, Gravitino will use the default file system provider to access the fileset.
For example, if the default file system provider is set to `builtin-local`, the
user can omit the prefix `file://` in the location.
+- `default-filesystem-provider` is used to set the default file system
provider for the Hadoop catalog. If the user does not specify the scheme in the
URI, Gravitino will use the default file system provider to access the fileset.
For example, if the default file system provider is set to `builtin-local`, the
user can omit the prefix `file:///` in the location.
:::
#### How to custom your own HCFS file system fileset?
diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md
index 6ea3a972d..5b79a80f8 100644
--- a/docs/how-to-use-gvfs.md
+++ b/docs/how-to-use-gvfs.md
@@ -457,6 +457,14 @@ The following properties are required if you want to
access the OSS fileset via
| `oss_access_key_id` | The access key of the Aliyun OSS. | (none)
| Yes if it's a OSS fileset. | 0.7.0-incubating |
| `oss_secret_access_key` | The secret key of the Aliyun OSS. | (none)
| Yes if it's a OSS fileset. | 0.7.0-incubating |
+For Azure Blob Storage fileset, you need to configure the following properties:
+
+| Configuration item | Description | Default value
| Required | Since version |
+|--------------------|----------------------------------------|---------------|-------------------------------------------|------------------|
+| `abs_account_name` | The account name of Azure Blob Storage | (none)
| Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating |
+| `abs_account_key` | The account key of Azure Blob Storage | (none)
| Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating |
+
+
You can configure these properties when obtaining the `Gravitino Virtual
FileSystem` in Python like this:
```python
diff --git a/docs/security/security.md b/docs/security/security.md
index 62ffa268d..a8b4ad35a 100644
--- a/docs/security/security.md
+++ b/docs/security/security.md
@@ -13,6 +13,8 @@ Gravitino provides features that ensure the highest levels of
security for you.
Gravitino has supported the following security features:
+### [Authorization push down](./authorization-pushdown.md)
+
### [Authentication](how-to-authenticate.md)
### [HTTPS](how-to-use-https.md)
