This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 39f3ca518 [#4279] feat(client-python): Add the `getFileLocation`
interface code skeleton in Python Client (#4373)
39f3ca518 is described below
commit 39f3ca51808f0c5641e94f11f1c56094c25216fe
Author: xloya <[email protected]>
AuthorDate: Tue Sep 24 13:41:28 2024 +0800
[#4279] feat(client-python): Add the `getFileLocation` interface code
skeleton in Python Client (#4373)
### What changes were proposed in this pull request?
Added an interface code skeleton in Python Client for obtaining the file
location so that the client can report some necessary information for
the server to audit and simplify some check logics in Python GVFS later.
Depend on #4320.
### Why are the changes needed?
Fix: #4279
### How was this patch tested?
Add UTs and ITs.
---
clients/client-python/gravitino/audit/__init__.py | 16 +++++
.../gravitino/audit/caller_context.py | 71 +++++++++++++++++++
.../gravitino/audit/fileset_audit_constants.py | 28 ++++++++
.../gravitino/audit/fileset_data_operation.py | 81 ++++++++++++++++++++++
.../gravitino/audit/internal_client_type.py | 35 ++++++++++
.../gravitino/catalog/fileset_catalog.py | 12 ++++
.../dto/responses/file_location_response.py | 42 +++++++++++
.../tests/unittests/audit/__init__.py | 16 +++++
.../tests/unittests/audit/test_caller_context.py | 64 +++++++++++++++++
.../tests/unittests/test_responses.py | 41 +++++++++++
.../org/apache/gravitino/audit/CallerContext.java | 2 +-
.../gravitino/audit/FilesetDataOperation.java | 36 ++++++----
12 files changed, 431 insertions(+), 13 deletions(-)
diff --git a/clients/client-python/gravitino/audit/__init__.py
b/clients/client-python/gravitino/audit/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/clients/client-python/gravitino/audit/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/clients/client-python/gravitino/audit/caller_context.py
b/clients/client-python/gravitino/audit/caller_context.py
new file mode 100644
index 000000000..07e30a4d3
--- /dev/null
+++ b/clients/client-python/gravitino/audit/caller_context.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import threading
+from typing import Dict
+
+caller_context_holder = threading.local()
+
+
+class CallerContext:
+ """A class defining the caller context for auditing coarse-grained
operations."""
+
+ _context: Dict[str, str]
+
+ def __init__(self, context: Dict[str, str]):
+ """Initialize the CallerContext.
+
+ Args:
+ context: The context dict.
+ """
+ self._context = context
+
+ def context(self) -> Dict[str, str]:
+ """Returns the context dict in the caller context.
+
+ Returns:
+ The context dict.
+ """
+ return self._context
+
+
+class CallerContextHolder:
+ """A thread local holder for the CallerContext."""
+
+ @staticmethod
+ def set(context: CallerContext):
+ """Set the CallerContext in the thread local.
+
+ Args:
+ context: The CallerContext to set.
+ """
+ caller_context_holder.caller_context = context
+
+ @staticmethod
+ def get():
+ """Get the CallerContext from the thread local.
+
+ Returns:
+ The CallerContext.
+ """
+ if not hasattr(caller_context_holder, "caller_context"):
+ return None
+ return caller_context_holder.caller_context
+
+ @staticmethod
+ def remove():
+ """Remove the CallerContext from the thread local."""
+ del caller_context_holder.caller_context
diff --git a/clients/client-python/gravitino/audit/fileset_audit_constants.py
b/clients/client-python/gravitino/audit/fileset_audit_constants.py
new file mode 100644
index 000000000..3ce0bfe95
--- /dev/null
+++ b/clients/client-python/gravitino/audit/fileset_audit_constants.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+class FilesetAuditConstants:
+ """Constants used for fileset data operation audits."""
+
+ HTTP_HEADER_INTERNAL_CLIENT_TYPE = "InternalClientType"
+ """The HTTP header used to pass the internal client type.
+ """
+
+ HTTP_HEADER_FILESET_DATA_OPERATION = "FilesetDataOperation"
+ """The HTTP header used to pass the fileset data operation.
+ """
diff --git a/clients/client-python/gravitino/audit/fileset_data_operation.py
b/clients/client-python/gravitino/audit/fileset_data_operation.py
new file mode 100644
index 000000000..5f7a5794b
--- /dev/null
+++ b/clients/client-python/gravitino/audit/fileset_data_operation.py
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from enum import Enum
+
+
+class FilesetDataOperation(Enum):
+ """An enum class containing fileset data operations that supported."""
+
+ CREATE = "CREATE"
+ """Creates a new file.
+ """
+
+ OPEN = "OPEN"
+ """Opens a file.
+ """
+
+ APPEND = "APPEND"
+ """Appends some content into a file.
+ """
+
+ RENAME = "RENAME"
+ """Renames a file or a directory.
+ """
+
+ DELETE = "DELETE"
+ """Deletes a file or a directory.
+ """
+
+ GET_FILE_STATUS = "GET_FILE_STATUS"
+ """Gets a file status from a file or a directory.
+ """
+
+ LIST_STATUS = "LIST_STATUS"
+ """Lists file statuses under a directory.
+ """
+
+ MKDIRS = "MKDIRS"
+ """Creates a directory.
+ """
+
+ EXISTS = "EXISTS"
+ """Checks if a file or a directory exists.
+ """
+
+ CREATED_TIME = "CREATED_TIME"
+ """Gets the created time of a file.
+ """
+
+ MODIFIED_TIME = "MODIFIED_TIME"
+ """Gets the modified time of a file.
+ """
+
+ COPY_FILE = "COPY_FILE"
+ """Copies a file.
+ """
+
+ CAT_FILE = "CAT_FILE"
+ """Gets the content of a file.
+ """
+
+ GET_FILE = "GET_FILE"
+ """Copies a remote file to local.
+ """
+
+ UNKNOWN = "UNKNOWN"
+ """Unknown data operation.
+ """
diff --git a/clients/client-python/gravitino/audit/internal_client_type.py
b/clients/client-python/gravitino/audit/internal_client_type.py
new file mode 100644
index 000000000..ae8305cbd
--- /dev/null
+++ b/clients/client-python/gravitino/audit/internal_client_type.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from enum import Enum
+
+
+class InternalClientType(Enum):
+ """An enum class containing internal client type that supported."""
+
+ HADOOP_GVFS = "HADOOP_GVFS"
+ """The client type is
`org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem` which in
+ the filesystem-hadoop3 module.
+ """
+
+ PYTHON_GVFS = "PYTHON_GVFS"
+ """The client type is
`gravitino.filesystem.gvfs.GravitinoVirtualFileSystem` which in the
+ client-python module.
+ """
+
+ UNKNOWN = "UNKNOWN"
+ """The client type is unknown.
+ """
diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py
b/clients/client-python/gravitino/catalog/fileset_catalog.py
index b7f9fa593..3b2f0f717 100644
--- a/clients/client-python/gravitino/catalog/fileset_catalog.py
+++ b/clients/client-python/gravitino/catalog/fileset_catalog.py
@@ -234,6 +234,18 @@ class FilesetCatalog(BaseSchemaCatalog):
return drop_resp.dropped()
+ def get_file_location(self, ident: NameIdentifier, sub_path: str) -> str:
+ """Get the actual location of a file or directory based on the storage
location of Fileset and the sub path.
+
+ Args:
+ ident: A fileset identifier, which should be "schema.fileset"
format.
+ sub_path: The sub path of the file or directory.
+
+ Returns:
+ The actual location of the file or directory.
+ """
+ raise NotImplementedError("Not implemented yet")
+
@staticmethod
def check_fileset_namespace(namespace: Namespace):
Namespace.check(
diff --git
a/clients/client-python/gravitino/dto/responses/file_location_response.py
b/clients/client-python/gravitino/dto/responses/file_location_response.py
new file mode 100644
index 000000000..e27dc156e
--- /dev/null
+++ b/clients/client-python/gravitino/dto/responses/file_location_response.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+
+from dataclasses_json import config
+from gravitino.dto.responses.base_response import BaseResponse
+from gravitino.exceptions.base import IllegalArgumentException
+
+
+@dataclass
+class FileLocationResponse(BaseResponse):
+ """Response for the actual file location."""
+
+ _file_location: str = field(metadata=config(field_name="fileLocation"))
+
+ def file_location(self) -> str:
+ return self._file_location
+
+ def validate(self):
+ """Validates the response data.
+
+ Raises:
+ IllegalArgumentException if file location is not set.
+ """
+ super().validate()
+ if self._file_location is None or len(self.file_location()) == 0:
+ raise IllegalArgumentException("file location must not be null")
diff --git a/clients/client-python/tests/unittests/audit/__init__.py
b/clients/client-python/tests/unittests/audit/__init__.py
new file mode 100644
index 000000000..13a83393a
--- /dev/null
+++ b/clients/client-python/tests/unittests/audit/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/clients/client-python/tests/unittests/audit/test_caller_context.py
b/clients/client-python/tests/unittests/audit/test_caller_context.py
new file mode 100644
index 000000000..93031a256
--- /dev/null
+++ b/clients/client-python/tests/unittests/audit/test_caller_context.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import threading
+import unittest
+from typing import Dict
+
+from gravitino.audit.caller_context import CallerContextHolder, CallerContext
+
+
+class TestCallerContext(unittest.TestCase):
+
+ def test_caller_context(self):
+ thread_names_and_values = [
+ ("Thread1", {"k1": "v1", "k2": "v2"}),
+ ("Thread2", {"k11": "v11", "k21": "v21"}),
+ ]
+ test_threads = []
+ for thread_name, value in thread_names_and_values:
+ t = threading.Thread(
+ target=self._set_thread_local_context, args=(thread_name,
value)
+ )
+ t.start()
+ test_threads.append(t)
+
+ for t in test_threads:
+ t.join()
+
+ def _set_thread_local_context(self, thread_name, context: Dict[str, str]):
+ caller_context: CallerContext = CallerContext(context)
+ CallerContextHolder.set(caller_context)
+
+ try:
+ if thread_name == "Thread1":
+ self.assertEqual(
+ CallerContextHolder.get().context()["k1"], context["k1"]
+ )
+ self.assertEqual(
+ CallerContextHolder.get().context()["k2"], context["k2"]
+ )
+ if thread_name == "Thread2":
+ self.assertEqual(
+ CallerContextHolder.get().context()["k11"], context["k11"]
+ )
+ self.assertEqual(
+ CallerContextHolder.get().context()["k21"], context["k21"]
+ )
+ finally:
+ CallerContextHolder.remove()
+
+ self.assertIsNone(CallerContextHolder.get())
diff --git a/clients/client-python/tests/unittests/test_responses.py
b/clients/client-python/tests/unittests/test_responses.py
new file mode 100644
index 000000000..19d403ad3
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_responses.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import unittest
+
+from gravitino.dto.responses.file_location_response import FileLocationResponse
+from gravitino.exceptions.base import IllegalArgumentException
+
+
+class TestResponses(unittest.TestCase):
+ def test_file_location_response(self):
+ json_data = {"code": 0, "fileLocation": "file:/test/1"}
+ json_str = json.dumps(json_data)
+ file_location_resp: FileLocationResponse =
FileLocationResponse.from_json(
+ json_str
+ )
+ self.assertEqual(file_location_resp.file_location(), "file:/test/1")
+ file_location_resp.validate()
+
+ def test_file_location_response_exception(self):
+ json_data = {"code": 0, "fileLocation": ""}
+ json_str = json.dumps(json_data)
+ file_location_resp: FileLocationResponse =
FileLocationResponse.from_json(
+ json_str
+ )
+ with self.assertRaises(IllegalArgumentException):
+ file_location_resp.validate()
diff --git a/common/src/main/java/org/apache/gravitino/audit/CallerContext.java
b/common/src/main/java/org/apache/gravitino/audit/CallerContext.java
index e0289779f..0ae2250d0 100644
--- a/common/src/main/java/org/apache/gravitino/audit/CallerContext.java
+++ b/common/src/main/java/org/apache/gravitino/audit/CallerContext.java
@@ -24,7 +24,7 @@ import com.google.common.base.Preconditions;
import java.util.Map;
/**
- * A class defining the caller context for auditing coarse granularity
operations.
+ * A class defining the caller context for auditing coarse-grained operations.
*
* <p>Reference:
*
diff --git
a/common/src/main/java/org/apache/gravitino/audit/FilesetDataOperation.java
b/common/src/main/java/org/apache/gravitino/audit/FilesetDataOperation.java
index 35ab16cd4..b76d1f91b 100644
--- a/common/src/main/java/org/apache/gravitino/audit/FilesetDataOperation.java
+++ b/common/src/main/java/org/apache/gravitino/audit/FilesetDataOperation.java
@@ -21,29 +21,41 @@ package org.apache.gravitino.audit;
/** An enum class containing fileset data operations that supported. */
public enum FilesetDataOperation {
- /** This data operation means that create a new file. */
+ /** Creates a new file. */
CREATE,
- /** This data operation means that open a file. */
+ /** Opens a file. */
OPEN,
- /** This data operation means that append some content into a file. */
+ /** Appends some content into a file. */
APPEND,
- /** This data operation means that rename a file or a directory. */
+ /** Renames a file or a directory. */
RENAME,
- /** This data operation means that delete a file or a directory. */
+ /** Deletes a file or a directory. */
DELETE,
- /** This data operation means that get a file status from a file or a
directory. */
+ /** Gets a file status from a file or a directory. */
GET_FILE_STATUS,
- /** This data operation means that list file statuses under a directory. */
+ /** Lists file statuses under a directory. */
LIST_STATUS,
- /** This data operation means that create a directory. */
+ /** Creates a directory. */
MKDIRS,
- /** This data operation means that get the default replication of a file
system. */
+ /** Gets the default replication of a file system. */
GET_DEFAULT_REPLICATION,
- /** This data operation means that get the default block size of a file
system. */
+ /** Gets the default block size of a file system. */
GET_DEFAULT_BLOCK_SIZE,
- /** This data operation means that set current working directory. */
+ /** Sets current working directory. */
SET_WORKING_DIR,
- /** This data operation means that it is an unknown data operation. */
+ /** Gets the current working directory. */
+ EXISTS,
+ /** Gets the created time of a file. */
+ CREATED_TIME,
+ /** Gets the modified time of a file. */
+ MODIFIED_TIME,
+ /** Copies a file. */
+ COPY_FILE,
+ /** Gets the content of a file. */
+ CAT_FILE,
+ /** Copies a remote file to local. */
+ GET_FILE,
+ /** Unknown data operation. */
UNKNOWN;
/**
