This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c913b9ed5a [Python] Reformat code for init_py file (#6159)
c913b9ed5a is described below
commit c913b9ed5afa5341b3fda164572369fca6d656c5
Author: ChengHui Chen <27797326+chenghuic...@users.noreply.github.com>
AuthorDate: Wed Aug 27 15:22:53 2025 +0800
[Python] Reformat code for init_py file (#6159)
---
paimon-python/pypaimon/api/__init__.py | 363 ---------------------
paimon-python/pypaimon/api/api_request.py | 6 +-
paimon-python/pypaimon/api/api_response.py | 10 +-
paimon-python/pypaimon/api/auth.py | 6 +-
paimon-python/pypaimon/api/client.py | 115 +------
paimon-python/pypaimon/api/resource_paths.py | 70 ++++
.../pypaimon/api/{__init__.py => rest_api.py} | 101 +-----
paimon-python/pypaimon/api/rest_exception.py | 111 +++++++
paimon-python/pypaimon/api/rest_util.py | 43 +++
paimon-python/pypaimon/api/token_loader.py | 5 +-
paimon-python/pypaimon/catalog/catalog.py | 2 +-
.../{table => catalog}/catalog_environment.py | 7 +-
paimon-python/pypaimon/catalog/catalog_factory.py | 1 +
.../pypaimon/catalog/filesystem_catalog.py | 4 +-
.../pypaimon/catalog/{ => rest}/property_change.py | 0
.../pypaimon/catalog/rest/rest_catalog.py | 45 +--
.../pypaimon/catalog/rest/rest_token_file_io.py | 2 +-
.../pypaimon/catalog/{ => rest}/table_metadata.py | 0
paimon-python/pypaimon/common/identifier.py | 2 +-
.../pypaimon/common/{rest_json.py => json_util.py} | 0
paimon-python/pypaimon/pvfs/__init__.py | 6 +-
paimon-python/pypaimon/schema/schema.py | 2 +-
paimon-python/pypaimon/schema/schema_manager.py | 2 +-
paimon-python/pypaimon/schema/table_schema.py | 2 +-
.../catalog_snapshot_commit.py | 3 +-
.../renaming_snapshot_commit.py | 5 +-
paimon-python/pypaimon/snapshot/snapshot.py | 2 +-
.../{catalog => snapshot}/snapshot_commit.py | 4 +-
.../pypaimon/snapshot/snapshot_manager.py | 2 +-
paimon-python/pypaimon/table/file_store_table.py | 2 +-
paimon-python/pypaimon/tests/api_test.py | 35 +-
paimon-python/pypaimon/tests/predicates_test.py | 2 +-
paimon-python/pypaimon/tests/pvfs_test.py | 4 +-
.../pypaimon/tests/reader_append_only_test.py | 2 +-
.../pypaimon/tests/reader_primary_key_test.py | 2 +-
.../pypaimon/tests/rest_catalog_base_test.py | 5 +-
paimon-python/pypaimon/tests/rest_server.py | 42 +--
.../pypaimon/tests/test_file_store_commit.py | 2 +-
.../tests/test_rest_catalog_commit_snapshot.py | 10 +-
paimon-python/pypaimon/write/commit_message.py | 1 +
paimon-python/pypaimon/write/file_store_commit.py | 3 +-
paimon-python/pypaimon/write/row_key_extractor.py | 1 +
42 files changed, 364 insertions(+), 668 deletions(-)
diff --git a/paimon-python/pypaimon/api/__init__.py
b/paimon-python/pypaimon/api/__init__.py
index a384c3f123..a67d5ea255 100644
--- a/paimon-python/pypaimon/api/__init__.py
+++ b/paimon-python/pypaimon/api/__init__.py
@@ -14,366 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-import logging
-from typing import Callable, Dict, List, Optional
-from urllib.parse import unquote
-
-from pypaimon.api.api_response import (CommitTableResponse, ConfigResponse,
- GetDatabaseResponse, GetTableResponse,
- GetTableTokenResponse,
- ListDatabasesResponse,
- ListTablesResponse, PagedList,
- PagedResponse)
-from pypaimon.api.api_request import (AlterDatabaseRequest,
- CommitTableRequest,
- CreateDatabaseRequest,
- CreateTableRequest, RenameTableRequest)
-from pypaimon.api.auth import AuthProviderFactory, RESTAuthFunction
-from pypaimon.api.client import HttpClient
-from pypaimon.api.typedef import T
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
-from pypaimon.common.config import CatalogOptions
-from pypaimon.common.identifier import Identifier
-from pypaimon.schema.schema import Schema
-from pypaimon.snapshot.snapshot import Snapshot
-
-
-class RESTException(Exception):
- pass
-
-
-class AlreadyExistsException(RESTException):
- pass
-
-
-class ForbiddenException(RESTException):
- pass
-
-
-class NoSuchResourceException(RESTException):
- pass
-
-
-class RESTUtil:
- @staticmethod
- def encode_string(value: str) -> str:
- import urllib.parse
-
- return urllib.parse.quote(value)
-
- @staticmethod
- def decode_string(encoded: str) -> str:
- """Decode URL-encoded string"""
- return unquote(encoded)
-
- @staticmethod
- def extract_prefix_map(
- options: Dict[str, str], prefix: str) -> Dict[str, str]:
- result = {}
- config = options
- for key, value in config.items():
- if key.startswith(prefix):
- new_key = key[len(prefix):]
- result[new_key] = str(value)
- return result
-
-
-class ResourcePaths:
- V1 = "v1"
- DATABASES = "databases"
- TABLES = "tables"
- TABLE_DETAILS = "table-details"
-
- def __init__(self, prefix: str):
- self.base_path = f"/{self.V1}/{prefix}".rstrip("/")
-
- @classmethod
- def for_catalog_properties(
- cls, options: dict[str, str]) -> "ResourcePaths":
- prefix = options.get(CatalogOptions.PREFIX, "")
- return cls(prefix)
-
- @staticmethod
- def config() -> str:
- return f"/{ResourcePaths.V1}/config"
-
- def databases(self) -> str:
- return f"{self.base_path}/{self.DATABASES}"
-
- def database(self, name: str) -> str:
- return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(name)}"
-
- def tables(self, database_name: Optional[str] = None) -> str:
- if database_name:
- return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}/{self.TABLES}"
- return f"{self.base_path}/{self.TABLES}"
-
- def table(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}")
-
- def table_details(self, database_name: str) -> str:
- return
f"{self.base_path}/{self.DATABASES}/{database_name}/{self.TABLE_DETAILS}"
-
- def table_token(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/token")
-
- def rename_table(self) -> str:
- return f"{self.base_path}/{self.TABLES}/rename"
-
- def commit_table(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/commit")
-
-
-class RESTApi:
- HEADER_PREFIX = "header."
- MAX_RESULTS = "maxResults"
- PAGE_TOKEN = "pageToken"
- DATABASE_NAME_PATTERN = "databaseNamePattern"
- TABLE_NAME_PATTERN = "tableNamePattern"
- TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000
-
- def __init__(self, options: Dict[str, str], config_required: bool = True):
- self.logger = logging.getLogger(self.__class__.__name__)
- self.client = HttpClient(options.get(CatalogOptions.URI))
- auth_provider = AuthProviderFactory.create_auth_provider(options)
- base_headers = RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
-
- if config_required:
- warehouse = options.get(CatalogOptions.WAREHOUSE)
- query_params = {}
- if warehouse:
- query_params[CatalogOptions.WAREHOUSE] =
RESTUtil.encode_string(
- warehouse)
-
- config_response = self.client.get_with_params(
- ResourcePaths.config(),
- query_params,
- ConfigResponse,
- RESTAuthFunction(base_headers, auth_provider),
- )
- options = config_response.merge(options)
- base_headers.update(
- RESTUtil.extract_prefix_map(options, self.HEADER_PREFIX)
- )
-
- self.rest_auth_function = RESTAuthFunction(base_headers, auth_provider)
- self.options = options
- self.resource_paths = ResourcePaths.for_catalog_properties(options)
-
- def __build_paged_query_params(
- max_results: Optional[int],
- page_token: Optional[str],
- name_patterns: Dict[str, str],
- ) -> Dict[str, str]:
- query_params = {}
- if max_results is not None and max_results > 0:
- query_params[RESTApi.MAX_RESULTS] = str(max_results)
-
- if page_token is not None and page_token.strip():
- query_params[RESTApi.PAGE_TOKEN] = page_token
-
- for key, value in name_patterns:
- if key and value and key.strip() and value.strip():
- query_params[key] = value
-
- return query_params
-
- def __list_data_from_page_api(
- self, page_api: Callable[[Dict[str, str]], PagedResponse[T]]
- ) -> List[T]:
- results = []
- query_params = {}
- page_token = None
-
- while True:
- if page_token:
- query_params[RESTApi.PAGE_TOKEN] = page_token
- elif RESTApi.PAGE_TOKEN in query_params:
- del query_params[RESTApi.PAGE_TOKEN]
-
- response = page_api(query_params)
-
- if response.data:
- results.extend(response.data())
-
- page_token = response.next_page_token
-
- if not page_token or not response.data:
- break
-
- return results
-
- def get_options(self) -> dict[str, str]:
- return self.options
-
- def list_databases(self) -> List[str]:
- return self.__list_data_from_page_api(
- lambda query_params: self.client.get_with_params(
- self.resource_paths.databases(),
- query_params,
- ListDatabasesResponse,
- self.rest_auth_function,
- )
- )
-
- def list_databases_paged(
- self,
- max_results: Optional[int] = None,
- page_token: Optional[str] = None,
- database_name_pattern: Optional[str] = None,
- ) -> PagedList[str]:
-
- response = self.client.get_with_params(
- self.resource_paths.databases(),
- self.__build_paged_query_params(
- max_results,
- page_token,
- {self.DATABASE_NAME_PATTERN: database_name_pattern},
- ),
- ListDatabasesResponse,
- self.rest_auth_function,
- )
-
- databases = response.data() or []
- return PagedList(databases, response.get_next_page_token())
-
- def create_database(self, name: str, options: Dict[str, str]) -> None:
- request = CreateDatabaseRequest(name, options)
- self.client.post(
- self.resource_paths.databases(), request, self.rest_auth_function
- )
-
- def get_database(self, name: str) -> GetDatabaseResponse:
- return self.client.get(
- self.resource_paths.database(name),
- GetDatabaseResponse,
- self.rest_auth_function,
- )
-
- def drop_database(self, name: str) -> None:
- self.client.delete(
- self.resource_paths.database(name),
- self.rest_auth_function)
-
- def alter_database(
- self,
- name: str,
- removals: Optional[List[str]] = None,
- updates: Optional[Dict[str, str]] = None,
- ):
- if not name or not name.strip():
- raise ValueError("Database name cannot be empty")
- removals = removals or []
- updates = updates or {}
- request = AlterDatabaseRequest(removals, updates)
-
- return self.client.post(
- self.resource_paths.database(name),
- request,
- self.rest_auth_function)
-
- def list_tables(self, database_name: str) -> List[str]:
- return self.__list_data_from_page_api(
- lambda query_params: self.client.get_with_params(
- self.resource_paths.tables(database_name),
- query_params,
- ListTablesResponse,
- self.rest_auth_function,
- )
- )
-
- def list_tables_paged(
- self,
- database_name: str,
- max_results: Optional[int] = None,
- page_token: Optional[str] = None,
- table_name_pattern: Optional[str] = None,
- ) -> PagedList[str]:
- response = self.client.get_with_params(
- self.resource_paths.tables(database_name),
- self.__build_paged_query_params(
- max_results, page_token, {self.TABLE_NAME_PATTERN:
table_name_pattern}
- ),
- ListTablesResponse,
- self.rest_auth_function,
- )
-
- tables = response.data() or []
- return PagedList(tables, response.get_next_page_token())
-
- def create_table(self, identifier: Identifier, schema: Schema) -> None:
- request = CreateTableRequest(identifier, schema)
- return self.client.post(
- self.resource_paths.tables(identifier.database_name),
- request,
- self.rest_auth_function)
-
- def get_table(self, identifier: Identifier) -> GetTableResponse:
- return self.client.get(
- self.resource_paths.table(
- identifier.database_name,
- identifier.object_name),
- GetTableResponse,
- self.rest_auth_function,
- )
-
- def drop_table(self, identifier: Identifier) -> GetTableResponse:
- return self.client.delete(
- self.resource_paths.table(
- identifier.database_name,
- identifier.object_name),
- self.rest_auth_function,
- )
-
- def rename_table(self, source_identifier: Identifier, target_identifier:
Identifier) -> None:
- request = RenameTableRequest(source_identifier, target_identifier)
- return self.client.post(
- self.resource_paths.rename_table(),
- request,
- self.rest_auth_function)
-
- def load_table_token(self, identifier: Identifier) ->
GetTableTokenResponse:
- return self.client.get(
- self.resource_paths.table_token(
- identifier.database_name,
- identifier.object_name),
- GetTableTokenResponse,
- self.rest_auth_function,
- )
-
- def commit_snapshot(
- self,
- identifier: Identifier,
- table_uuid: Optional[str],
- snapshot: Snapshot,
- statistics: List[PartitionStatistics]
- ) -> bool:
- """
- Commit snapshot for table.
-
- Args:
- identifier: Database name and table name
- table_uuid: UUID of the table to avoid wrong commit
- snapshot: Snapshot for committing
- statistics: Statistics for this snapshot incremental
-
- Returns:
- True if commit success
-
- Raises:
- NoSuchResourceException: Exception thrown on HTTP 404 means the
table not exists
- ForbiddenException: Exception thrown on HTTP 403 means don't have
the permission for this table
- """
- request = CommitTableRequest(table_uuid, snapshot, statistics)
- response = self.client.post_with_response_type(
- self.resource_paths.commit_table(
- identifier.database_name, identifier.object_name),
- request,
- CommitTableResponse,
- self.rest_auth_function
- )
- return response.is_success()
diff --git a/paimon-python/pypaimon/api/api_request.py
b/paimon-python/pypaimon/api/api_request.py
index cf22d9853b..d453250757 100644
--- a/paimon-python/pypaimon/api/api_request.py
+++ b/paimon-python/pypaimon/api/api_request.py
@@ -20,15 +20,15 @@ from abc import ABC
from dataclasses import dataclass
from typing import Dict, List, Optional
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.common.identifier import Identifier
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
from pypaimon.schema.schema import Schema
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
class RESTRequest(ABC):
- pass
+ """RESTRequest"""
@dataclass
diff --git a/paimon-python/pypaimon/api/api_response.py
b/paimon-python/pypaimon/api/api_response.py
index 5658f5b351..0cc82f31b5 100644
--- a/paimon-python/pypaimon/api/api_response.py
+++ b/paimon-python/pypaimon/api/api_response.py
@@ -20,11 +20,9 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Dict, Generic, List, Optional
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import T, json_field
from pypaimon.schema.schema import Schema
-from .typedef import T
-
@dataclass
class PagedList(Generic[T]):
@@ -33,7 +31,7 @@ class PagedList(Generic[T]):
class RESTResponse(ABC):
- pass
+ """RESTResponse"""
@dataclass
@@ -99,11 +97,11 @@ class PagedResponse(RESTResponse, Generic[T]):
@abstractmethod
def data(self) -> List[T]:
- pass
+ """data"""
@abstractmethod
def get_next_page_token(self) -> str:
- pass
+ """get_next_page_token"""
@dataclass
diff --git a/paimon-python/pypaimon/api/auth.py
b/paimon-python/pypaimon/api/auth.py
index de20d96c8e..a0d62e2860 100644
--- a/paimon-python/pypaimon/api/auth.py
+++ b/paimon-python/pypaimon/api/auth.py
@@ -25,11 +25,11 @@ from collections import OrderedDict
from datetime import datetime, timezone
from typing import Dict, Optional
+from pypaimon.api.token_loader import (DLFToken, DLFTokenLoader,
+ DLFTokenLoaderFactory)
+from pypaimon.api.typedef import RESTAuthParameter
from pypaimon.common.config import CatalogOptions
-from .token_loader import DLFToken, DLFTokenLoader, DLFTokenLoaderFactory
-from .typedef import RESTAuthParameter
-
class AuthProvider(ABC):
diff --git a/paimon-python/pypaimon/api/client.py
b/paimon-python/pypaimon/api/client.py
index e7e42abe7a..9e30df598b 100644
--- a/paimon-python/pypaimon/api/client.py
+++ b/paimon-python/pypaimon/api/client.py
@@ -19,124 +19,39 @@ limitations under the License.
import json
import logging
import time
-import traceback
import urllib.parse
from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Optional, Type, TypeVar
+from typing import Callable, Dict, Optional, Type, TypeVar
import requests
from requests.adapters import HTTPAdapter
from urllib3 import Retry
-from pypaimon.common.rest_json import JSON
-
-from .api_response import ErrorResponse
-from .typedef import RESTAuthParameter
+from pypaimon.api.api_response import ErrorResponse
+from pypaimon.api.rest_exception import (AlreadyExistsException,
+ BadRequestException,
+ ForbiddenException,
+ NoSuchResourceException,
+ NotAuthorizedException,
+ NotImplementedException,
+ RESTException,
+ ServiceFailureException,
+ ServiceUnavailableException)
+from pypaimon.api.typedef import RESTAuthParameter
+from pypaimon.common.json_util import JSON
T = TypeVar('T', bound='RESTResponse')
class RESTRequest(ABC):
- pass
-
-
-class RESTException(Exception):
- def __init__(self, message: str = None, *args: Any, cause:
Optional[Exception] = None):
- if message and args:
- try:
- formatted_message = message % args
- except (TypeError, ValueError):
- formatted_message = f"{message} {' '.join(str(arg) for arg in
args)}"
- else:
- formatted_message = message or "REST API error occurred"
-
- super().__init__(formatted_message)
- self.__cause__ = cause
-
- def get_cause(self) -> Optional[Exception]:
- return self.__cause__
-
- def get_message(self) -> str:
- return str(self)
-
- def print_stack_trace(self) -> None:
- traceback.print_exception(type(self), self, self.__traceback__)
-
- def get_stack_trace(self) -> str:
- return ''.join(traceback.format_exception(type(self), self,
self.__traceback__))
-
- def __repr__(self) -> str:
- if self.__cause__:
- return f"{self.__class__.__name__}('{self}', caused by
{type(self.__cause__).__name__}: {self.__cause__})"
- return f"{self.__class__.__name__}('{self}')"
-
-
-class BadRequestException(RESTException):
-
- def __init__(self, message: str = None, *args: Any):
- super().__init__(message, *args)
-
-
-class NotAuthorizedException(RESTException):
- """Exception for not authorized (401)"""
-
- def __init__(self, message: str, *args: Any):
- super().__init__(message, *args)
-
-
-class ForbiddenException(RESTException):
- """Exception for forbidden access (403)"""
-
- def __init__(self, message: str, *args: Any):
- super().__init__(message, *args)
-
-
-class NoSuchResourceException(RESTException):
- """Exception for resource not found (404)"""
-
- def __init__(self, resource_type: Optional[str], resource_name:
Optional[str],
- message: str, *args: Any):
- self.resource_type = resource_type
- self.resource_name = resource_name
- super().__init__(message, *args)
-
-
-class AlreadyExistsException(RESTException):
- """Exception for resource already exists (409)"""
-
- def __init__(self, resource_type: Optional[str], resource_name:
Optional[str],
- message: str, *args: Any):
- self.resource_type = resource_type
- self.resource_name = resource_name
- super().__init__(message, *args)
-
-
-class ServiceFailureException(RESTException):
- """Exception for service failure (500)"""
-
- def __init__(self, message: str, *args: Any):
- super().__init__(message, *args)
-
-
-class NotImplementedException(RESTException):
- """Exception for not implemented (501)"""
-
- def __init__(self, message: str, *args: Any):
- super().__init__(message, *args)
-
-
-class ServiceUnavailableException(RESTException):
- """Exception for service unavailable (503)"""
-
- def __init__(self, message: str, *args: Any):
- super().__init__(message, *args)
+ """RESTRequest"""
class ErrorHandler(ABC):
@abstractmethod
def accept(self, error: ErrorResponse, request_id: str) -> None:
- pass
+ """accept"""
# DefaultErrorHandler implementation
diff --git a/paimon-python/pypaimon/api/resource_paths.py
b/paimon-python/pypaimon/api/resource_paths.py
new file mode 100644
index 0000000000..0214ab0529
--- /dev/null
+++ b/paimon-python/pypaimon/api/resource_paths.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Optional
+
+from pypaimon.api.rest_util import RESTUtil
+from pypaimon.common.config import CatalogOptions
+
+
+class ResourcePaths:
+ V1 = "v1"
+ DATABASES = "databases"
+ TABLES = "tables"
+ TABLE_DETAILS = "table-details"
+
+ def __init__(self, prefix: str):
+ self.base_path = f"/{self.V1}/{prefix}".rstrip("/")
+
+ @classmethod
+ def for_catalog_properties(
+ cls, options: dict[str, str]) -> "ResourcePaths":
+ prefix = options.get(CatalogOptions.PREFIX, "")
+ return cls(prefix)
+
+ @staticmethod
+ def config() -> str:
+ return f"/{ResourcePaths.V1}/config"
+
+ def databases(self) -> str:
+ return f"{self.base_path}/{self.DATABASES}"
+
+ def database(self, name: str) -> str:
+ return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(name)}"
+
+ def tables(self, database_name: Optional[str] = None) -> str:
+ if database_name:
+ return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}/{self.TABLES}"
+ return f"{self.base_path}/{self.TABLES}"
+
+ def table(self, database_name: str, table_name: str) -> str:
+ return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
+ f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}")
+
+ def table_details(self, database_name: str) -> str:
+ return
f"{self.base_path}/{self.DATABASES}/{database_name}/{self.TABLE_DETAILS}"
+
+ def table_token(self, database_name: str, table_name: str) -> str:
+ return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
+ f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/token")
+
+ def rename_table(self) -> str:
+ return f"{self.base_path}/{self.TABLES}/rename"
+
+ def commit_table(self, database_name: str, table_name: str) -> str:
+ return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
+ f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/commit")
diff --git a/paimon-python/pypaimon/api/__init__.py
b/paimon-python/pypaimon/api/rest_api.py
similarity index 77%
copy from paimon-python/pypaimon/api/__init__.py
copy to paimon-python/pypaimon/api/rest_api.py
index a384c3f123..2e341733bd 100644
--- a/paimon-python/pypaimon/api/__init__.py
+++ b/paimon-python/pypaimon/api/rest_api.py
@@ -17,115 +17,26 @@
import logging
from typing import Callable, Dict, List, Optional
-from urllib.parse import unquote
+from pypaimon.api.api_request import (AlterDatabaseRequest, CommitTableRequest,
+ CreateDatabaseRequest,
+ CreateTableRequest, RenameTableRequest)
from pypaimon.api.api_response import (CommitTableResponse, ConfigResponse,
GetDatabaseResponse, GetTableResponse,
GetTableTokenResponse,
ListDatabasesResponse,
ListTablesResponse, PagedList,
PagedResponse)
-from pypaimon.api.api_request import (AlterDatabaseRequest,
- CommitTableRequest,
- CreateDatabaseRequest,
- CreateTableRequest, RenameTableRequest)
from pypaimon.api.auth import AuthProviderFactory, RESTAuthFunction
from pypaimon.api.client import HttpClient
+from pypaimon.api.resource_paths import ResourcePaths
+from pypaimon.api.rest_util import RESTUtil
from pypaimon.api.typedef import T
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.common.config import CatalogOptions
from pypaimon.common.identifier import Identifier
from pypaimon.schema.schema import Schema
from pypaimon.snapshot.snapshot import Snapshot
-
-
-class RESTException(Exception):
- pass
-
-
-class AlreadyExistsException(RESTException):
- pass
-
-
-class ForbiddenException(RESTException):
- pass
-
-
-class NoSuchResourceException(RESTException):
- pass
-
-
-class RESTUtil:
- @staticmethod
- def encode_string(value: str) -> str:
- import urllib.parse
-
- return urllib.parse.quote(value)
-
- @staticmethod
- def decode_string(encoded: str) -> str:
- """Decode URL-encoded string"""
- return unquote(encoded)
-
- @staticmethod
- def extract_prefix_map(
- options: Dict[str, str], prefix: str) -> Dict[str, str]:
- result = {}
- config = options
- for key, value in config.items():
- if key.startswith(prefix):
- new_key = key[len(prefix):]
- result[new_key] = str(value)
- return result
-
-
-class ResourcePaths:
- V1 = "v1"
- DATABASES = "databases"
- TABLES = "tables"
- TABLE_DETAILS = "table-details"
-
- def __init__(self, prefix: str):
- self.base_path = f"/{self.V1}/{prefix}".rstrip("/")
-
- @classmethod
- def for_catalog_properties(
- cls, options: dict[str, str]) -> "ResourcePaths":
- prefix = options.get(CatalogOptions.PREFIX, "")
- return cls(prefix)
-
- @staticmethod
- def config() -> str:
- return f"/{ResourcePaths.V1}/config"
-
- def databases(self) -> str:
- return f"{self.base_path}/{self.DATABASES}"
-
- def database(self, name: str) -> str:
- return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(name)}"
-
- def tables(self, database_name: Optional[str] = None) -> str:
- if database_name:
- return
f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}/{self.TABLES}"
- return f"{self.base_path}/{self.TABLES}"
-
- def table(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}")
-
- def table_details(self, database_name: str) -> str:
- return
f"{self.base_path}/{self.DATABASES}/{database_name}/{self.TABLE_DETAILS}"
-
- def table_token(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/token")
-
- def rename_table(self) -> str:
- return f"{self.base_path}/{self.TABLES}/rename"
-
- def commit_table(self, database_name: str, table_name: str) -> str:
- return
(f"{self.base_path}/{self.DATABASES}/{RESTUtil.encode_string(database_name)}"
- f"/{self.TABLES}/{RESTUtil.encode_string(table_name)}/commit")
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
class RESTApi:
diff --git a/paimon-python/pypaimon/api/rest_exception.py
b/paimon-python/pypaimon/api/rest_exception.py
new file mode 100644
index 0000000000..738de5efdd
--- /dev/null
+++ b/paimon-python/pypaimon/api/rest_exception.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import traceback
+from typing import Any, Optional
+
+
+class RESTException(Exception):
+ def __init__(self, message: str = None, *args: Any, cause:
Optional[Exception] = None):
+ if message and args:
+ try:
+ formatted_message = message % args
+ except (TypeError, ValueError):
+ formatted_message = f"{message} {' '.join(str(arg) for arg in
args)}"
+ else:
+ formatted_message = message or "REST API error occurred"
+
+ super().__init__(formatted_message)
+ self.__cause__ = cause
+
+ def get_cause(self) -> Optional[Exception]:
+ return self.__cause__
+
+ def get_message(self) -> str:
+ return str(self)
+
+ def print_stack_trace(self) -> None:
+ traceback.print_exception(type(self), self, self.__traceback__)
+
+ def get_stack_trace(self) -> str:
+ return ''.join(traceback.format_exception(type(self), self,
self.__traceback__))
+
+ def __repr__(self) -> str:
+ if self.__cause__:
+ return f"{self.__class__.__name__}('{self}', caused by
{type(self.__cause__).__name__}: {self.__cause__})"
+ return f"{self.__class__.__name__}('{self}')"
+
+
+class BadRequestException(RESTException):
+
+ def __init__(self, message: str = None, *args: Any):
+ super().__init__(message, *args)
+
+
+class NotAuthorizedException(RESTException):
+ """Exception for not authorized (401)"""
+
+ def __init__(self, message: str, *args: Any):
+ super().__init__(message, *args)
+
+
+class ForbiddenException(RESTException):
+ """Exception for forbidden access (403)"""
+
+ def __init__(self, message: str, *args: Any):
+ super().__init__(message, *args)
+
+
+class NoSuchResourceException(RESTException):
+ """Exception for resource not found (404)"""
+
+ def __init__(self, resource_type: Optional[str], resource_name:
Optional[str],
+ message: str, *args: Any):
+ self.resource_type = resource_type
+ self.resource_name = resource_name
+ super().__init__(message, *args)
+
+
+class AlreadyExistsException(RESTException):
+ """Exception for resource already exists (409)"""
+
+ def __init__(self, resource_type: Optional[str], resource_name:
Optional[str],
+ message: str, *args: Any):
+ self.resource_type = resource_type
+ self.resource_name = resource_name
+ super().__init__(message, *args)
+
+
+class ServiceFailureException(RESTException):
+ """Exception for service failure (500)"""
+
+ def __init__(self, message: str, *args: Any):
+ super().__init__(message, *args)
+
+
+class NotImplementedException(RESTException):
+ """Exception for not implemented (501)"""
+
+ def __init__(self, message: str, *args: Any):
+ super().__init__(message, *args)
+
+
+class ServiceUnavailableException(RESTException):
+ """Exception for service unavailable (503)"""
+
+ def __init__(self, message: str, *args: Any):
+ super().__init__(message, *args)
diff --git a/paimon-python/pypaimon/api/rest_util.py
b/paimon-python/pypaimon/api/rest_util.py
new file mode 100644
index 0000000000..6c2dd3c644
--- /dev/null
+++ b/paimon-python/pypaimon/api/rest_util.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Dict
+from urllib.parse import unquote
+
+
+class RESTUtil:
+ @staticmethod
+ def encode_string(value: str) -> str:
+ import urllib.parse
+
+ return urllib.parse.quote(value)
+
+ @staticmethod
+ def decode_string(encoded: str) -> str:
+ """Decode URL-encoded string"""
+ return unquote(encoded)
+
+ @staticmethod
+ def extract_prefix_map(
+ options: Dict[str, str], prefix: str) -> Dict[str, str]:
+ result = {}
+ config = options
+ for key, value in config.items():
+ if key.startswith(prefix):
+ new_key = key[len(prefix):]
+ result[new_key] = str(value)
+ return result
diff --git a/paimon-python/pypaimon/api/token_loader.py
b/paimon-python/pypaimon/api/token_loader.py
index 0d332c6c1f..8e65846bf8 100644
--- a/paimon-python/pypaimon/api/token_loader.py
+++ b/paimon-python/pypaimon/api/token_loader.py
@@ -25,10 +25,9 @@ import requests
from requests.adapters import HTTPAdapter
from requests.exceptions import RequestException
+from pypaimon.api.client import ExponentialRetry
from pypaimon.common.config import CatalogOptions
-from pypaimon.common.rest_json import JSON, json_field
-
-from .client import ExponentialRetry
+from pypaimon.common.json_util import JSON, json_field
@dataclass
diff --git a/paimon-python/pypaimon/catalog/catalog.py
b/paimon-python/pypaimon/catalog/catalog.py
index c2b1f6915e..a8e7dcd065 100644
--- a/paimon-python/pypaimon/catalog/catalog.py
+++ b/paimon-python/pypaimon/catalog/catalog.py
@@ -19,10 +19,10 @@
from abc import ABC, abstractmethod
from typing import List, Optional, Union
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.common.identifier import Identifier
from pypaimon.schema.schema import Schema
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
class Catalog(ABC):
diff --git a/paimon-python/pypaimon/table/catalog_environment.py
b/paimon-python/pypaimon/catalog/catalog_environment.py
similarity index 93%
rename from paimon-python/pypaimon/table/catalog_environment.py
rename to paimon-python/pypaimon/catalog/catalog_environment.py
index f025b367b3..762a42dd6a 100644
--- a/paimon-python/pypaimon/table/catalog_environment.py
+++ b/paimon-python/pypaimon/catalog/catalog_environment.py
@@ -15,13 +15,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
from typing import Optional
from pypaimon.catalog.catalog_loader import CatalogLoader
-from pypaimon.catalog.catalog_snapshot_commit import CatalogSnapshotCommit
-from pypaimon.catalog.renaming_snapshot_commit import RenamingSnapshotCommit
-from pypaimon.catalog.snapshot_commit import SnapshotCommit
from pypaimon.common.identifier import Identifier
+from pypaimon.snapshot.catalog_snapshot_commit import CatalogSnapshotCommit
+from pypaimon.snapshot.renaming_snapshot_commit import RenamingSnapshotCommit
+from pypaimon.snapshot.snapshot_commit import SnapshotCommit
class CatalogEnvironment:
diff --git a/paimon-python/pypaimon/catalog/catalog_factory.py
b/paimon-python/pypaimon/catalog/catalog_factory.py
index 7bae23be7e..865ffe4766 100644
--- a/paimon-python/pypaimon/catalog/catalog_factory.py
+++ b/paimon-python/pypaimon/catalog/catalog_factory.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
from pypaimon.api.options import Options
from pypaimon.catalog.catalog import Catalog
from pypaimon.catalog.catalog_context import CatalogContext
diff --git a/paimon-python/pypaimon/catalog/filesystem_catalog.py
b/paimon-python/pypaimon/catalog/filesystem_catalog.py
index 1aed6e8137..8de2850e4a 100644
--- a/paimon-python/pypaimon/catalog/filesystem_catalog.py
+++ b/paimon-python/pypaimon/catalog/filesystem_catalog.py
@@ -21,19 +21,19 @@ from typing import List, Optional, Union
from urllib.parse import urlparse
from pypaimon.catalog.catalog import Catalog
+from pypaimon.catalog.catalog_environment import CatalogEnvironment
from pypaimon.catalog.catalog_exception import (DatabaseAlreadyExistException,
DatabaseNotExistException,
TableAlreadyExistException,
TableNotExistException)
from pypaimon.catalog.database import Database
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.common.config import CatalogOptions
from pypaimon.common.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
from pypaimon.common.identifier import Identifier
from pypaimon.schema.schema_manager import SchemaManager
from pypaimon.snapshot.snapshot import Snapshot
-from pypaimon.table.catalog_environment import CatalogEnvironment
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
from pypaimon.table.file_store_table import FileStoreTable
from pypaimon.table.table import Table
diff --git a/paimon-python/pypaimon/catalog/property_change.py
b/paimon-python/pypaimon/catalog/rest/property_change.py
similarity index 100%
rename from paimon-python/pypaimon/catalog/property_change.py
rename to paimon-python/pypaimon/catalog/rest/property_change.py
diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
index f35ee04322..31cafb2578 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
@@ -19,35 +19,36 @@ from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from urllib.parse import urlparse
-from pypaimon.api import (CatalogOptions,
- NoSuchResourceException, RESTApi)
from pypaimon.api.api_response import GetTableResponse, PagedList
from pypaimon.api.options import Options
+from pypaimon.api.rest_api import RESTApi
+from pypaimon.api.rest_exception import NoSuchResourceException
from pypaimon.catalog.catalog import Catalog
from pypaimon.catalog.catalog_context import CatalogContext
+from pypaimon.catalog.catalog_environment import CatalogEnvironment
from pypaimon.catalog.catalog_exception import TableNotExistException
from pypaimon.catalog.database import Database
-from pypaimon.catalog.property_change import PropertyChange
+from pypaimon.catalog.rest.property_change import PropertyChange
from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
-from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.catalog.rest.table_metadata import TableMetadata
+from pypaimon.common.config import CatalogOptions
from pypaimon.common.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
from pypaimon.common.identifier import Identifier
from pypaimon.schema.schema import Schema
from pypaimon.schema.table_schema import TableSchema
from pypaimon.snapshot.snapshot import Snapshot
-from pypaimon.table.catalog_environment import CatalogEnvironment
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
from pypaimon.table.file_store_table import FileStoreTable
class RESTCatalog(Catalog):
def __init__(self, context: CatalogContext, config_required:
Optional[bool] = True):
self.warehouse = context.options.get(CatalogOptions.WAREHOUSE)
- self.api = RESTApi(context.options.to_map(), config_required)
- self.context = CatalogContext.create(Options(self.api.options),
context.hadoop_conf, context.prefer_io_loader,
- context.fallback_io_loader)
- self.data_token_enabled =
self.api.options.get(CatalogOptions.DATA_TOKEN_ENABLED)
+ self.rest_api = RESTApi(context.options.to_map(), config_required)
+ self.context = CatalogContext.create(Options(self.rest_api.options),
context.hadoop_conf,
+ context.prefer_io_loader,
context.fallback_io_loader)
+ self.data_token_enabled =
self.rest_api.options.get(CatalogOptions.DATA_TOKEN_ENABLED)
def catalog_loader(self):
"""
@@ -91,7 +92,7 @@ class RESTCatalog(Catalog):
TableNotExistException: If the target table does not exist
"""
try:
- return self.api.commit_snapshot(identifier, table_uuid, snapshot,
statistics)
+ return self.rest_api.commit_snapshot(identifier, table_uuid,
snapshot, statistics)
except NoSuchResourceException as e:
raise TableNotExistException(identifier) from e
except Exception as e:
@@ -99,17 +100,17 @@ class RESTCatalog(Catalog):
raise RuntimeError(f"Failed to commit snapshot for table
{identifier.get_full_name()}: {e}") from e
def list_databases(self) -> List[str]:
- return self.api.list_databases()
+ return self.rest_api.list_databases()
def list_databases_paged(self, max_results: Optional[int] = None,
page_token: Optional[str] = None,
database_name_pattern: Optional[str] = None) ->
PagedList[str]:
- return self.api.list_databases_paged(max_results, page_token,
database_name_pattern)
+ return self.rest_api.list_databases_paged(max_results, page_token,
database_name_pattern)
def create_database(self, name: str, ignore_if_exists: bool, properties:
Dict[str, str] = None):
- self.api.create_database(name, properties)
+ self.rest_api.create_database(name, properties)
def get_database(self, name: str) -> Database:
- response = self.api.get_database(name)
+ response = self.rest_api.get_database(name)
options = response.options
options[Catalog.DB_LOCATION_PROP] = response.location
response.put_audit_options_to(options)
@@ -117,14 +118,14 @@ class RESTCatalog(Catalog):
return Database(name, options)
def drop_database(self, name: str):
- self.api.drop_database(name)
+ self.rest_api.drop_database(name)
def alter_database(self, name: str, changes: List[PropertyChange]):
set_properties, remove_keys =
PropertyChange.get_set_properties_to_remove_keys(changes)
- self.api.alter_database(name, list(remove_keys), set_properties)
+ self.rest_api.alter_database(name, list(remove_keys), set_properties)
def list_tables(self, database_name: str) -> List[str]:
- return self.api.list_tables(database_name)
+ return self.rest_api.list_tables(database_name)
def list_tables_paged(
self,
@@ -133,7 +134,7 @@ class RESTCatalog(Catalog):
page_token: Optional[str] = None,
table_name_pattern: Optional[str] = None
) -> PagedList[str]:
- return self.api.list_tables_paged(
+ return self.rest_api.list_tables_paged(
database_name,
max_results,
page_token,
@@ -153,15 +154,15 @@ class RESTCatalog(Catalog):
def create_table(self, identifier: Union[str, Identifier], schema: Schema,
ignore_if_exists: bool):
if not isinstance(identifier, Identifier):
identifier = Identifier.from_string(identifier)
- self.api.create_table(identifier, schema)
+ self.rest_api.create_table(identifier, schema)
def drop_table(self, identifier: Union[str, Identifier]):
if not isinstance(identifier, Identifier):
identifier = Identifier.from_string(identifier)
- self.api.drop_table(identifier)
+ self.rest_api.drop_table(identifier)
def load_table_metadata(self, identifier: Identifier) -> TableMetadata:
- response = self.api.get_table(identifier)
+ response = self.rest_api.get_table(identifier)
return self.to_table_metadata(identifier.get_database_name(), response)
def to_table_metadata(self, db: str, response: GetTableResponse) ->
TableMetadata:
diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
index 6142fa6373..b9671c8ae9 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
@@ -23,7 +23,7 @@ from typing import Optional
from pyarrow._fs import FileSystem
-from pypaimon.api import RESTApi
+from pypaimon.api.rest_api import RESTApi
from pypaimon.catalog.rest.rest_token import RESTToken
from pypaimon.common.file_io import FileIO
from pypaimon.common.identifier import Identifier
diff --git a/paimon-python/pypaimon/catalog/table_metadata.py
b/paimon-python/pypaimon/catalog/rest/table_metadata.py
similarity index 100%
rename from paimon-python/pypaimon/catalog/table_metadata.py
rename to paimon-python/pypaimon/catalog/rest/table_metadata.py
diff --git a/paimon-python/pypaimon/common/identifier.py
b/paimon-python/pypaimon/common/identifier.py
index 3a27870516..d3a4fcda7f 100644
--- a/paimon-python/pypaimon/common/identifier.py
+++ b/paimon-python/pypaimon/common/identifier.py
@@ -18,7 +18,7 @@
from dataclasses import dataclass
from typing import Optional
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
SYSTEM_TABLE_SPLITTER = '$'
SYSTEM_BRANCH_PREFIX = 'branch-'
diff --git a/paimon-python/pypaimon/common/rest_json.py
b/paimon-python/pypaimon/common/json_util.py
similarity index 100%
rename from paimon-python/pypaimon/common/rest_json.py
rename to paimon-python/pypaimon/common/json_util.py
diff --git a/paimon-python/pypaimon/pvfs/__init__.py
b/paimon-python/pypaimon/pvfs/__init__.py
index 78b3b73691..7e60a558d1 100644
--- a/paimon-python/pypaimon/pvfs/__init__.py
+++ b/paimon-python/pypaimon/pvfs/__init__.py
@@ -29,10 +29,12 @@ from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
from readerwriterlock import rwlock
-from pypaimon.api import (GetTableResponse, GetTableTokenResponse, Identifier,
- RESTApi, Schema)
+from pypaimon.api.api_response import GetTableTokenResponse, GetTableResponse
from pypaimon.api.client import AlreadyExistsException, NoSuchResourceException
+from pypaimon.api.rest_api import RESTApi
from pypaimon.common.config import CatalogOptions, OssOptions, PVFSOptions
+from pypaimon.common.identifier import Identifier
+from pypaimon.schema.schema import Schema
PROTOCOL_NAME = "pvfs"
diff --git a/paimon-python/pypaimon/schema/schema.py
b/paimon-python/pypaimon/schema/schema.py
index 20e0720087..965fe2255b 100644
--- a/paimon-python/pypaimon/schema/schema.py
+++ b/paimon-python/pypaimon/schema/schema.py
@@ -20,7 +20,7 @@ from typing import Dict, List, Optional
import pyarrow as pa
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
from pypaimon.schema.data_types import DataField, PyarrowFieldParser
diff --git a/paimon-python/pypaimon/schema/schema_manager.py
b/paimon-python/pypaimon/schema/schema_manager.py
index f03b9e111b..2af8481b49 100644
--- a/paimon-python/pypaimon/schema/schema_manager.py
+++ b/paimon-python/pypaimon/schema/schema_manager.py
@@ -19,7 +19,7 @@ from pathlib import Path
from typing import Optional
from pypaimon.common.file_io import FileIO
-from pypaimon.common.rest_json import JSON
+from pypaimon.common.json_util import JSON
from pypaimon.schema.schema import Schema
from pypaimon.schema.table_schema import TableSchema
diff --git a/paimon-python/pypaimon/schema/table_schema.py
b/paimon-python/pypaimon/schema/table_schema.py
index dc1872deb6..852ef8c4ea 100644
--- a/paimon-python/pypaimon/schema/table_schema.py
+++ b/paimon-python/pypaimon/schema/table_schema.py
@@ -24,7 +24,7 @@ from typing import Dict, List, Optional
from pypaimon.common.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
from pypaimon.schema.data_types import DataField
from pypaimon.schema.schema import Schema
diff --git a/paimon-python/pypaimon/catalog/catalog_snapshot_commit.py
b/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
similarity index 95%
rename from paimon-python/pypaimon/catalog/catalog_snapshot_commit.py
rename to paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
index cc1ed258f8..26796f7766 100644
--- a/paimon-python/pypaimon/catalog/catalog_snapshot_commit.py
+++ b/paimon-python/pypaimon/snapshot/catalog_snapshot_commit.py
@@ -19,9 +19,10 @@
from typing import List
from pypaimon.catalog.catalog import Catalog
-from pypaimon.catalog.snapshot_commit import PartitionStatistics,
SnapshotCommit
from pypaimon.common.identifier import Identifier
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import (PartitionStatistics,
+ SnapshotCommit)
class CatalogSnapshotCommit(SnapshotCommit):
diff --git a/paimon-python/pypaimon/catalog/renaming_snapshot_commit.py
b/paimon-python/pypaimon/snapshot/renaming_snapshot_commit.py
similarity index 95%
rename from paimon-python/pypaimon/catalog/renaming_snapshot_commit.py
rename to paimon-python/pypaimon/snapshot/renaming_snapshot_commit.py
index b85c4200a1..27c5a54a2c 100644
--- a/paimon-python/pypaimon/catalog/renaming_snapshot_commit.py
+++ b/paimon-python/pypaimon/snapshot/renaming_snapshot_commit.py
@@ -18,10 +18,11 @@
from typing import List
-from pypaimon.catalog.snapshot_commit import PartitionStatistics,
SnapshotCommit
from pypaimon.common.file_io import FileIO
-from pypaimon.common.rest_json import JSON
+from pypaimon.common.json_util import JSON
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import (PartitionStatistics,
+ SnapshotCommit)
from pypaimon.snapshot.snapshot_manager import SnapshotManager
diff --git a/paimon-python/pypaimon/snapshot/snapshot.py
b/paimon-python/pypaimon/snapshot/snapshot.py
index cc27c5a530..5bc92dcad4 100644
--- a/paimon-python/pypaimon/snapshot/snapshot.py
+++ b/paimon-python/pypaimon/snapshot/snapshot.py
@@ -19,7 +19,7 @@
from dataclasses import dataclass
from typing import Dict, Optional
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
@dataclass
diff --git a/paimon-python/pypaimon/catalog/snapshot_commit.py
b/paimon-python/pypaimon/snapshot/snapshot_commit.py
similarity index 98%
rename from paimon-python/pypaimon/catalog/snapshot_commit.py
rename to paimon-python/pypaimon/snapshot/snapshot_commit.py
index f66b123343..50727b6ce3 100644
--- a/paimon-python/pypaimon/catalog/snapshot_commit.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_commit.py
@@ -16,12 +16,12 @@
# limitations under the License.
################################################################################
+import time
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Dict, List
-import time
-from pypaimon.common.rest_json import json_field
+from pypaimon.common.json_util import json_field
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/snapshot/snapshot_manager.py
b/paimon-python/pypaimon/snapshot/snapshot_manager.py
index f23ad03dae..2ded357802 100644
--- a/paimon-python/pypaimon/snapshot/snapshot_manager.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_manager.py
@@ -19,7 +19,7 @@ from pathlib import Path
from typing import Optional
from pypaimon.common.file_io import FileIO
-from pypaimon.common.rest_json import JSON
+from pypaimon.common.json_util import JSON
from pypaimon.snapshot.snapshot import Snapshot
diff --git a/paimon-python/pypaimon/table/file_store_table.py
b/paimon-python/pypaimon/table/file_store_table.py
index eb763d2938..74aaaff4e6 100644
--- a/paimon-python/pypaimon/table/file_store_table.py
+++ b/paimon-python/pypaimon/table/file_store_table.py
@@ -19,6 +19,7 @@
from pathlib import Path
from typing import Optional
+from pypaimon.catalog.catalog_environment import CatalogEnvironment
from pypaimon.common.core_options import CoreOptions
from pypaimon.common.file_io import FileIO
from pypaimon.common.identifier import Identifier
@@ -26,7 +27,6 @@ from pypaimon.read.read_builder import ReadBuilder
from pypaimon.schema.schema_manager import SchemaManager
from pypaimon.schema.table_schema import TableSchema
from pypaimon.table.bucket_mode import BucketMode
-from pypaimon.table.catalog_environment import CatalogEnvironment
from pypaimon.table.table import Table
from pypaimon.write.batch_write_builder import BatchWriteBuilder
from pypaimon.write.row_key_extractor import (DynamicBucketRowKeyExtractor,
diff --git a/paimon-python/pypaimon/tests/api_test.py
b/paimon-python/pypaimon/tests/api_test.py
index 6de26917eb..c63b912635 100644
--- a/paimon-python/pypaimon/tests/api_test.py
+++ b/paimon-python/pypaimon/tests/api_test.py
@@ -19,20 +19,19 @@ import logging
import unittest
import uuid
-import pypaimon.api as api
+from pypaimon.api.api_response import ConfigResponse
+from pypaimon.api.auth import BearTokenAuthProvider
+from pypaimon.api.rest_api import RESTApi
+from pypaimon.api.token_loader import DLFToken, DLFTokenLoaderFactory
+from pypaimon.catalog.rest.table_metadata import TableMetadata
+from pypaimon.common.config import CatalogOptions
from pypaimon.common.identifier import Identifier
-from pypaimon.common.rest_json import JSON
+from pypaimon.common.json_util import JSON
from pypaimon.schema.data_types import (ArrayType, AtomicInteger, AtomicType,
DataField, DataTypeParser, MapType,
RowType)
from pypaimon.schema.table_schema import TableSchema
-
-from ..api import RESTApi
-from ..api.api_response import ConfigResponse
-from ..api.auth import BearTokenAuthProvider
-from ..api.token_loader import DLFToken, DLFTokenLoaderFactory
-from ..catalog.table_metadata import TableMetadata
-from .rest_server import RESTCatalogServer
+from pypaimon.tests.rest_server import RESTCatalogServer
class ApiTestCase(unittest.TestCase):
@@ -170,10 +169,10 @@ class ApiTestCase(unittest.TestCase):
"token.provider": "bear",
'token': token
}
- api = RESTApi(options)
- self.assertSetEqual(set(api.list_databases()), {*test_databases})
- self.assertEqual(api.get_database('default'),
test_databases.get('default'))
- table = api.get_table(Identifier.from_string('default.user'))
+ rest_api = RESTApi(options)
+ self.assertSetEqual(set(rest_api.list_databases()),
{*test_databases})
+ self.assertEqual(rest_api.get_database('default'),
test_databases.get('default'))
+ table = rest_api.get_table(Identifier.from_string('default.user'))
self.assertEqual(table.id, str(test_tables['default.user'].uuid))
finally:
@@ -204,8 +203,8 @@ class ApiTestCase(unittest.TestCase):
server.start()
ecs_metadata_url =
f"http://localhost:{server.port}/ram/security-credential/";
options = {
- api.CatalogOptions.DLF_TOKEN_LOADER: 'ecs',
- api.CatalogOptions.DLF_TOKEN_ECS_METADATA_URL: ecs_metadata_url
+ CatalogOptions.DLF_TOKEN_LOADER: 'ecs',
+ CatalogOptions.DLF_TOKEN_ECS_METADATA_URL: ecs_metadata_url
}
loader = DLFTokenLoaderFactory.create_token_loader(options)
load_token = loader.load_token()
@@ -214,9 +213,9 @@ class ApiTestCase(unittest.TestCase):
self.assertEqual(load_token.security_token, token.security_token)
self.assertEqual(load_token.expiration, token.expiration)
options_with_role = {
- api.CatalogOptions.DLF_TOKEN_LOADER: 'ecs',
- api.CatalogOptions.DLF_TOKEN_ECS_METADATA_URL:
ecs_metadata_url,
- api.CatalogOptions.DLF_TOKEN_ECS_ROLE_NAME: role_name,
+ CatalogOptions.DLF_TOKEN_LOADER: 'ecs',
+ CatalogOptions.DLF_TOKEN_ECS_METADATA_URL: ecs_metadata_url,
+ CatalogOptions.DLF_TOKEN_ECS_ROLE_NAME: role_name,
}
loader =
DLFTokenLoaderFactory.create_token_loader(options_with_role)
token = loader.load_token()
diff --git a/paimon-python/pypaimon/tests/predicates_test.py
b/paimon-python/pypaimon/tests/predicates_test.py
index 5ce806bbcb..19d101fb28 100644
--- a/paimon-python/pypaimon/tests/predicates_test.py
+++ b/paimon-python/pypaimon/tests/predicates_test.py
@@ -23,8 +23,8 @@ import unittest
import pandas as pd
import pyarrow as pa
-from pypaimon.api import Schema
from pypaimon.catalog.catalog_factory import CatalogFactory
+from pypaimon.schema.schema import Schema
def _check_filtered_result(read_builder, expected_df):
diff --git a/paimon-python/pypaimon/tests/pvfs_test.py
b/paimon-python/pypaimon/tests/pvfs_test.py
index 3f80d1eb63..8bebb9855d 100644
--- a/paimon-python/pypaimon/tests/pvfs_test.py
+++ b/paimon-python/pypaimon/tests/pvfs_test.py
@@ -23,9 +23,9 @@ from pathlib import Path
import pandas
-from pypaimon.api import ConfigResponse
+from pypaimon.api.api_response import ConfigResponse
from pypaimon.api.auth import BearTokenAuthProvider
-from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.catalog.rest.table_metadata import TableMetadata
from pypaimon.pvfs import PaimonVirtualFileSystem
from pypaimon.schema.data_types import AtomicType, DataField
from pypaimon.schema.table_schema import TableSchema
diff --git a/paimon-python/pypaimon/tests/reader_append_only_test.py
b/paimon-python/pypaimon/tests/reader_append_only_test.py
index f82f28d7a0..17acb9a183 100644
--- a/paimon-python/pypaimon/tests/reader_append_only_test.py
+++ b/paimon-python/pypaimon/tests/reader_append_only_test.py
@@ -22,8 +22,8 @@ import unittest
import pyarrow as pa
-from pypaimon.api import Schema
from pypaimon.catalog.catalog_factory import CatalogFactory
+from pypaimon.schema.schema import Schema
class AoReaderTest(unittest.TestCase):
diff --git a/paimon-python/pypaimon/tests/reader_primary_key_test.py
b/paimon-python/pypaimon/tests/reader_primary_key_test.py
index b9b115dfa4..8b9b853350 100644
--- a/paimon-python/pypaimon/tests/reader_primary_key_test.py
+++ b/paimon-python/pypaimon/tests/reader_primary_key_test.py
@@ -23,8 +23,8 @@ import unittest
import pyarrow as pa
-from pypaimon.api import Schema
from pypaimon.catalog.catalog_factory import CatalogFactory
+from pypaimon.schema.schema import Schema
class PkReaderTest(unittest.TestCase):
diff --git a/paimon-python/pypaimon/tests/rest_catalog_base_test.py
b/paimon-python/pypaimon/tests/rest_catalog_base_test.py
index c035957ccc..e56580e0ae 100644
--- a/paimon-python/pypaimon/tests/rest_catalog_base_test.py
+++ b/paimon-python/pypaimon/tests/rest_catalog_base_test.py
@@ -25,13 +25,14 @@ import uuid
import pyarrow as pa
-from pypaimon.api import ConfigResponse, Identifier
+from pypaimon.api.api_response import ConfigResponse
from pypaimon.api.auth import BearTokenAuthProvider
from pypaimon.api.options import Options
from pypaimon.catalog.catalog_context import CatalogContext
from pypaimon.catalog.catalog_factory import CatalogFactory
from pypaimon.catalog.rest.rest_catalog import RESTCatalog
-from pypaimon.catalog.table_metadata import TableMetadata
+from pypaimon.catalog.rest.table_metadata import TableMetadata
+from pypaimon.common.identifier import Identifier
from pypaimon.schema.data_types import (ArrayType, AtomicType, DataField,
MapType)
from pypaimon.schema.schema import Schema
diff --git a/paimon-python/pypaimon/tests/rest_server.py
b/paimon-python/pypaimon/tests/rest_server.py
index c326f3525d..8804a4f5e2 100644
--- a/paimon-python/pypaimon/tests/rest_server.py
+++ b/paimon-python/pypaimon/tests/rest_server.py
@@ -27,22 +27,24 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from urllib.parse import urlparse
-import pypaimon.api as api
-from pypaimon.common.rest_json import JSON
+from pypaimon.api.api_request import (CreateDatabaseRequest,
+ CreateTableRequest, RenameTableRequest)
+from pypaimon.api.api_response import (ConfigResponse, GetDatabaseResponse,
+ GetTableResponse, ListDatabasesResponse,
+ ListTablesResponse, PagedList,
+ RESTResponse)
+from pypaimon.api.resource_paths import ResourcePaths
+from pypaimon.api.rest_util import RESTUtil
+from pypaimon.catalog.catalog_exception import (DatabaseNoPermissionException,
+ DatabaseNotExistException,
+ TableNoPermissionException,
+ TableNotExistException)
+from pypaimon.catalog.rest.table_metadata import TableMetadata
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.json_util import JSON
+from pypaimon.schema.schema import Schema
from pypaimon.schema.table_schema import TableSchema
-from ..api import (CreateDatabaseRequest, CreateTableRequest, Identifier,
- RenameTableRequest)
-from ..api.api_response import (ConfigResponse, GetDatabaseResponse,
- GetTableResponse, ListDatabasesResponse,
- ListTablesResponse, PagedList, RESTResponse)
-from ..catalog.catalog_exception import (DatabaseNoPermissionException,
- DatabaseNotExistException,
- TableNoPermissionException,
- TableNotExistException)
-from ..catalog.table_metadata import TableMetadata
-from ..schema.schema import Schema
-
@dataclass
class ErrorResponse(RESTResponse):
@@ -99,7 +101,7 @@ class RESTCatalogServer:
# Initialize resource paths
prefix = config.defaults.get("prefix")
- self.resource_paths = api.ResourcePaths(prefix=prefix)
+ self.resource_paths = ResourcePaths(prefix=prefix)
self.database_uri = self.resource_paths.databases()
# Initialize storage
@@ -199,7 +201,7 @@ class RESTCatalogServer:
for pair in query.split('&'):
if '=' in pair:
key, value = pair.split('=', 1)
- params[key.strip()] =
api.RESTUtil.decode_string(value.strip())
+ params[key.strip()] =
RESTUtil.decode_string(value.strip())
return params
def _authenticate(self, token: str, path: str, params: Dict[str,
str],
@@ -265,7 +267,7 @@ class RESTCatalogServer:
"""Handle database-specific resource requests"""
# Extract database name and resource path
path_parts = resource_path[len(self.database_uri) +
1:].split('/')
- database_name = api.RESTUtil.decode_string(path_parts[0])
+ database_name = RESTUtil.decode_string(path_parts[0])
# Check database permissions
if database_name in self.no_permission_databases:
@@ -283,16 +285,16 @@ class RESTCatalogServer:
# Collection operations (tables, views, functions)
resource_type = path_parts[1]
- if resource_type.startswith(api.ResourcePaths.TABLES):
+ if resource_type.startswith(ResourcePaths.TABLES):
return self._tables_handle(method, data,
database_name, parameters)
elif len(path_parts) >= 3:
# Individual resource operations
resource_type = path_parts[1]
- resource_name = api.RESTUtil.decode_string(path_parts[2])
+ resource_name = RESTUtil.decode_string(path_parts[2])
identifier = Identifier.create(database_name,
resource_name)
- if resource_type == api.ResourcePaths.TABLES:
+ if resource_type == ResourcePaths.TABLES:
return self._handle_table_resource(method, path_parts,
identifier, data, parameters)
return self._mock_response(ErrorResponse(None, None, "Not
Found", 404), 404)
diff --git a/paimon-python/pypaimon/tests/test_file_store_commit.py
b/paimon-python/pypaimon/tests/test_file_store_commit.py
index ced0afbf4a..6f32894c90 100644
--- a/paimon-python/pypaimon/tests/test_file_store_commit.py
+++ b/paimon-python/pypaimon/tests/test_file_store_commit.py
@@ -21,8 +21,8 @@ from datetime import datetime
from pathlib import Path
from unittest.mock import Mock, patch
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.manifest.schema.data_file_meta import DataFileMeta
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
from pypaimon.write.commit_message import CommitMessage
from pypaimon.write.file_store_commit import FileStoreCommit
diff --git a/paimon-python/pypaimon/tests/test_rest_catalog_commit_snapshot.py
b/paimon-python/pypaimon/tests/test_rest_catalog_commit_snapshot.py
index 25350ebea0..84c6557c1e 100644
--- a/paimon-python/pypaimon/tests/test_rest_catalog_commit_snapshot.py
+++ b/paimon-python/pypaimon/tests/test_rest_catalog_commit_snapshot.py
@@ -22,15 +22,15 @@ import time
import unittest
from unittest.mock import Mock, patch
-from pypaimon.api import NoSuchResourceException
from pypaimon.api.api_response import CommitTableResponse
from pypaimon.api.options import Options
+from pypaimon.api.rest_exception import NoSuchResourceException
from pypaimon.catalog.catalog_context import CatalogContext
from pypaimon.catalog.catalog_exception import TableNotExistException
from pypaimon.catalog.rest.rest_catalog import RESTCatalog
-from pypaimon.catalog.snapshot_commit import PartitionStatistics
from pypaimon.common.identifier import Identifier
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import PartitionStatistics
class TestRESTCatalogCommitSnapshot(unittest.TestCase):
@@ -120,7 +120,7 @@ class TestRESTCatalogCommitSnapshot(unittest.TestCase):
# Configure mock to raise NoSuchResourceException
mock_api_instance = Mock()
mock_api_instance.options = self.test_options
- mock_api_instance.commit_snapshot.side_effect =
NoSuchResourceException("Table not found")
+ mock_api_instance.commit_snapshot.side_effect =
NoSuchResourceException("Table", None, "not found")
mock_rest_api.return_value = mock_api_instance
# Create RESTCatalog
@@ -188,9 +188,9 @@ class TestRESTCatalogCommitSnapshot(unittest.TestCase):
def test_rest_api_commit_snapshot(self):
"""Test RESTApi commit_snapshot method."""
- from pypaimon.api import RESTApi
+ from pypaimon.api.rest_api import RESTApi
- with patch('pypaimon.api.HttpClient') as mock_client_class:
+ with patch('pypaimon.api.client.HttpClient') as mock_client_class:
# Configure mock
mock_client = Mock()
mock_response = CommitTableResponse(success=True)
diff --git a/paimon-python/pypaimon/write/commit_message.py
b/paimon-python/pypaimon/write/commit_message.py
index 4e4c0f0b48..b36a1b1bbf 100644
--- a/paimon-python/pypaimon/write/commit_message.py
+++ b/paimon-python/pypaimon/write/commit_message.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
from dataclasses import dataclass
from typing import List, Tuple
diff --git a/paimon-python/pypaimon/write/file_store_commit.py
b/paimon-python/pypaimon/write/file_store_commit.py
index ca26981333..e7bf7ba534 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -21,12 +21,13 @@ import uuid
from pathlib import Path
from typing import List
-from pypaimon.catalog.snapshot_commit import PartitionStatistics,
SnapshotCommit
from pypaimon.manifest.manifest_file_manager import ManifestFileManager
from pypaimon.manifest.manifest_list_manager import ManifestListManager
from pypaimon.manifest.schema.manifest_file_meta import ManifestFileMeta
from pypaimon.manifest.schema.simple_stats import SimpleStats
from pypaimon.snapshot.snapshot import Snapshot
+from pypaimon.snapshot.snapshot_commit import (PartitionStatistics,
+ SnapshotCommit)
from pypaimon.snapshot.snapshot_manager import SnapshotManager
from pypaimon.table.row.binary_row import BinaryRow
from pypaimon.write.commit_message import CommitMessage
diff --git a/paimon-python/pypaimon/write/row_key_extractor.py
b/paimon-python/pypaimon/write/row_key_extractor.py
index bec8e08fb7..aa28aacef2 100644
--- a/paimon-python/pypaimon/write/row_key_extractor.py
+++ b/paimon-python/pypaimon/write/row_key_extractor.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
import hashlib
import json
from abc import ABC, abstractmethod
