This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 6778dca2e refactor(dev): use minijinja (#5494)
6778dca2e is described below
commit 6778dca2ec834d4e378972f7e107fc7942f28210
Author: tison <[email protected]>
AuthorDate: Thu Jan 2 14:37:27 2025 +0800
refactor(dev): use minijinja (#5494)
---
bindings/python/Cargo.toml | 2 +-
bindings/python/python/opendal/__base.pyi | 78 +++++++++++--
dev/Cargo.lock | 123 +--------------------
dev/Cargo.toml | 9 +-
dev/src/generate/binding_python.rs | 89 ---------------
dev/src/generate/mod.rs | 14 +--
dev/src/generate/parser.rs | 40 +++----
.../python.py.jinja2 => src/generate/python.j2} | 35 +++---
dev/src/generate/python.rs | 71 ++++++++++++
dev/src/main.rs | 23 ++--
10 files changed, 206 insertions(+), 278 deletions(-)
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index bf5739794..84c38d73e 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -47,7 +47,7 @@ default = [
# NOTE: this is the feature we used to build pypi wheels.
# When enable or disable some features,
-# also need to update dev/src/generate/binding_python.rs `enabled_service` to
match it.
+# Also, you need to update the `enabled_service` function in
dev/src/generate/python.rs to match it.
services-all = [
"default",
"services-aliyun-drive",
diff --git a/bindings/python/python/opendal/__base.pyi
b/bindings/python/python/opendal/__base.pyi
index c9e794b81..6096af429 100644
--- a/bindings/python/python/opendal/__base.pyi
+++ b/bindings/python/python/opendal/__base.pyi
@@ -1,9 +1,3 @@
-"""
-this file is generated by opendal/dev/generate/binding_python.rs, and
opendal.__base doesn't exists.
-
-DO NOT EDIT IT Manually
-"""
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -21,6 +15,12 @@ DO NOT EDIT IT Manually
# specific language governing permissions and limitations
# under the License.
+"""
+> DO NOT EDIT IT MANUALLY <
+
+This file is generated by opendal/dev/generate/python.rs. `opendal.__base`
doesn't exists.
+"""
+
from typing import overload, Literal, TypeAlias
# `true`/`false`` in any case, for example, `true`/`True`/`TRUE`
`false`/`False`/`FALSE`
@@ -33,6 +33,7 @@ _int: TypeAlias = str
# for more details
_duration: TypeAlias = str
+
# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"`
_strings: TypeAlias = str
@@ -42,6 +43,7 @@ class _Base:
The services list here is support by opendal pypi wheel.
"""
+
@overload
def __init__(
self,
@@ -55,6 +57,7 @@ class _Base:
client_secret: str = ...,
refresh_token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -64,6 +67,7 @@ class _Base:
root: str = ...,
endpoint: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -76,6 +80,7 @@ class _Base:
public_key: str = ...,
parent_resource_id: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -93,6 +98,7 @@ class _Base:
sas_token: str = ...,
batch_max_operations: _int = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -105,6 +111,7 @@ class _Base:
account_name: str = ...,
account_key: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -118,6 +125,7 @@ class _Base:
account_key: str = ...,
sas_token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -130,6 +138,7 @@ class _Base:
application_key_id: str = ...,
application_key: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -138,6 +147,7 @@ class _Base:
*,
datadir: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -148,6 +158,7 @@ class _Base:
root: str = ...,
api_key: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -159,6 +170,7 @@ class _Base:
namespace_id: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -167,6 +179,7 @@ class _Base:
*,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -180,6 +193,7 @@ class _Base:
bucket: str = ...,
disable_config_load: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -194,6 +208,7 @@ class _Base:
key_field: str = ...,
value_field: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -202,6 +217,7 @@ class _Base:
*,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -212,6 +228,7 @@ class _Base:
endpoint: str = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -224,6 +241,7 @@ class _Base:
client_id: str = ...,
client_secret: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -233,6 +251,7 @@ class _Base:
root: str = ...,
atomic_write_dir: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -253,6 +272,7 @@ class _Base:
disable_config_load: _bool = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -265,6 +285,7 @@ class _Base:
client_id: str = ...,
client_secret: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -276,6 +297,7 @@ class _Base:
endpoint: str = ...,
runtime_token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -287,6 +309,7 @@ class _Base:
root: str = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -299,6 +322,7 @@ class _Base:
chunk_size: _int = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -309,6 +333,7 @@ class _Base:
url: str = ...,
enable_append: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -321,6 +346,7 @@ class _Base:
token: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -333,6 +359,7 @@ class _Base:
root: str = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -346,6 +373,7 @@ class _Base:
ds_web_auth_token: str = ...,
is_china_mainland: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -355,6 +383,7 @@ class _Base:
endpoint: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -364,6 +393,7 @@ class _Base:
root: str = ...,
endpoint: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -375,6 +405,7 @@ class _Base:
root: str = ...,
password: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -388,6 +419,7 @@ class _Base:
repository: str = ...,
branch: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -401,6 +433,7 @@ class _Base:
value_field: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -413,6 +446,7 @@ class _Base:
password: str = ...,
default_ttl: _duration = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -421,6 +455,7 @@ class _Base:
*,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -432,6 +467,7 @@ class _Base:
time_to_idle: _duration = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -445,6 +481,7 @@ class _Base:
num_segments: _int = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -458,6 +495,7 @@ class _Base:
key_field: str = ...,
value_field: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -466,6 +504,7 @@ class _Base:
*,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -478,6 +517,7 @@ class _Base:
value_field: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -494,6 +534,7 @@ class _Base:
value_field: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -506,6 +547,7 @@ class _Base:
secret_access_key: str = ...,
bucket: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -515,6 +557,7 @@ class _Base:
access_token: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -539,6 +582,7 @@ class _Base:
oidc_token_file: str = ...,
sts_endpoint: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -550,6 +594,7 @@ class _Base:
username: str = ...,
password: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -560,6 +605,7 @@ class _Base:
segment: str = ...,
index: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -572,6 +618,7 @@ class _Base:
key_field: str = ...,
value_field: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -582,6 +629,7 @@ class _Base:
root: str = ...,
table: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -596,6 +644,7 @@ class _Base:
root: str = ...,
default_ttl: _duration = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -630,6 +679,7 @@ class _Base:
checksum_algorithm: str = ...,
disable_write_with_if_match: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -642,6 +692,7 @@ class _Base:
username: str = ...,
password: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -655,6 +706,7 @@ class _Base:
known_hosts_strategy: str = ...,
enable_copy: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -665,6 +717,7 @@ class _Base:
root: str = ...,
tree: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -677,6 +730,7 @@ class _Base:
value_field: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -688,6 +742,7 @@ class _Base:
endpoint: str = ...,
key: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -704,6 +759,7 @@ class _Base:
value_field: str = ...,
root: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -715,6 +771,7 @@ class _Base:
root: str = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -726,6 +783,7 @@ class _Base:
operator: str = ...,
password: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -734,6 +792,7 @@ class _Base:
*,
access_token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -743,6 +802,7 @@ class _Base:
root: str = ...,
token: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -756,6 +816,7 @@ class _Base:
root: str = ...,
disable_copy: _bool = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -768,6 +829,7 @@ class _Base:
disable_list_batch: _bool = ...,
atomic_write_dir: str = ...,
) -> None: ...
+
@overload
def __init__(
self,
@@ -777,5 +839,7 @@ class _Base:
access_token: str,
root: str = ...,
) -> None: ...
+
+
@overload
- def __init__(self, scheme: str, /, **kwargs: str) -> None: ...
+ def __init__(self, scheme: str, /, **kwargs: str) -> None: ...
\ No newline at end of file
diff --git a/dev/Cargo.lock b/dev/Cargo.lock
index f0479e853..9557672df 100644
--- a/dev/Cargo.lock
+++ b/dev/Cargo.lock
@@ -66,15 +66,6 @@ version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
-[[package]]
-name = "basic-toml"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "823388e228f614e9558c6804262db37960ec8821856535f5c3f59913140558f8"
-dependencies = [
- "serde",
-]
-
[[package]]
name = "clap"
version = "4.5.23"
@@ -156,15 +147,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-[[package]]
-name = "humansize"
-version = "2.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
-dependencies = [
- "libm",
-]
-
[[package]]
name = "humantime"
version = "2.1.0"
@@ -177,18 +159,6 @@ version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
-[[package]]
-name = "itoa"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
-
-[[package]]
-name = "libm"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
-
[[package]]
name = "log"
version = "0.4.22"
@@ -202,35 +172,12 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "mime_guess"
-version = "2.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
-dependencies = [
- "mime",
- "unicase",
-]
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "nom"
-version = "7.1.3"
+name = "minijinja"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+checksum = "2c37e1b517d1dcd0e51dc36c4567b9d5a29262b3ec8da6cb5d35e27a8fb529b5"
dependencies = [
- "memchr",
- "minimal-lexical",
+ "serde",
]
[[package]]
@@ -241,17 +188,12 @@ dependencies = [
"clap",
"env_logger",
"log",
+ "minijinja",
"pretty_assertions",
- "rinja",
+ "serde",
"syn",
]
-[[package]]
-name = "percent-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-
[[package]]
name = "pretty_assertions"
version = "1.4.1"
@@ -309,53 +251,6 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
-[[package]]
-name = "rinja"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5"
-dependencies = [
- "humansize",
- "itoa",
- "percent-encoding",
- "rinja_derive",
-]
-
-[[package]]
-name = "rinja_derive"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b"
-dependencies = [
- "basic-toml",
- "memchr",
- "mime",
- "mime_guess",
- "proc-macro2",
- "quote",
- "rinja_parser",
- "rustc-hash",
- "serde",
- "syn",
-]
-
-[[package]]
-name = "rinja_parser"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610"
-dependencies = [
- "memchr",
- "nom",
- "serde",
-]
-
-[[package]]
-name = "rustc-hash"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
-
[[package]]
name = "serde"
version = "1.0.216"
@@ -393,12 +288,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "unicase"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
-
[[package]]
name = "unicode-ident"
version = "1.0.14"
diff --git a/dev/Cargo.toml b/dev/Cargo.toml
index 9d0e98acd..2acaeecae 100644
--- a/dev/Cargo.toml
+++ b/dev/Cargo.toml
@@ -28,11 +28,12 @@ rust-version = "1.75"
version = "0.0.1"
[dependencies]
-anyhow = "1.0.95"
+anyhow = { version = "1.0.95" }
clap = { version = "4.5.23", features = ["derive"] }
-env_logger = "0.11.6"
-log = "0.4.22"
-rinja = "0.3.5"
+env_logger = { version = "0.11.5" }
+log = { version = "0.4.22" }
+minijinja = { version = "2.5.0" }
+serde = { version = "1.0.216", features = ["derive"] }
syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] }
[dev-dependencies]
diff --git a/dev/src/generate/binding_python.rs
b/dev/src/generate/binding_python.rs
deleted file mode 100644
index 68bd9b457..000000000
--- a/dev/src/generate/binding_python.rs
+++ /dev/null
@@ -1,89 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::generate::parser::Services;
-use anyhow::Result;
-use rinja::Template;
-use std::fs;
-use std::path::PathBuf;
-
-use super::parser::{ConfigType, Service};
-
-// Using the template in this path, relative
-// to the `templates` dir in the crate root
-#[derive(Template)]
-#[template(path = "python.py.jinja2", escape = "none")]
-struct PythonTemplate {
- services: Vec<(String, Service)>,
-}
-
-/// TODO: add a common utils to parse enabled features from cargo.toml
-fn enabled_service(srv: &str) -> bool {
- match srv {
- // not enabled in bindings/python/Cargo.toml
- "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false,
- _ => true,
- }
-}
-
-pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> {
- let mut v = Vec::from_iter(
- services
- .clone()
- .into_iter()
- .filter(|x| enabled_service(x.0.as_str())),
- );
-
- // move required options at beginning.
- for srv in &mut v {
- let mut v =
Vec::from_iter(srv.1.config.clone().into_iter().enumerate());
-
- v.sort_by_key(|a| (a.1.optional, a.0));
-
- srv.1.config = v.iter().map(|f| f.1.clone()).collect();
- }
-
- let tmpl = PythonTemplate { services: v };
-
- let t = tmpl.render().expect("should render template");
-
- let output_file: String = project_root
- .join("bindings/python/python/opendal/__base.pyi")
- .to_str()
- .expect("should build output file path")
- .into();
-
- fs::write(output_file, t).expect("failed to write result to file");
-
- Ok(())
-}
-
-impl ConfigType {
- pub fn python_type(&self) -> String {
- match self {
- ConfigType::Bool => "_bool".into(),
- ConfigType::Duration => "_duration".into(),
- ConfigType::I64
- | ConfigType::Usize
- | ConfigType::U64
- | ConfigType::U32
- | ConfigType::U16 => "_int".into(),
- ConfigType::Vec => "_strings".into(),
- ConfigType::String => "str".into(),
- }
- }
-}
diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs
index 71ec44429..780f19a74 100644
--- a/dev/src/generate/mod.rs
+++ b/dev/src/generate/mod.rs
@@ -16,20 +16,18 @@
// under the License.
mod parser;
+mod python;
-mod binding_python;
-
+use crate::workspace_dir;
use anyhow::Result;
-use std::path::PathBuf;
pub fn run(language: &str) -> Result<()> {
- let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- let services_path =
manifest_dir.join("../core/src/services").canonicalize()?;
- let project_root = manifest_dir.join("..").canonicalize()?;
+ let workspace_dir = workspace_dir();
+ let services_path =
workspace_dir.join("core/src/services").canonicalize()?;
let services = parser::parse(&services_path.to_string_lossy())?;
match language {
- "python" | "py" => binding_python::generate(project_root, &services),
- _ => Err(anyhow::anyhow!("Unsupported language: {}", language)),
+ "python" | "py" => python::generate(workspace_dir, services),
+ _ => anyhow::bail!("unsupported language: {}", language),
}
}
diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs
index 45cfe7cb4..086d7be5a 100644
--- a/dev/src/generate/parser.rs
+++ b/dev/src/generate/parser.rs
@@ -15,40 +15,29 @@
// specific language governing permissions and limitations
// under the License.
-use anyhow::Result;
use anyhow::{anyhow, Context};
+use anyhow::{bail, Result};
use log::debug;
+use serde::{Deserialize, Serialize};
use std::collections::HashMap;
+use std::fs;
use std::fs::read_dir;
use std::str::FromStr;
-use std::{fs, vec};
use syn::{
Expr, ExprLit, Field, GenericArgument, Item, Lit, LitStr, Meta,
PathArguments, Type, TypePath,
};
-#[derive(Debug, Clone)]
-pub struct Services(HashMap<String, Service>);
-
-impl IntoIterator for Services {
- type Item = (String, Service);
- type IntoIter = vec::IntoIter<(String, Service)>;
-
- fn into_iter(self) -> Self::IntoIter {
- let mut v = Vec::from_iter(self.0);
- v.sort();
- v.into_iter()
- }
-}
+pub type Services = HashMap<String, Service>;
/// Service represents a service supported by opendal core, like `s3` and `fs`
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
pub struct Service {
/// All configurations for this service.
pub config: Vec<Config>,
}
/// Config represents a configuration item for a service.
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
pub struct Config {
/// The name of this config, for example, `access_key_id` and
`secret_access_key`
pub name: String,
@@ -64,7 +53,7 @@ pub struct Config {
pub comments: String,
}
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize,
Deserialize)]
pub enum ConfigType {
/// Mapping to rust's `bool`
Bool,
@@ -94,7 +83,7 @@ impl FromStr for ConfigType {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
- let ct = match s {
+ Ok(match s {
"bool" => ConfigType::Bool,
"String" => ConfigType::String,
"Duration" => ConfigType::Duration,
@@ -106,10 +95,9 @@ impl FromStr for ConfigType {
"u16" => ConfigType::U16,
"Vec" => ConfigType::Vec,
- v => return Err(anyhow!("unsupported config type {v:?}")),
- };
- Ok(ct)
+ v => bail!("unsupported config type {v}"),
+ })
}
}
@@ -135,11 +123,11 @@ impl FromStr for ConfigType {
/// ```
///
/// - since = "0.52.0"
-#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize,
Deserialize)]
pub struct AttrDeprecated {
- /// The since of this deprecated field.
+ /// The `since` of this deprecated field.
pub since: String,
- /// The note for this deprecated field.
+ /// The `note` for this deprecated field.
pub note: String,
}
@@ -163,7 +151,7 @@ pub fn parse(path: &str) -> Result<Services> {
map.insert(parser.service, service);
}
- Ok(Services(map))
+ Ok(map)
}
/// ServiceParser is used to parse a service config file.
diff --git a/dev/templates/python.py.jinja2 b/dev/src/generate/python.j2
similarity index 71%
rename from dev/templates/python.py.jinja2
rename to dev/src/generate/python.j2
index 7c8b2732e..772407d52 100644
--- a/dev/templates/python.py.jinja2
+++ b/dev/src/generate/python.j2
@@ -1,9 +1,3 @@
-"""
-this file is generated by opendal/dev/generate/binding_python.rs, and
opendal.__base doesn't exists.
-
-DO NOT EDIT IT Manually
-"""
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -21,6 +15,12 @@ DO NOT EDIT IT Manually
# specific language governing permissions and limitations
# under the License.
+"""
+> DO NOT EDIT IT MANUALLY <
+
+This file is generated by opendal/dev/generate/python.rs. `opendal.__base`
doesn't exists.
+"""
+
from typing import overload, Literal, TypeAlias
# `true`/`false`` in any case, for example, `true`/`True`/`TRUE`
`false`/`False`/`FALSE`
@@ -43,20 +43,21 @@ class _Base:
The services list here is support by opendal pypi wheel.
"""
-{% for srv in services %}
+{% for srv in srvs %}
@overload
- def __init__(self,
- scheme: Literal["{{srv.0}}"],
+ def __init__(
+ self,
+ scheme: Literal["{{srv}}"],
/,
*,
- {% for field in srv.1.config.clone().into_iter() %}
- {% if field.deprecated.is_some() %}
- # deprecated: {{ field.deprecated.unwrap().note }}
- {% endif %}
- {{field.name}}: {{field.value.python_type()}} {% if field.optional %}
= ... {% endif %},
- {% endfor %}
- ) -> None: ...
+ {%- for field in srvs[srv].config -%}
+ {% if field.deprecated %}
+ # deprecated: {{field.deprecated["note"]}}
+ {%- endif %}
+ {{field.name}}: {{make_pytype(field.value)}}{% if field.optional %} =
...{% endif %},
+ {%- endfor %}
+ ) -> None: ...
{% endfor %}
@overload
- def __init__(self, scheme:str, /, **kwargs: str) -> None: ...
+ def __init__(self, scheme: str, /, **kwargs: str) -> None: ...
diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs
new file mode 100644
index 000000000..268a6e89d
--- /dev/null
+++ b/dev/src/generate/python.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::generate::parser::{ConfigType, Service, Services};
+use anyhow::Result;
+use minijinja::value::ViaDeserialize;
+use minijinja::{context, Environment};
+use std::fs;
+use std::path::PathBuf;
+
+/// TODO: add a common utils to parse enabled features from cargo.toml
+fn enabled_service(srv: &str) -> bool {
+ match srv {
+ // not enabled in bindings/python/Cargo.toml
+ "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false,
+ _ => true,
+ }
+}
+
+pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> {
+ let mut srvs = Services::new();
+ for (k, srv) in services.into_iter() {
+ if !enabled_service(k.as_str()) {
+ continue;
+ }
+
+ let mut sorted =
srv.config.into_iter().enumerate().collect::<Vec<_>>();
+ sorted.sort_by_key(|(i, v)| (v.optional, *i));
+ let config = sorted.into_iter().map(|(_, v)| v).collect();
+ srvs.insert(k, Service { config });
+ }
+
+ let mut env = Environment::new();
+ env.add_template("python", include_str!("python.j2"))?;
+ env.add_function("make_pytype", make_pytype);
+ let tmpl = env.get_template("python")?;
+
+ let output =
workspace_dir.join("bindings/python/python/opendal/__base.pyi");
+ fs::write(output, tmpl.render(context! { srvs => srvs })?)?;
+
+ Ok(())
+}
+
+fn make_pytype(ty: ViaDeserialize<ConfigType>) -> Result<String,
minijinja::Error> {
+ Ok(match ty.0 {
+ ConfigType::Bool => "_bool",
+ ConfigType::Duration => "_duration",
+ ConfigType::I64
+ | ConfigType::Usize
+ | ConfigType::U64
+ | ConfigType::U32
+ | ConfigType::U16 => "_int",
+ ConfigType::Vec => "_strings",
+ ConfigType::String => "str",
+ }
+ .to_string())
+}
diff --git a/dev/src/main.rs b/dev/src/main.rs
index a56430abf..1858e89b1 100644
--- a/dev/src/main.rs
+++ b/dev/src/main.rs
@@ -19,10 +19,21 @@ mod generate;
use anyhow::Result;
use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+
+fn manifest_dir() -> PathBuf {
+ PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .canonicalize()
+ .unwrap()
+}
+
+fn workspace_dir() -> PathBuf {
+ manifest_dir().join("..").canonicalize().unwrap()
+}
#[derive(Parser)]
#[command(version, about, long_about = None)]
-struct Cli {
+struct Cmd {
#[command(subcommand)]
command: Commands,
}
@@ -39,13 +50,7 @@ enum Commands {
fn main() -> Result<()> {
env_logger::init();
- let cli = Cli::parse();
-
- match cli.command {
- Commands::Generate { language } => {
- generate::run(&language)?;
- }
+ match Cmd::parse().command {
+ Commands::Generate { language } => generate::run(&language),
}
-
- Ok(())
}
