This is an automated email from the ASF dual-hosted git repository.
jiacai2050 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/horaedb.git
The following commit(s) were added to refs/heads/main by this push:
new 115b002f feat: sort record batch of write request (#1585)
115b002f is described below
commit 115b002fca240dec5480c51377a6ed10f40a3dd2
Author: 鲍金日 <[email protected]>
AuthorDate: Mon Nov 4 16:13:58 2024 +0800
feat: sort record batch of write request (#1585)
## Rationale
Support sorted record batch of write request by primary key.
## Detailed Changes
- use datafusion to sort record batch
## Test Plan
- CI
- Unit tests
---
.github/workflows/metric-engine-ci.yml | 27 +
horaedb/Cargo.lock | 883 ++++++++++-----------------------
horaedb/Cargo.toml | 2 +-
horaedb/Makefile | 3 +
horaedb/metric_engine/src/storage.rs | 110 +++-
5 files changed, 408 insertions(+), 617 deletions(-)
diff --git a/.github/workflows/metric-engine-ci.yml
b/.github/workflows/metric-engine-ci.yml
index be80ebe8..cad7d3ab 100644
--- a/.github/workflows/metric-engine-ci.yml
+++ b/.github/workflows/metric-engine-ci.yml
@@ -70,3 +70,30 @@ jobs:
working-directory: horaedb
run: |
git diff --exit-code
+
+ unit-test:
+ name: unit-test
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: true
+ - run: |
+ rustup set auto-self-update disable
+ rustup toolchain install ${RUST_VERSION} --profile minimal
+ - name: Release Disk Quota
+ run: |
+ sudo make ensure-disk-quota
+ - name: Setup Build Environment
+ run: |
+ sudo apt update
+ sudo apt install --yes protobuf-compiler
+ - name: Run Unit Tests
+ working-directory: horaedb
+ run: |
+ make test
+ - name: Check lock
+ working-directory: horaedb
+ run: |
+ git diff --exit-code
diff --git a/horaedb/Cargo.lock b/horaedb/Cargo.lock
index 632b74b7..64912637 100644
--- a/horaedb/Cargo.lock
+++ b/horaedb/Cargo.lock
@@ -96,71 +96,35 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85"
+checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3"
dependencies = [
- "arrow-arith 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-cast 52.2.0",
- "arrow-csv 52.2.0",
- "arrow-data 52.2.0",
- "arrow-ipc 52.2.0",
- "arrow-json 52.2.0",
- "arrow-ord 52.2.0",
- "arrow-row 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-select 52.2.0",
- "arrow-string 52.2.0",
-]
-
-[[package]]
-name = "arrow"
-version = "53.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba"
-dependencies = [
- "arrow-arith 53.0.0",
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-cast 53.1.0",
- "arrow-csv 53.0.0",
- "arrow-data 53.1.0",
- "arrow-ipc 53.1.0",
- "arrow-json 53.0.0",
- "arrow-ord 53.0.0",
- "arrow-row 53.0.0",
- "arrow-schema 53.1.0",
- "arrow-select 53.1.0",
- "arrow-string 53.0.0",
+ "arrow-arith",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-csv",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-json",
+ "arrow-ord",
+ "arrow-row",
+ "arrow-schema",
+ "arrow-select",
+ "arrow-string",
]
[[package]]
name = "arrow-arith"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c"
+checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a"
dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "chrono",
- "half",
- "num",
-]
-
-[[package]]
-name = "arrow-arith"
-version = "53.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a"
-dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"half",
"num",
@@ -168,14 +132,14 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c"
+checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
dependencies = [
"ahash",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"chrono-tz",
"half",
@@ -183,343 +147,164 @@ dependencies = [
"num",
]
-[[package]]
-name = "arrow-array"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da"
-dependencies = [
- "ahash",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "chrono",
- "half",
- "hashbrown",
- "num",
-]
-
[[package]]
name = "arrow-buffer"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1"
+checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
dependencies = [
"bytes",
"half",
"num",
]
-[[package]]
-name = "arrow-buffer"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80"
-dependencies = [
- "bytes",
- "half",
- "num",
-]
-
-[[package]]
-name = "arrow-cast"
-version = "52.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e"
-dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-select 52.2.0",
- "atoi",
- "base64",
- "chrono",
- "comfy-table",
- "half",
- "lexical-core 0.8.5",
- "num",
- "ryu",
-]
-
[[package]]
name = "arrow-cast"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4"
+checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "arrow-select 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
"atoi",
"base64",
"chrono",
"comfy-table",
"half",
- "lexical-core 1.0.2",
+ "lexical-core",
"num",
"ryu",
]
[[package]]
name = "arrow-csv"
-version = "52.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2"
-dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-cast 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "chrono",
- "csv",
- "csv-core",
- "lazy_static",
- "lexical-core 0.8.5",
- "regex",
-]
-
-[[package]]
-name = "arrow-csv"
-version = "53.0.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740"
+checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d"
dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-cast 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"csv",
"csv-core",
"lazy_static",
- "lexical-core 0.8.5",
+ "lexical-core",
"regex",
]
[[package]]
name = "arrow-data"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5"
+checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
dependencies = [
- "arrow-buffer 52.2.0",
- "arrow-schema 52.2.0",
- "half",
- "num",
-]
-
-[[package]]
-name = "arrow-data"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634"
-dependencies = [
- "arrow-buffer 53.1.0",
- "arrow-schema 53.1.0",
+ "arrow-buffer",
+ "arrow-schema",
"half",
"num",
]
[[package]]
name = "arrow-ipc"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f"
+checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-cast 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-schema",
"flatbuffers",
"lz4_flex",
]
-[[package]]
-name = "arrow-ipc"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2"
-dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-cast 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "flatbuffers",
-]
-
[[package]]
name = "arrow-json"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445"
+checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c"
dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-cast 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-schema",
"chrono",
"half",
"indexmap",
- "lexical-core 0.8.5",
+ "lexical-core",
"num",
"serde",
"serde_json",
]
-[[package]]
-name = "arrow-json"
-version = "53.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6"
-dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-cast 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "chrono",
- "half",
- "indexmap",
- "lexical-core 0.8.5",
- "num",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "arrow-ord"
-version = "52.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc"
-dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-select 52.2.0",
- "half",
- "num",
-]
-
[[package]]
name = "arrow-ord"
-version = "53.0.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b"
+checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52"
dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "arrow-select 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
"half",
"num",
]
[[package]]
name = "arrow-row"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c"
+checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be"
dependencies = [
"ahash",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "half",
-]
-
-[[package]]
-name = "arrow-row"
-version = "53.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb"
-dependencies = [
- "ahash",
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"half",
]
[[package]]
name = "arrow-schema"
-version = "52.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8"
-
-[[package]]
-name = "arrow-schema"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794"
-
-[[package]]
-name = "arrow-select"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3"
-dependencies = [
- "ahash",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "num",
-]
+checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
[[package]]
name = "arrow-select"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a"
+checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
dependencies = [
"ahash",
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
"num",
]
[[package]]
name = "arrow-string"
-version = "52.2.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6"
+checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c"
dependencies = [
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-data 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-select 52.2.0",
- "memchr",
- "num",
- "regex",
- "regex-syntax",
-]
-
-[[package]]
-name = "arrow-string"
-version = "53.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230"
-dependencies = [
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-data 53.1.0",
- "arrow-schema 53.1.0",
- "arrow-select 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
"memchr",
"num",
"regex",
@@ -552,7 +337,7 @@ checksum =
"a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -725,9 +510,9 @@ dependencies = [
[[package]]
name = "chrono-tz"
-version = "0.9.0"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
+checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6"
dependencies = [
"chrono",
"chrono-tz-build",
@@ -736,12 +521,11 @@ dependencies = [
[[package]]
name = "chrono-tz-build"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
+checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7"
dependencies = [
"parse-zoneinfo",
- "phf",
"phf_codegen",
]
@@ -865,15 +649,15 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb"
+checksum = "3e8053b4cedc24eb158e4c041b38cfa0677ef5f4a7ccaa31ee5dcad61dd7aa54"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-ipc 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-ipc",
+ "arrow-schema",
"async-compression",
"async-trait",
"bytes",
@@ -888,6 +672,7 @@ dependencies = [
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-nested",
+ "datafusion-functions-window",
"datafusion-optimizer",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
@@ -900,12 +685,12 @@ dependencies = [
"half",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools 0.13.0",
"log",
"num_cpus",
- "object_store 0.10.2",
+ "object_store",
"parking_lot",
- "parquet 52.2.0",
+ "parquet",
"paste",
"pin-project-lite",
"rand",
@@ -921,56 +706,60 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529"
+checksum = "7d95efedb3a32f6f74df5bb8fda7b69fb9babe80e92137f25de6ddb15e8e8801"
dependencies = [
- "arrow-schema 52.2.0",
+ "arrow-schema",
"async-trait",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
+ "parking_lot",
]
[[package]]
name = "datafusion-common"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c"
+checksum = "b7d766e0d3dec01a0ab70b1b31678c286cddc0bd7afc9bd82504a1d9a70a7d94"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
"chrono",
"half",
"hashbrown",
"instant",
"libc",
"num_cpus",
- "object_store 0.10.2",
- "parquet 52.2.0",
+ "object_store",
+ "parquet",
+ "paste",
"sqlparser",
+ "tokio",
]
[[package]]
name = "datafusion-common-runtime"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1"
+checksum = "4e55db6df319f9e7cf366d0d4ffae793c863823421b2f2b7314a0fefd8e8c11a"
dependencies = [
+ "log",
"tokio",
]
[[package]]
name = "datafusion-execution"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1"
+checksum = "6d0c6dc013f955c382438a78fa3de8b0a8bf7b1a4cda5bc46335fe445ff3ff1a"
dependencies = [
- "arrow 52.2.0",
+ "arrow",
"chrono",
"dashmap",
"datafusion-common",
@@ -978,7 +767,7 @@ dependencies = [
"futures",
"hashbrown",
"log",
- "object_store 0.10.2",
+ "object_store",
"parking_lot",
"rand",
"tempfile",
@@ -987,16 +776,19 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2"
+checksum = "7f31405c0bb854451d755b224d41dc466a8f7fd36f8c041c29d2d8432bd0c08c"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-buffer",
"chrono",
"datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr-common",
"paste",
"serde_json",
"sqlparser",
@@ -1004,14 +796,25 @@ dependencies = [
"strum_macros",
]
+[[package]]
+name = "datafusion-expr-common"
+version = "42.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ebc8266b6627c8264c87bc7c82564e3d89ed5f0f9943b49a30dac1f1ac12e4c0"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "paste",
+]
+
[[package]]
name = "datafusion-functions"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb"
+checksum = "5712668780bc43666ecd10acd188b7df58e2a5501d4dbbd972bf209f1790138b"
dependencies = [
- "arrow 52.2.0",
- "arrow-buffer 52.2.0",
+ "arrow",
+ "arrow-buffer",
"base64",
"blake2",
"blake3",
@@ -1021,7 +824,7 @@ dependencies = [
"datafusion-expr",
"hashbrown",
"hex",
- "itertools 0.12.1",
+ "itertools 0.13.0",
"log",
"md-5",
"rand",
@@ -1033,51 +836,81 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb"
+checksum = "8ec138af6b7482fb726f1bfeec010fc063b9614594c36a1051a4d3b365ba6a5f"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-schema",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-functions-aggregate-common",
+ "datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "half",
"log",
"paste",
"sqlparser",
]
+[[package]]
+name = "datafusion-functions-aggregate-common"
+version = "42.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "564499c6bdd3ab9f76c7ad727e858bc6791e4de6c1a484d21d2bf49daaa658d6"
+dependencies = [
+ "ahash",
+ "arrow",
+ "datafusion-common",
+ "datafusion-expr-common",
+ "datafusion-physical-expr-common",
+ "rand",
+]
+
[[package]]
name = "datafusion-functions-nested"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d"
+checksum = "9b55ea2221ae1c1e37d524f8330f763dcdc205edb74fe5f54cbdea475c17fd18"
dependencies = [
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-ord 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-ord",
+ "arrow-schema",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
- "itertools 0.12.1",
+ "datafusion-physical-expr-common",
+ "itertools 0.13.0",
"log",
"paste",
"rand",
]
+[[package]]
+name = "datafusion-functions-window"
+version = "42.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6932996c4407ee1ebf23ffd706e982729cb9b6f7a31a281abac51fe524c3a049"
+dependencies = [
+ "datafusion-common",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "log",
+]
+
[[package]]
name = "datafusion-optimizer"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906"
+checksum = "f7d8afa1eb44e2f00cc8d82b88803e456a681474b8877ceecc04e9517d5c843c"
dependencies = [
- "arrow 52.2.0",
+ "arrow",
"async-trait",
"chrono",
"datafusion-common",
@@ -1085,7 +918,7 @@ dependencies = [
"datafusion-physical-expr",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools 0.13.0",
"log",
"paste",
"regex-syntax",
@@ -1093,28 +926,30 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4"
+checksum = "570666d84df483473626fab4e69997d048b40d0e7c67c540299714f244d99e73"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-ord 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-string 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-ord",
+ "arrow-schema",
+ "arrow-string",
"base64",
"chrono",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-expr-common",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
"hashbrown",
"hex",
"indexmap",
- "itertools 0.12.1",
+ "itertools 0.13.0",
"log",
"paste",
"petgraph",
@@ -1123,42 +958,44 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6"
+checksum = "3746cbdfb32d67399dcaad17042e419ac6da454a7e38ff098aa2fbf0a7388982"
dependencies = [
"ahash",
- "arrow 52.2.0",
+ "arrow",
"datafusion-common",
- "datafusion-expr",
+ "datafusion-expr-common",
"hashbrown",
"rand",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0"
+checksum = "696f06e79d44f7c50f57cea23493881d86d9d9647884d38ce467c7f75c13e286"
dependencies = [
+ "arrow-schema",
"datafusion-common",
"datafusion-execution",
"datafusion-physical-expr",
"datafusion-physical-plan",
+ "itertools 0.13.0",
]
[[package]]
name = "datafusion-physical-plan"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe"
+checksum = "04e1d084224023e09cdea14d01ded0f2092c319c7b4594ebc821283b9c7c4a35"
dependencies = [
"ahash",
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-ord 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-ord",
+ "arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -1166,13 +1003,14 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"futures",
"half",
"hashbrown",
"indexmap",
- "itertools 0.12.1",
+ "itertools 0.13.0",
"log",
"once_cell",
"parking_lot",
@@ -1183,13 +1021,13 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "41.0.0"
+version = "42.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0"
+checksum = "c105148357dcbd9e4c97eada2930a59f7923215461d9f47de6e76edd60eab2d5"
dependencies = [
- "arrow 52.2.0",
- "arrow-array 52.2.0",
- "arrow-schema 52.2.0",
+ "arrow",
+ "arrow-array",
+ "arrow-schema",
"datafusion-common",
"datafusion-expr",
"log",
@@ -1209,12 +1047,6 @@ dependencies = [
"subtle",
]
-[[package]]
-name = "doc-comment"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
-
[[package]]
name = "either"
version = "1.13.0"
@@ -1334,7 +1166,7 @@ checksum =
"87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -1421,12 +1253,6 @@ dependencies = [
"allocator-api2",
]
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
[[package]]
name = "heck"
version = "0.5.0"
@@ -1529,15 +1355,6 @@ version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "16b73f1c685cfd8ff8d75698ed87e6188cd09944b30c0863d45c2c3699d1da0c"
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
[[package]]
name = "itertools"
version = "0.13.0"
@@ -1577,41 +1394,17 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
-[[package]]
-name = "lexical-core"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46"
-dependencies = [
- "lexical-parse-float 0.8.5",
- "lexical-parse-integer 0.8.6",
- "lexical-util 0.8.5",
- "lexical-write-float 0.8.5",
- "lexical-write-integer 0.8.5",
-]
-
[[package]]
name = "lexical-core"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458"
dependencies = [
- "lexical-parse-float 1.0.2",
- "lexical-parse-integer 1.0.2",
- "lexical-util 1.0.3",
- "lexical-write-float 1.0.2",
- "lexical-write-integer 1.0.2",
-]
-
-[[package]]
-name = "lexical-parse-float"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f"
-dependencies = [
- "lexical-parse-integer 0.8.6",
- "lexical-util 0.8.5",
- "static_assertions",
+ "lexical-parse-float",
+ "lexical-parse-integer",
+ "lexical-util",
+ "lexical-write-float",
+ "lexical-write-integer",
]
[[package]]
@@ -1620,18 +1413,8 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0"
dependencies = [
- "lexical-parse-integer 1.0.2",
- "lexical-util 1.0.3",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-parse-integer"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9"
-dependencies = [
- "lexical-util 0.8.5",
+ "lexical-parse-integer",
+ "lexical-util",
"static_assertions",
]
@@ -1641,16 +1424,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61"
dependencies = [
- "lexical-util 1.0.3",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-util"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc"
-dependencies = [
+ "lexical-util",
"static_assertions",
]
@@ -1663,35 +1437,14 @@ dependencies = [
"static_assertions",
]
-[[package]]
-name = "lexical-write-float"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862"
-dependencies = [
- "lexical-util 0.8.5",
- "lexical-write-integer 0.8.5",
- "static_assertions",
-]
-
[[package]]
name = "lexical-write-float"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809"
dependencies = [
- "lexical-util 1.0.3",
- "lexical-write-integer 1.0.2",
- "static_assertions",
-]
-
-[[package]]
-name = "lexical-write-integer"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446"
-dependencies = [
- "lexical-util 0.8.5",
+ "lexical-util",
+ "lexical-write-integer",
"static_assertions",
]
@@ -1701,7 +1454,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162"
dependencies = [
- "lexical-util 1.0.3",
+ "lexical-util",
"static_assertions",
]
@@ -1784,7 +1537,7 @@ name = "metric_engine"
version = "2.0.0"
dependencies = [
"anyhow",
- "arrow 53.0.0",
+ "arrow",
"async-trait",
"bytes",
"datafusion",
@@ -1792,8 +1545,8 @@ dependencies = [
"itertools 0.3.25",
"lazy_static",
"macros",
- "object_store 0.11.0",
- "parquet 53.1.0",
+ "object_store",
+ "parquet",
"pb_types",
"prost",
"thiserror",
@@ -1930,27 +1683,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "object_store"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3"
-dependencies = [
- "async-trait",
- "bytes",
- "chrono",
- "futures",
- "humantime",
- "itertools 0.13.0",
- "parking_lot",
- "percent-encoding",
- "snafu 0.7.5",
- "tokio",
- "tracing",
- "url",
- "walkdir",
-]
-
[[package]]
name = "object_store"
version = "0.11.0"
@@ -1965,7 +1697,7 @@ dependencies = [
"itertools 0.13.0",
"parking_lot",
"percent-encoding",
- "snafu 0.8.4",
+ "snafu",
"tokio",
"tracing",
"url",
@@ -2016,42 +1748,6 @@ dependencies = [
"windows-targets",
]
-[[package]]
-name = "parquet"
-version = "52.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd"
-dependencies = [
- "ahash",
- "arrow-array 52.2.0",
- "arrow-buffer 52.2.0",
- "arrow-cast 52.2.0",
- "arrow-data 52.2.0",
- "arrow-ipc 52.2.0",
- "arrow-schema 52.2.0",
- "arrow-select 52.2.0",
- "base64",
- "brotli",
- "bytes",
- "chrono",
- "flate2",
- "futures",
- "half",
- "hashbrown",
- "lz4_flex",
- "num",
- "num-bigint",
- "object_store 0.10.2",
- "paste",
- "seq-macro",
- "snap",
- "thrift",
- "tokio",
- "twox-hash",
- "zstd",
- "zstd-sys",
-]
-
[[package]]
name = "parquet"
version = "53.1.0"
@@ -2059,13 +1755,13 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e"
dependencies = [
"ahash",
- "arrow-array 53.1.0",
- "arrow-buffer 53.1.0",
- "arrow-cast 53.1.0",
- "arrow-data 53.1.0",
- "arrow-ipc 53.1.0",
- "arrow-schema 53.1.0",
- "arrow-select 53.1.0",
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-ipc",
+ "arrow-schema",
+ "arrow-select",
"base64",
"brotli",
"bytes",
@@ -2077,7 +1773,7 @@ dependencies = [
"lz4_flex",
"num",
"num-bigint",
- "object_store 0.11.0",
+ "object_store",
"paste",
"seq-macro",
"snap",
@@ -2199,7 +1895,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d"
dependencies = [
"proc-macro2",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2228,7 +1924,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15"
dependencies = [
"bytes",
- "heck 0.5.0",
+ "heck",
"itertools 0.13.0",
"log",
"multimap",
@@ -2238,7 +1934,7 @@ dependencies = [
"prost",
"prost-types",
"regex",
- "syn 2.0.82",
+ "syn",
"tempfile",
]
@@ -2252,7 +1948,7 @@ dependencies = [
"itertools 0.13.0",
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2425,7 +2121,7 @@ checksum =
"243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2496,35 +2192,13 @@ version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
-[[package]]
-name = "snafu"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
-dependencies = [
- "doc-comment",
- "snafu-derive 0.7.5",
-]
-
[[package]]
name = "snafu"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d"
dependencies = [
- "snafu-derive 0.8.4",
-]
-
-[[package]]
-name = "snafu-derive"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
+ "snafu-derive",
]
[[package]]
@@ -2533,10 +2207,10 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2557,9 +2231,9 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.49.0"
+version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e"
+checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac"
dependencies = [
"log",
"sqlparser_derive",
@@ -2573,7 +2247,7 @@ checksum =
"01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2597,11 +2271,11 @@ version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2610,17 +2284,6 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
[[package]]
name = "syn"
version = "2.0.82"
@@ -2662,7 +2325,7 @@ checksum =
"a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2736,7 +2399,7 @@ checksum =
"693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2771,7 +2434,7 @@ checksum =
"34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
@@ -2928,7 +2591,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
"wasm-bindgen-shared",
]
@@ -2950,7 +2613,7 @@ checksum =
"afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -3120,7 +2783,7 @@ checksum =
"fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.82",
+ "syn",
]
[[package]]
diff --git a/horaedb/Cargo.toml b/horaedb/Cargo.toml
index ee231cbe..628e47a1 100644
--- a/horaedb/Cargo.toml
+++ b/horaedb/Cargo.toml
@@ -30,7 +30,7 @@ anyhow = { version = "1.0" }
metric_engine = { path = "metric_engine" }
thiserror = "1"
bytes = "1"
-datafusion = "41"
+datafusion = "42"
parquet = { version = "53" }
object_store = { version = "0.11" }
macros = { path = "../src/components/macros" }
diff --git a/horaedb/Makefile b/horaedb/Makefile
index 72736b0a..189b1c2b 100644
--- a/horaedb/Makefile
+++ b/horaedb/Makefile
@@ -30,3 +30,6 @@ fmt:
fix:
cargo fmt
cargo sort --workspace
+
+test:
+ cargo test --workspace
diff --git a/horaedb/metric_engine/src/storage.rs
b/horaedb/metric_engine/src/storage.rs
index 4c5b2667..3a8d42f0 100644
--- a/horaedb/metric_engine/src/storage.rs
+++ b/horaedb/metric_engine/src/storage.rs
@@ -15,13 +15,25 @@
// specific language governing permissions and limitations
// under the License.
+use std::{sync::Arc, vec};
+
use anyhow::Context;
use arrow::{
array::{Int64Array, RecordBatch},
datatypes::SchemaRef,
};
use async_trait::async_trait;
-use datafusion::logical_expr::Expr;
+use datafusion::{
+ common::DFSchema,
+ execution::{
+ context::ExecutionProps, SendableRecordBatchStream as
DFSendableRecordBatchStream,
+ },
+ logical_expr::Expr,
+ physical_plan::{execute_stream, memory::MemoryExec, sorts::sort::SortExec},
+ physical_planner::create_physical_sort_exprs,
+ prelude::{ident, SessionContext},
+};
+use futures::StreamExt;
use macros::ensure;
use object_store::path::Path;
use parquet::{
@@ -69,6 +81,7 @@ pub struct CloudObjectStorage {
path: String,
store: ObjectStoreRef,
arrow_schema: SchemaRef,
+ num_primary_key: usize,
timestamp_index: usize,
manifest: Manifest,
}
@@ -88,6 +101,7 @@ impl CloudObjectStorage {
root_path: String,
store: ObjectStoreRef,
arrow_schema: SchemaRef,
+ num_primary_key: usize,
timestamp_index: usize,
) -> Result<Self> {
let manifest_prefix = crate::manifest::PREFIX_PATH;
@@ -95,6 +109,7 @@ impl CloudObjectStorage {
Manifest::try_new(format!("{root_path}/{manifest_prefix}"),
store.clone()).await?;
Ok(Self {
path: root_path,
+ num_primary_key,
timestamp_index,
store,
arrow_schema,
@@ -117,15 +132,39 @@ impl CloudObjectStorage {
AsyncArrowWriter::try_new(object_store_writer,
self.schema().clone(), req.props)
.context("create arrow writer")?;
- // TODO: sort record batch according to primary key columns.
- writer
- .write(&req.batch)
- .await
- .context("write arrow batch")?;
+ // sort record batch
+ let mut batches = self.sort_batch(req.batch).await?;
+ while let Some(batch) = batches.next().await {
+ let batch = batch.context("get sorted batch")?;
+ writer.write(&batch).await.context("write arrow batch")?;
+ }
writer.close().await.context("close arrow writer")?;
Ok(file_id)
}
+
+ async fn sort_batch(&self, batch: RecordBatch) ->
Result<DFSendableRecordBatchStream> {
+ let ctx = SessionContext::default();
+ let schema = batch.schema();
+ let df_schema = DFSchema::try_from(schema.clone()).context("build
DFSchema")?;
+
+ let sort_exprs = (0..self.num_primary_key)
+ .collect::<Vec<_>>()
+ .iter()
+ .map(|i| ident(schema.clone().field(*i).name()).sort(true, true))
+ .collect::<Vec<_>>();
+ let physical_sort_exprs =
+ create_physical_sort_exprs(&sort_exprs, &df_schema,
&ExecutionProps::default())
+ .context("create physical sort exprs")?;
+
+ let batch_plan =
+ MemoryExec::try_new(&[vec![batch]], schema, None).context("build
batch plan")?;
+ let physical_plan = Arc::new(SortExec::new(physical_sort_exprs,
Arc::new(batch_plan)));
+
+ let res =
+ execute_stream(physical_plan, ctx.task_ctx()).context("execute
sort physical plan")?;
+ Ok(res)
+ }
}
#[async_trait]
@@ -174,3 +213,62 @@ impl TimeMergeStorage for CloudObjectStorage {
todo!()
}
}
+
+#[cfg(test)]
+mod tests {
+ use arrow::{
+ array::UInt8Array,
+ datatypes::{DataType, Field, Schema},
+ };
+ use object_store::local::LocalFileSystem;
+
+ use super::*;
+
+ #[tokio::test]
+ async fn test_sort_batch() {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::UInt8, false),
+ Field::new("b", DataType::UInt8, false),
+ Field::new("c", DataType::UInt8, false),
+ Field::new("d", DataType::UInt8, false),
+ ]));
+
+ let store = Arc::new(LocalFileSystem::new());
+ let storage =
+ CloudObjectStorage::try_new("/tmp/storage".to_string(), store,
schema.clone(), 1, 1)
+ .await
+ .unwrap();
+
+ let batch = RecordBatch::try_new(
+ schema.clone(),
+ vec![
+ Arc::new(UInt8Array::from(vec![2, 1, 3, 4, 8, 6, 5, 7])),
+ Arc::new(UInt8Array::from(vec![1, 3, 4, 8, 2, 6, 5, 7])),
+ Arc::new(UInt8Array::from(vec![8, 6, 2, 4, 3, 1, 5, 7])),
+ Arc::new(UInt8Array::from(vec![2, 7, 4, 6, 1, 3, 5, 8])),
+ ],
+ )
+ .unwrap();
+
+ let mut sorted_batches = storage.sort_batch(batch).await.unwrap();
+ let expected_bacth = RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(UInt8Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8])),
+ Arc::new(UInt8Array::from(vec![3, 1, 4, 8, 5, 6, 7, 2])),
+ Arc::new(UInt8Array::from(vec![6, 8, 2, 4, 5, 1, 7, 3])),
+ Arc::new(UInt8Array::from(vec![7, 2, 4, 6, 5, 3, 8, 1])),
+ ],
+ )
+ .unwrap();
+
+ let mut offset = 0;
+ while let Some(sorted_batch) = sorted_batches.next().await {
+ let sorted_batch = sorted_batch.unwrap();
+ let length = sorted_batch.num_rows();
+ let batch = expected_bacth.slice(offset, length);
+ assert!(sorted_batch.eq(&batch));
+ offset += length;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
