This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new ba013b4 feat: add index file metadata and handler infrastructure (#78)
ba013b4 is described below
commit ba013b476c5bbc5cb27b338e47bb02e962828f59
Author: lszskye <[email protected]>
AuthorDate: Wed Jun 17 19:16:57 2026 -0700
feat: add index file metadata and handler infrastructure (#78)
---
src/paimon/core/index/deletion_vector_meta.h | 97 +++++++
.../core/index/deletion_vector_meta_test.cpp | 46 +++
src/paimon/core/index/global_index_meta.cpp | 124 ++++++++
src/paimon/core/index/global_index_meta.h | 57 ++++
src/paimon/core/index/index_file.h | 72 +++++
src/paimon/core/index/index_file_handler.cpp | 76 +++++
src/paimon/core/index/index_file_handler.h | 102 +++++++
src/paimon/core/index/index_file_handler_test.cpp | 320 +++++++++++++++++++++
src/paimon/core/index/index_file_meta.h | 183 ++++++++++++
.../core/index/index_file_meta_serializer.cpp | 112 ++++++++
src/paimon/core/index/index_file_meta_serializer.h | 62 ++++
.../core/index/index_file_meta_serializer_test.cpp | 146 ++++++++++
.../core/index/index_file_meta_v1_deserializer.h | 88 ++++++
.../core/index/index_file_meta_v2_deserializer.h | 84 ++++++
.../core/index/index_file_meta_v3_deserializer.h | 88 ++++++
.../index/index_in_data_file_dir_path_factory.h | 70 +++++
.../index_in_data_file_dir_path_factory_test.cpp | 84 ++++++
src/paimon/core/index/index_path_factory.h | 40 +++
18 files changed, 1851 insertions(+)
diff --git a/src/paimon/core/index/deletion_vector_meta.h
b/src/paimon/core/index/deletion_vector_meta.h
new file mode 100644
index 0000000..f951580
--- /dev/null
+++ b/src/paimon/core/index/deletion_vector_meta.h
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
+
+namespace paimon {
+/// Indicates the deletion vector info of member data_file_name, e.g., the
length of dv.
+/// * DeletionVectorMeta is used when serialize to manifest file.
+class DeletionVectorMeta {
+ public:
+ static const std::shared_ptr<arrow::DataType>& DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_(
+ {arrow::field("f0", arrow::utf8(), false), arrow::field("f1",
arrow::int32(), false),
+ arrow::field("f2", arrow::int32(), false),
+ arrow::field("_CARDINALITY", arrow::int64(), true)});
+ return schema;
+ }
+ DeletionVectorMeta(const std::string& data_file_name, int32_t offset,
int32_t length,
+ const std::optional<int64_t>& cardinality)
+ : data_file_name_(data_file_name),
+ offset_(offset),
+ length_(length),
+ cardinality_(cardinality) {}
+
+ bool operator==(const DeletionVectorMeta& other) const {
+ if (this == &other) {
+ return true;
+ }
+ return data_file_name_ == other.data_file_name_ && offset_ ==
other.offset_ &&
+ length_ == other.length_ && cardinality_ == other.cardinality_;
+ }
+
+ bool TEST_Equal(const DeletionVectorMeta& other) const {
+ if (this == &other) {
+ return true;
+ }
+ // ignore data_file_name
+ return offset_ == other.offset_ && length_ == other.length_ &&
+ cardinality_ == other.cardinality_;
+ }
+
+ std::string ToString() const {
+ return fmt::format(
+ "DeletionVectorMeta{{data_file_name = {}, offset = {}, length =
{}, cardinality = {}}}",
+ data_file_name_, offset_, length_,
+ cardinality_ == std::nullopt ? "null" :
std::to_string(cardinality_.value()));
+ }
+
+ const std::string& GetDataFileName() const {
+ return data_file_name_;
+ }
+
+ int32_t GetOffset() const {
+ return offset_;
+ }
+
+ int32_t GetLength() const {
+ return length_;
+ }
+
+ std::optional<int64_t> GetCardinality() const {
+ return cardinality_;
+ }
+
+ private:
+ std::string data_file_name_;
+ int32_t offset_;
+ int32_t length_;
+ std::optional<int64_t> cardinality_;
+};
+} // namespace paimon
diff --git a/src/paimon/core/index/deletion_vector_meta_test.cpp
b/src/paimon/core/index/deletion_vector_meta_test.cpp
new file mode 100644
index 0000000..94230ea
--- /dev/null
+++ b/src/paimon/core/index/deletion_vector_meta_test.cpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/deletion_vector_meta.h"
+
+#include "gtest/gtest.h"
+
+namespace paimon::test {
+TEST(DeletionVectorMetaTest, EqualityOperator) {
+ DeletionVectorMeta meta1("file1", 0, 100, 1000);
+ DeletionVectorMeta meta2("file1", 0, 100, 1000);
+ DeletionVectorMeta meta3("file2", 0, 100, 1000);
+
+ EXPECT_TRUE(meta1 == meta2);
+ EXPECT_FALSE(meta1 == meta3);
+}
+
+TEST(DeletionVectorMetaTest, ToString) {
+ DeletionVectorMeta meta("file1", 0, 100, 1000);
+ std::string expected =
+ "DeletionVectorMeta{data_file_name = file1, offset = 0, length = 100,
cardinality = 1000}";
+ EXPECT_EQ(meta.ToString(), expected);
+
+ DeletionVectorMeta meta_null("file1", 0, 100, std::nullopt);
+ expected =
+ "DeletionVectorMeta{data_file_name = file1, offset = 0, length = 100,
cardinality = null}";
+ EXPECT_EQ(meta_null.ToString(), expected);
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/index/global_index_meta.cpp
b/src/paimon/core/index/global_index_meta.cpp
new file mode 100644
index 0000000..ffe05e3
--- /dev/null
+++ b/src/paimon/core/index/global_index_meta.cpp
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/global_index_meta.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/data/binary_array.h"
+#include "paimon/common/data/binary_row_writer.h"
+
+namespace paimon {
+GlobalIndexMeta::GlobalIndexMeta(int64_t _row_range_start, int64_t
_row_range_end,
+ int32_t _index_field_id,
+ const std::optional<std::vector<int32_t>>&
_extra_field_ids,
+ const std::shared_ptr<Bytes>& _index_meta)
+ : row_range_start(_row_range_start),
+ row_range_end(_row_range_end),
+ index_field_id(_index_field_id),
+ extra_field_ids(_extra_field_ids),
+ index_meta(_index_meta) {}
+
+bool GlobalIndexMeta::operator==(const GlobalIndexMeta& other) const {
+ if (this == &other) {
+ return true;
+ }
+ if ((index_meta && !other.index_meta) || (!index_meta &&
other.index_meta)) {
+ return false;
+ }
+ if (index_meta && other.index_meta && !(*index_meta == *other.index_meta))
{
+ return false;
+ }
+ return row_range_start == other.row_range_start && row_range_end ==
other.row_range_end &&
+ index_field_id == other.index_field_id && extra_field_ids ==
other.extra_field_ids;
+}
+
+std::string GlobalIndexMeta::ToString() const {
+ std::string extra_field_ids_str =
+ extra_field_ids == std::nullopt
+ ? "null"
+ : fmt::format("{}", fmt::join(extra_field_ids.value(), ", "));
+
+ std::string index_meta_str =
+ index_meta == nullptr ? "null" : std::string(index_meta->data(),
index_meta->size());
+ return fmt::format(
+ "{{row_range_start={}, row_range_end={}, index_field_id={},
extra_field_ids={}, "
+ "index_meta={}}}",
+ row_range_start, row_range_end, index_field_id, extra_field_ids_str,
index_meta_str);
+}
+
+BinaryRow GlobalIndexMeta::ToRow(MemoryPool* pool) const {
+ BinaryRow row(5);
+ BinaryRowWriter writer(&row, 32 * 1024, pool);
+ writer.WriteLong(0, row_range_start);
+ writer.WriteLong(1, row_range_end);
+ writer.WriteInt(2, index_field_id);
+ if (!extra_field_ids) {
+ writer.SetNullAt(3);
+ } else {
+ writer.WriteArray(3,
BinaryArray::FromIntArray(extra_field_ids.value(), pool));
+ }
+ if (index_meta == nullptr) {
+ writer.SetNullAt(4);
+ } else {
+ writer.WriteBinary(4, *index_meta);
+ }
+ writer.Complete();
+ return row;
+}
+
+Result<GlobalIndexMeta> GlobalIndexMeta::FromRow(const InternalRow& row) {
+ int64_t row_range_start = row.GetLong(0);
+ int64_t row_range_end = row.GetLong(1);
+ int32_t index_field_id = row.GetInt(2);
+ std::optional<std::vector<int32_t>> extra_field_ids;
+ if (!row.IsNullAt(3)) {
+ std::shared_ptr<InternalArray> array = row.GetArray(3);
+ if (!array) {
+ return Status::Invalid("GlobalIndexMeta FromRow failed with
nullptr extra field ids");
+ }
+ PAIMON_ASSIGN_OR_RAISE(extra_field_ids, array->ToIntArray());
+ }
+ std::shared_ptr<Bytes> index_meta;
+ if (!row.IsNullAt(4)) {
+ index_meta = row.GetBinary(4);
+ assert(index_meta);
+ }
+ return GlobalIndexMeta(row_range_start, row_range_end, index_field_id,
extra_field_ids,
+ index_meta);
+}
+
+const std::shared_ptr<arrow::DataType>& GlobalIndexMeta::DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_({
+ arrow::field("_ROW_RANGE_START", arrow::int64(), /*nullable=*/false),
+ arrow::field("_ROW_RANGE_END", arrow::int64(), /*nullable=*/false),
+ arrow::field("_INDEX_FIELD_ID", arrow::int32(), /*nullable=*/false),
+ arrow::field("_EXTRA_FIELD_IDS",
+ arrow::list(arrow::field("item", arrow::int32(),
/*nullable=*/false)),
+ /*nullable=*/true),
+ arrow::field("_INDEX_META", arrow::binary(), /*nullable=*/true),
+ });
+ return schema;
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/index/global_index_meta.h
b/src/paimon/core/index/global_index_meta.h
new file mode 100644
index 0000000..a11cb82
--- /dev/null
+++ b/src/paimon/core/index/global_index_meta.h
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "arrow/api.h"
+#include "fmt/format.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/memory/bytes.h"
+namespace paimon {
+/// Schema for global index.
+struct GlobalIndexMeta {
+ static constexpr int32_t NUM_FIELDS = 5;
+
+ GlobalIndexMeta(int64_t _row_range_start, int64_t _row_range_end, int32_t
_index_field_id,
+ const std::optional<std::vector<int32_t>>&
_extra_field_ids,
+ const std::shared_ptr<Bytes>& _index_meta);
+
+ bool operator==(const GlobalIndexMeta& other) const;
+
+ std::string ToString() const;
+
+ BinaryRow ToRow(MemoryPool* pool) const;
+
+ static Result<GlobalIndexMeta> FromRow(const InternalRow& row);
+
+ static const std::shared_ptr<arrow::DataType>& DataType();
+
+ int64_t row_range_start;
+ int64_t row_range_end;
+ int32_t index_field_id;
+ std::optional<std::vector<int32_t>> extra_field_ids;
+ std::shared_ptr<Bytes> index_meta;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file.h
b/src/paimon/core/index/index_file.h
new file mode 100644
index 0000000..e76c19d
--- /dev/null
+++ b/src/paimon/core/index/index_file.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "paimon/core/index/index_path_factory.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+// Base index file
+class IndexFile {
+ public:
+ IndexFile(const std::shared_ptr<FileSystem>& fs,
+ const std::shared_ptr<IndexPathFactory>& path_factory)
+ : fs_(fs), path_factory_(path_factory) {}
+ virtual ~IndexFile() = default;
+
+ virtual std::string Path(const std::shared_ptr<IndexFileMeta>& file) const
{
+ return path_factory_->ToPath(file);
+ }
+
+ virtual Result<uint64_t> FileSize(const std::shared_ptr<IndexFileMeta>&
file) const {
+ return FileSize(Path(file));
+ }
+
+ virtual Result<uint64_t> FileSize(const std::string& file) const {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FileStatus> file_status,
fs_->GetFileStatus(file));
+ return file_status->GetLen();
+ }
+
+ virtual void Delete(const std::shared_ptr<IndexFileMeta>& file) const {
+ // Deletion is best-effort
+ auto status = fs_->Delete(Path(file), /*recursive=*/false);
+ (void)status;
+ }
+
+ virtual Result<bool> Exists(const std::shared_ptr<IndexFileMeta>& file)
const {
+ return fs_->Exists(Path(file));
+ }
+
+ virtual bool IsExternalPath() const {
+ return path_factory_->IsExternalPath();
+ }
+
+ protected:
+ std::shared_ptr<FileSystem> fs_;
+ std::shared_ptr<IndexPathFactory> path_factory_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_handler.cpp
b/src/paimon/core/index/index_file_handler.cpp
new file mode 100644
index 0000000..9b45e74
--- /dev/null
+++ b/src/paimon/core/index/index_file_handler.cpp
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/index_file_handler.h"
+
+#include <functional>
+#include <optional>
+
+#include "paimon/core/snapshot.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+Result<IndexFileHandler::IndexFileMetaGroups> IndexFileHandler::Scan(
+ const Snapshot& snapshot, const std::string& index_type,
+ const std::unordered_set<BinaryRow>& partitions) const {
+ IndexFileHandler::IndexFileMetaGroups result;
+ std::function<Result<bool>(const IndexManifestEntry&)> filter =
+ [&](const IndexManifestEntry& entry) -> bool {
+ if (entry.index_file->IndexType() == index_type &&
+ partitions.find(entry.partition) != partitions.end()) {
+ return true;
+ }
+ return false;
+ };
+
+ PAIMON_ASSIGN_OR_RAISE(std::vector<IndexManifestEntry> index_entries,
Scan(snapshot, filter));
+ for (const auto& entry : index_entries) {
+ std::pair<BinaryRow, int32_t> key(entry.partition, entry.bucket);
+ result[key].push_back(entry.index_file);
+ }
+ return result;
+}
+
+Result<std::vector<IndexManifestEntry>> IndexFileHandler::Scan(
+ const Snapshot& snapshot, std::function<Result<bool>(const
IndexManifestEntry&)> filter) const {
+ const std::optional<std::string>& index_manifest =
snapshot.IndexManifest();
+ if (index_manifest == std::nullopt) {
+ return std::vector<IndexManifestEntry>();
+ }
+ std::vector<IndexManifestEntry> index_entries;
+ PAIMON_RETURN_NOT_OK(
+ index_manifest_file_->Read(index_manifest.value(), filter,
&index_entries));
+ return index_entries;
+}
+
+Result<std::vector<std::shared_ptr<IndexFileMeta>>> IndexFileHandler::Scan(
+ const Snapshot& snapshot, const std::string& index_type, const BinaryRow&
partition,
+ int32_t bucket) const {
+ PAIMON_ASSIGN_OR_RAISE(IndexFileHandler::IndexFileMetaGroups
index_file_meta_groups,
+ Scan(snapshot, index_type, {partition}));
+ std::pair<BinaryRow, int32_t> key(partition, bucket);
+ auto iter = index_file_meta_groups.find(key);
+ if (iter != index_file_meta_groups.end()) {
+ return iter->second;
+ }
+ return std::vector<std::shared_ptr<IndexFileMeta>>{};
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_handler.h
b/src/paimon/core/index/index_file_handler.h
new file mode 100644
index 0000000..a84840a
--- /dev/null
+++ b/src/paimon/core/index/index_file_handler.h
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/index/index_path_factory.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class Snapshot;
+
+class IndexFileHandler {
+ public:
+ using IndexFileMetaGroups = std::unordered_map<std::pair<BinaryRow,
int32_t>,
+
std::vector<std::shared_ptr<IndexFileMeta>>>;
+
+ IndexFileHandler(const std::shared_ptr<FileSystem>& fs,
+ std::unique_ptr<IndexManifestFile>&& index_manifest_file,
+ const std::shared_ptr<IndexFilePathFactories>&
path_factories,
+ bool dv_bitmap64, const std::shared_ptr<MemoryPool>& pool)
+ : fs_(fs),
+ index_manifest_file_(std::move(index_manifest_file)),
+ path_factories_(path_factories),
+ dv_bitmap64_(dv_bitmap64),
+ pool_(pool) {}
+
+ /// 1.Scan specified index_type index. 2.Cluster with partition & bucket.
+ Result<IndexFileMetaGroups> Scan(const Snapshot& snapshot, const
std::string& index_type,
+ const std::unordered_set<BinaryRow>&
partitions) const;
+
+ Result<std::vector<std::shared_ptr<IndexFileMeta>>> Scan(const Snapshot&
snapshot,
+ const
std::string& index_type,
+ const BinaryRow&
partition,
+ int32_t bucket)
const;
+
+ /// Scan specified all typed index.
+ Result<std::vector<IndexManifestEntry>> Scan(
+ const Snapshot& snapshot,
+ std::function<Result<bool>(const IndexManifestEntry&)> filter) const;
+
+ Result<std::string> FilePath(const BinaryRow& partition, int32_t bucket,
+ const std::shared_ptr<IndexFileMeta>& file)
const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<IndexPathFactory> factory,
+ path_factories_->Get(partition, bucket));
+ return factory->ToPath(file);
+ }
+
+ Result<std::unique_ptr<DeletionVectorsIndexFile>> DvIndex(const BinaryRow&
partition,
+ int32_t bucket)
const {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<IndexPathFactory>
index_path_factory,
+ path_factories_->Get(partition, bucket));
+ return std::make_unique<DeletionVectorsIndexFile>(fs_,
index_path_factory, dv_bitmap64_,
+ pool_);
+ }
+
+ Result<std::map<std::string, std::shared_ptr<DeletionVector>>>
ReadAllDeletionVectors(
+ const BinaryRow& partition, int32_t bucket,
+ const std::vector<std::shared_ptr<IndexFileMeta>>& file_metas) const {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DeletionVectorsIndexFile>
dv_index,
+ DvIndex(partition, bucket));
+ return dv_index->ReadAllDeletionVectors(file_metas);
+ }
+
+ private:
+ std::shared_ptr<FileSystem> fs_;
+ std::unique_ptr<IndexManifestFile> index_manifest_file_;
+ std::shared_ptr<IndexFilePathFactories> path_factories_;
+ bool dv_bitmap64_;
+ std::shared_ptr<MemoryPool> pool_;
+};
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_handler_test.cpp
b/src/paimon/core/index/index_file_handler_test.cpp
new file mode 100644
index 0000000..8236d18
--- /dev/null
+++ b/src/paimon/core/index/index_file_handler_test.cpp
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/index_file_handler.h"
+
+#include <map>
+#include <optional>
+#include <variant>
+
+#include "gtest/gtest.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/common/utils/object_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/defs.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/binary_row_generator.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+class IndexFileHandlerTest : public testing::Test {
+ public:
+ void SetUp() override {
+ memory_pool_ = GetDefaultPool();
+ }
+
+ Result<std::unique_ptr<IndexFileHandler>> CreateIndexFileHandler(
+ const std::string& table_path, const CoreOptions& core_options) const {
+ SchemaManager schema_manager(core_options.GetFileSystem(), table_path);
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> table_schema,
+ schema_manager.ReadSchema(/*schema_id=*/0));
+ auto schema =
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
+ core_options.CreateExternalPaths());
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::string>
global_index_external_path,
+ core_options.CreateGlobalIndexExternalPath());
+
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileStorePathFactory> path_factory,
+ FileStorePathFactory::Create(
+ table_path, schema, table_schema->PartitionKeys(),
+ core_options.GetPartitionDefaultName(),
+ /*identifier=*/"orc", core_options.DataFilePrefix(),
+ core_options.LegacyPartitionNameEnabled(), external_paths,
+ global_index_external_path,
+
/*index_file_in_data_file_dir=*/core_options.IndexFileInDataFileDir(),
+ memory_pool_));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<IndexManifestFile>
index_manifest_file,
+ IndexManifestFile::Create(
+ core_options.GetFileSystem(),
core_options.GetManifestFormat(),
+ core_options.GetManifestCompression(),
path_factory,
+ core_options.GetBucket(), memory_pool_,
core_options));
+ auto path_factories =
std::make_shared<IndexFilePathFactories>(path_factory);
+ return std::make_unique<IndexFileHandler>(
+ core_options.GetFileSystem(), std::move(index_manifest_file),
path_factories,
+ core_options.DeletionVectorsBitmap64(), memory_pool_);
+ }
+ std::shared_ptr<MemoryPool> memory_pool_;
+};
+
+TEST_F(IndexFileHandlerTest, TestFilePath) {
+ std::string table_path = paimon::test::GetDataDir() +
+
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
+ {
+ // test without external path & index-file-in-data-file-dir" = false
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
CoreOptions::FromMap({}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler>
index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+ auto index_file_meta = std::make_shared<IndexFileMeta>(
+ /*index_type=*/"DELETION_VECTOR", /*file_name=*/"deletion-file",
/*file_size=*/1000,
+ /*row_count=*/100, /*dv_ranges=*/std::nullopt,
/*external_path=*/std::nullopt);
+ auto partition = BinaryRowGenerator::GenerateRow({10},
memory_pool_.get());
+ ASSERT_OK_AND_ASSIGN(std::string file_path,
index_file_handler->FilePath(
+ partition,
/*bucket=*/1, index_file_meta));
+ ASSERT_EQ(file_path, table_path + "index/deletion-file");
+ }
+ {
+ // test with external path & index-file-in-data-file-dir" = false
+ ASSERT_OK_AND_ASSIGN(
+ CoreOptions core_options,
+ CoreOptions::FromMap({{Options::DATA_FILE_EXTERNAL_PATHS,
"FILE:///tmp/"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler>
index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+ auto index_file_meta = std::make_shared<IndexFileMeta>(
+ /*index_type=*/"DELETION_VECTOR", /*file_name=*/"deletion-file",
/*file_size=*/1000,
+ /*row_count=*/100, /*dv_ranges=*/std::nullopt,
/*external_path=*/std::nullopt);
+ auto partition = BinaryRowGenerator::GenerateRow({10},
memory_pool_.get());
+ ASSERT_OK_AND_ASSIGN(std::string file_path,
index_file_handler->FilePath(
+ partition,
/*bucket=*/1, index_file_meta));
+ ASSERT_EQ(file_path, table_path + "index/deletion-file");
+ }
+ {
+ // test without external path & index-file-in-data-file-dir" = true
+ ASSERT_OK_AND_ASSIGN(
+ CoreOptions core_options,
+ CoreOptions::FromMap({{Options::INDEX_FILE_IN_DATA_FILE_DIR,
"true"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler>
index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+ auto index_file_meta = std::make_shared<IndexFileMeta>(
+ /*index_type=*/"DELETION_VECTOR", /*file_name=*/"deletion-file",
/*file_size=*/1000,
+ /*row_count=*/100, /*dv_ranges=*/std::nullopt,
/*external_path=*/std::nullopt);
+ auto partition = BinaryRowGenerator::GenerateRow({10},
memory_pool_.get());
+ ASSERT_OK_AND_ASSIGN(std::string file_path,
index_file_handler->FilePath(
+ partition,
/*bucket=*/1, index_file_meta));
+ ASSERT_EQ(file_path, table_path + "f1=10/bucket-1/deletion-file");
+ }
+ {
+ // test with external path & index-file-in-data-file-dir" = true
+ ASSERT_OK_AND_ASSIGN(
+ CoreOptions core_options,
+ CoreOptions::FromMap({{Options::INDEX_FILE_IN_DATA_FILE_DIR,
"true"},
+ {Options::DATA_FILE_EXTERNAL_PATHS,
"FILE:///tmp/"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler>
index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+ auto index_file_meta = std::make_shared<IndexFileMeta>(
+ /*index_type=*/"DELETION_VECTOR", /*file_name=*/"deletion-file",
/*file_size=*/1000,
+ /*row_count=*/100, /*dv_ranges=*/std::nullopt,
+ /*external_path=*/"FILE:///tmp/f1=10/bucket-1/deletion-file");
+ auto partition = BinaryRowGenerator::GenerateRow({10},
memory_pool_.get());
+ ASSERT_OK_AND_ASSIGN(std::string file_path,
index_file_handler->FilePath(
+ partition,
/*bucket=*/1, index_file_meta));
+ ASSERT_EQ(file_path, "FILE:///tmp/f1=10/bucket-1/deletion-file");
+ }
+}
+TEST_F(IndexFileHandlerTest, TestScan) {
+ std::string table_path = paimon::test::GetDataDir() +
+
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
+
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::MANIFEST_FORMAT,
"orc"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+
+ SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(Snapshot snapshot,
snapshot_manager.LoadSnapshot(/*snapshot_id=*/4));
+
+ auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get());
+ std::unordered_set<BinaryRow> partitions = {partition};
+ ASSERT_OK_AND_ASSIGN(
+ auto index_file_metas,
+ index_file_handler->Scan(
+ snapshot,
std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX), partitions));
+ ASSERT_EQ(2, index_file_metas.size());
+
+ // check index metas equal
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_meta_p10_b0;
+ dv_meta_p10_b0.insert_or_assign(
+ "data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-0.orc",
+ DeletionVectorMeta("data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-0.orc",
/*offset=*/1,
+ /*length=*/22, /*cardinality=*/1));
+ std::vector<std::shared_ptr<IndexFileMeta>> index_meta_p10_b0 = {
+ std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "index-86356766-3238-46e6-990b-656cd7409eaa-0",
+ /*file_size=*/31, /*row_count=*/1, dv_meta_p10_b0,
/*external_path=*/std::nullopt)};
+
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_meta_p10_b1;
+ dv_meta_p10_b1.insert_or_assign(
+ "data-2ffe7ae9-2cf7-41e9-944b-2065585cde31-0.orc",
+ DeletionVectorMeta("data-2ffe7ae9-2cf7-41e9-944b-2065585cde31-0.orc",
/*offset=*/1,
+ /*length=*/24, /*cardinality=*/2));
+ std::vector<std::shared_ptr<IndexFileMeta>> index_meta_p10_b1 = {
+ std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "index-86356766-3238-46e6-990b-656cd7409eaa-1",
+ /*file_size=*/33, /*row_count=*/1, dv_meta_p10_b1,
/*external_path=*/std::nullopt)};
+ ASSERT_TRUE(
+ ObjectUtils::Equal(index_file_metas[std::make_pair(partition, 0)],
index_meta_p10_b0));
+ ASSERT_TRUE(
+ ObjectUtils::Equal(index_file_metas[std::make_pair(partition, 1)],
index_meta_p10_b1));
+
+ // test FilePath
+ ASSERT_OK_AND_ASSIGN(auto index_file_path,
index_file_handler->FilePath(partition, /*bucket=*/0,
+
index_meta_p10_b0[0]));
+ ASSERT_EQ(index_file_path,
+ PathUtil::JoinPath(table_path, "/index/" +
index_meta_p10_b0[0]->FileName()));
+}
+
+TEST_F(IndexFileHandlerTest, Test09VersionScan) {
+ std::string table_path = paimon::test::GetDataDir() +
"/orc/pk_09.db/pk_09/";
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::MANIFEST_FORMAT,
"orc"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+
+ SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(Snapshot snapshot,
snapshot_manager.LoadSnapshot(/*snapshot_id=*/6));
+
+ auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get());
+ std::unordered_set<BinaryRow> partitions = {partition};
+ ASSERT_OK_AND_ASSIGN(
+ auto index_file_metas,
+ index_file_handler->Scan(
+ snapshot,
std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX), partitions));
+
+ ASSERT_EQ(2, index_file_metas.size());
+
+ // check index metas equal
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_meta_p10_b0;
+ dv_meta_p10_b0.insert_or_assign(
+ "data-1c7a85f1-55bd-424f-b503-34a33be0fb96-0.orc",
+ DeletionVectorMeta("data-1c7a85f1-55bd-424f-b503-34a33be0fb96-0.orc",
/*offset=*/1,
+ /*length=*/22, /*cardinality=*/std::nullopt));
+ dv_meta_p10_b0.insert_or_assign(
+ "data-980e82b4-2345-4976-bc1d-ea989fcdbffa-0.orc",
+ DeletionVectorMeta("data-980e82b4-2345-4976-bc1d-ea989fcdbffa-0.orc",
/*offset=*/31,
+ /*length=*/22, /*cardinality=*/std::nullopt));
+ std::vector<std::shared_ptr<IndexFileMeta>> index_meta_p10_b0 = {
+ std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "index-7badd250-6c0b-49e9-8e40-2449ae9a2539-0",
+ /*file_size=*/61, /*row_count=*/2, dv_meta_p10_b0,
/*external_path=*/std::nullopt)};
+
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_meta_p10_b1;
+ dv_meta_p10_b1.insert_or_assign(
+ "data-6871b960-edd9-40fc-9859-aaca9ea205cf-0.orc",
+ DeletionVectorMeta("data-6871b960-edd9-40fc-9859-aaca9ea205cf-0.orc",
/*offset=*/1,
+ /*length=*/22, /*cardinality=*/std::nullopt));
+ std::vector<std::shared_ptr<IndexFileMeta>> index_meta_p10_b1 = {
+ std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "index-7badd250-6c0b-49e9-8e40-2449ae9a2539-1",
+ /*file_size=*/31, /*row_count=*/1, dv_meta_p10_b1,
/*external_path=*/std::nullopt)};
+ ASSERT_TRUE(
+ ObjectUtils::Equal(index_file_metas[std::make_pair(partition, 0)],
index_meta_p10_b0));
+ ASSERT_TRUE(
+ ObjectUtils::Equal(index_file_metas[std::make_pair(partition, 1)],
index_meta_p10_b1));
+}
+
+TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) {
+ std::string table_path = paimon::test::GetDataDir() +
"/orc/pk_09.db/pk_09/";
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::MANIFEST_FORMAT,
"orc"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+
+ SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(Snapshot snapshot,
snapshot_manager.LoadSnapshot(/*snapshot_id=*/6));
+
+ Snapshot no_index_manifest_snapshot(
+ std::optional<int32_t>(snapshot.Version()), snapshot.Id(),
snapshot.SchemaId(),
+ snapshot.BaseManifestList(), snapshot.BaseManifestListSize(),
snapshot.DeltaManifestList(),
+ snapshot.DeltaManifestListSize(), snapshot.ChangelogManifestList(),
+ snapshot.ChangelogManifestListSize(), /*index_manifest=*/std::nullopt,
+ snapshot.CommitUser(), snapshot.CommitIdentifier(),
snapshot.GetCommitKind(),
+ snapshot.TimeMillis(), snapshot.LogOffsets(),
snapshot.TotalRecordCount(),
+ snapshot.DeltaRecordCount(), snapshot.ChangelogRecordCount(),
snapshot.Watermark(),
+ snapshot.Statistics(), snapshot.Properties(), snapshot.NextRowId());
+
+ auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get());
+ std::unordered_set<BinaryRow> partitions = {partition};
+ ASSERT_OK_AND_ASSIGN(
+ auto index_file_metas,
+ index_file_handler->Scan(no_index_manifest_snapshot,
+
std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ partitions));
+ ASSERT_TRUE(index_file_metas.empty());
+
+ ASSERT_OK_AND_ASSIGN(
+ auto index_entries,
+ index_file_handler->Scan(no_index_manifest_snapshot,
+ [](const IndexManifestEntry&) -> Result<bool>
{ return true; }));
+ ASSERT_TRUE(index_entries.empty());
+}
+
+TEST_F(IndexFileHandlerTest,
TestScanByPartitionBucketAndReadAllDeletionVectors) {
+ std::string table_path = paimon::test::GetDataDir() +
+
"/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/";
+
+ ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+ CoreOptions::FromMap({{Options::MANIFEST_FORMAT,
"orc"}}));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<IndexFileHandler> index_file_handler,
+ CreateIndexFileHandler(table_path, core_options));
+
+ SnapshotManager snapshot_manager(core_options.GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(Snapshot snapshot,
snapshot_manager.LoadSnapshot(/*snapshot_id=*/4));
+
+ auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get());
+ ASSERT_OK_AND_ASSIGN(
+ auto index_file_metas,
+ index_file_handler->Scan(
+ snapshot,
std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX), partition,
+ /*bucket=*/0));
+ ASSERT_EQ(index_file_metas.size(), 1);
+
+ ASSERT_OK_AND_ASSIGN(auto deletion_vectors,
index_file_handler->ReadAllDeletionVectors(
+ partition, /*bucket=*/0,
index_file_metas));
+ ASSERT_EQ(deletion_vectors.size(), 1);
+
ASSERT_TRUE(deletion_vectors.find("data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-0.orc")
!=
+ deletion_vectors.end());
+
ASSERT_EQ(deletion_vectors["data-0d0f29cc-63c6-4fab-a594-71bd7d06fcde-0.orc"]->GetCardinality(),
+ 1);
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/index/index_file_meta.h
b/src/paimon/core/index/index_file_meta.h
new file mode 100644
index 0000000..3d495a3
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta.h
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "arrow/api.h"
+#include "fmt/core.h"
+#include "fmt/format.h"
+#include "fmt/ranges.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/index/global_index_meta.h"
+
+namespace paimon {
+/// Metadata of index file.
+class IndexFileMeta {
+ public:
+ static constexpr int32_t NUM_FIELDS = 7;
+
+ IndexFileMeta(const std::string& index_type, const std::string& file_name,
int64_t file_size,
+ int64_t row_count,
+ const std::optional<LinkedHashMap<std::string,
DeletionVectorMeta>>& dv_ranges,
+ const std::optional<std::string>& external_path)
+ : IndexFileMeta(index_type, file_name, file_size, row_count,
dv_ranges, external_path,
+ /*global_index_meta=*/std::nullopt) {}
+
+ IndexFileMeta(const std::string& index_type, const std::string& file_name,
int64_t file_size,
+ int64_t row_count,
+ const std::optional<LinkedHashMap<std::string,
DeletionVectorMeta>>& dv_ranges,
+ const std::optional<std::string>& external_path,
+ const std::optional<GlobalIndexMeta>& global_index_meta)
+ : index_type_(index_type),
+ file_name_(file_name),
+ file_size_(file_size),
+ row_count_(row_count),
+ dv_ranges_(dv_ranges),
+ external_path_(external_path),
+ global_index_meta_(global_index_meta) {}
+
+ const std::string& IndexType() const {
+ return index_type_;
+ }
+
+ const std::string& FileName() const {
+ return file_name_;
+ }
+
+ int64_t FileSize() const {
+ return file_size_;
+ }
+
+ int64_t RowCount() const {
+ return row_count_;
+ }
+
+ const std::optional<LinkedHashMap<std::string, DeletionVectorMeta>>&
DvRanges() const {
+ return dv_ranges_;
+ }
+
+ const std::optional<std::string>& ExternalPath() const {
+ return external_path_;
+ }
+
+ const std::optional<GlobalIndexMeta>& GetGlobalIndexMeta() const {
+ return global_index_meta_;
+ }
+
+ bool operator==(const IndexFileMeta& other) const {
+ if (this == &other) {
+ return true;
+ }
+ return index_type_ == other.index_type_ && file_name_ ==
other.file_name_ &&
+ file_size_ == other.file_size_ && row_count_ ==
other.row_count_ &&
+ dv_ranges_ == other.dv_ranges_ && external_path_ ==
other.external_path_ &&
+ global_index_meta_ == other.global_index_meta_;
+ }
+
+ bool TEST_Equal(const IndexFileMeta& other) const {
+ if (this == &other) {
+ return true;
+ }
+
+ if ((dv_ranges_ && !other.dv_ranges_) || (!dv_ranges_ &&
other.dv_ranges_)) {
+ return false;
+ }
+ if (dv_ranges_ && other.dv_ranges_) {
+ if (dv_ranges_.value().size() != other.dv_ranges_.value().size()) {
+ return false;
+ }
+ for (auto iter1 = dv_ranges_.value().begin(), iter2 =
other.dv_ranges_.value().begin();
+ iter1 != dv_ranges_.value().end() && iter2 !=
other.dv_ranges_.value().end();
+ ++iter1, ++iter2) {
+ if (!iter1->second.TEST_Equal(iter2->second)) {
+ return false;
+ }
+ }
+ }
+
+ if ((external_path_ && !other.external_path_) ||
+ (!external_path_ && other.external_path_)) {
+ return false;
+ }
+
+ // ignore file_name & file_size
+ return index_type_ == other.index_type_ && row_count_ ==
other.row_count_ &&
+ global_index_meta_ == other.global_index_meta_;
+ }
+
+ std::string ToString() const {
+ std::string dv_str = dv_ranges_ == std::nullopt ? "null" :
Format(dv_ranges_.value());
+ std::string external_path_str =
+ external_path_ == std::nullopt ? "null" : external_path_.value();
+ std::string global_index_meta_str =
+ global_index_meta_ == std::nullopt ? "null" :
global_index_meta_.value().ToString();
+ return fmt::format(
+ "IndexManifestEntry{{indexType={}, fileName={}, fileSize={},
rowCount={}, "
+ "dvRanges={}, externalPath={}, globalIndexMeta={}}}",
+ index_type_, file_name_, file_size_, row_count_, dv_str,
external_path_str,
+ global_index_meta_str);
+ }
+
+ static const std::shared_ptr<arrow::DataType>& DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_({
+ arrow::field("_INDEX_TYPE", arrow::utf8(), false),
+ arrow::field("_FILE_NAME", arrow::utf8(), false),
+ arrow::field("_FILE_SIZE", arrow::int64(), false),
+ arrow::field("_ROW_COUNT", arrow::int64(), false),
+ arrow::field("_DELETIONS_VECTORS_RANGES",
+ arrow::list(arrow::field("item",
DeletionVectorMeta::DataType(), true)),
+ true),
+ arrow::field("_EXTERNAL_PATH", arrow::utf8(), true),
+ arrow::field("_GLOBAL_INDEX", GlobalIndexMeta::DataType(), true),
+ });
+ return schema;
+ }
+
+ private:
+ static std::string Format(const LinkedHashMap<std::string,
DeletionVectorMeta>& val) {
+ std::string result = "{";
+ for (const auto& iter : val) {
+ result.append(fmt::format("{}: {};", iter.first,
iter.second.ToString()));
+ }
+ if (!val.empty()) {
+ result.pop_back();
+ }
+ result.append("}");
+ return result;
+ }
+
+ std::string index_type_;
+ std::string file_name_;
+ int64_t file_size_ = 0;
+ int64_t row_count_ = 0;
+
+ /// Metadata only used by `DeletionVectorsIndexFile`, use LinkedHashMap to
ensure that the
+ /// order of DeletionVectorRanges and the written DeletionVectors is
consistent.
+ std::optional<LinkedHashMap<std::string, DeletionVectorMeta>> dv_ranges_;
+
+ std::optional<std::string> external_path_;
+
+ std::optional<GlobalIndexMeta> global_index_meta_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_meta_serializer.cpp
b/src/paimon/core/index/index_file_meta_serializer.cpp
new file mode 100644
index 0000000..94fe445
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_serializer.cpp
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "paimon/core/index/index_file_meta_serializer.h"
+
+#include "paimon/common/data/binary_array_writer.h"
+#include "paimon/common/data/binary_row_writer.h"
+#include "paimon/core/index/index_file_meta_v2_deserializer.h"
+
+namespace paimon {
+Result<BinaryRow> IndexFileMetaSerializer::ToRow(const
std::shared_ptr<IndexFileMeta>& meta) const {
+ BinaryRow row(IndexFileMeta::NUM_FIELDS);
+ BinaryRowWriter writer(&row, 32 * 1024, pool_.get());
+ WriteIndexFileMeta(/*start_pos=*/0, meta, &writer, pool_.get());
+ writer.Complete();
+ return row;
+}
+
+Result<std::shared_ptr<IndexFileMeta>> IndexFileMetaSerializer::FromRow(
+ const InternalRow& row) const {
+ auto file_type = row.GetString(0);
+ auto file_name = row.GetString(1);
+ auto file_size = row.GetLong(2);
+ auto row_count = row.GetLong(3);
+ std::optional<LinkedHashMap<std::string, DeletionVectorMeta>> dv_ranges;
+ if (!row.IsNullAt(4)) {
+ dv_ranges =
IndexFileMetaV2Deserializer::RowArrayDataToDvRanges(row.GetArray(4).get());
+ }
+ std::optional<std::string> external_path;
+ if (!row.IsNullAt(5)) {
+ external_path = row.GetString(5).ToString();
+ }
+ std::optional<GlobalIndexMeta> global_index_meta;
+ if (!row.IsNullAt(6)) {
+ std::shared_ptr<InternalRow> global_index_meta_row =
+ row.GetRow(6, GlobalIndexMeta::NUM_FIELDS);
+ assert(global_index_meta_row);
+ PAIMON_ASSIGN_OR_RAISE(global_index_meta,
GlobalIndexMeta::FromRow(*global_index_meta_row));
+ }
+
+ return std::make_shared<IndexFileMeta>(file_type.ToString(),
file_name.ToString(), file_size,
+ row_count, dv_ranges,
external_path, global_index_meta);
+}
+
+void IndexFileMetaSerializer::WriteIndexFileMeta(int32_t start_pos,
+ const
std::shared_ptr<IndexFileMeta>& meta,
+ BinaryRowWriter* writer,
MemoryPool* pool) {
+ writer->WriteString(start_pos + 0,
BinaryString::FromString(meta->IndexType(), pool));
+ writer->WriteString(start_pos + 1,
BinaryString::FromString(meta->FileName(), pool));
+ writer->WriteLong(start_pos + 2, meta->FileSize());
+ writer->WriteLong(start_pos + 3, meta->RowCount());
+ const auto& dv_ranges = meta->DvRanges();
+ if (dv_ranges == std::nullopt) {
+ writer->SetNullAt(start_pos + 4);
+ } else {
+ auto array = DvRangesToRowArrayData(dv_ranges.value(), pool);
+ writer->WriteArray(start_pos + 4, array);
+ }
+ auto external_path = meta->ExternalPath();
+ if (external_path == std::nullopt) {
+ writer->SetNullAt(start_pos + 5);
+ } else {
+ writer->WriteString(start_pos + 5,
BinaryString::FromString(external_path.value(), pool));
+ }
+ auto global_index_meta = meta->GetGlobalIndexMeta();
+ if (global_index_meta == std::nullopt) {
+ writer->SetNullAt(start_pos + 6);
+ } else {
+ writer->WriteRow(start_pos + 6, global_index_meta.value().ToRow(pool));
+ }
+}
+
+BinaryArray IndexFileMetaSerializer::DvRangesToRowArrayData(
+ const LinkedHashMap<std::string, DeletionVectorMeta>& dv_metas,
MemoryPool* pool) {
+ BinaryArray array;
+ BinaryArrayWriter array_writer(&array, dv_metas.size(),
/*element_size=*/8, pool);
+ int32_t pos = 0;
+ for (const auto& dv_meta : dv_metas) {
+ const auto& meta = dv_meta.second;
+ BinaryRow dv_data(4);
+ BinaryRowWriter writer(&dv_data, 1024, pool);
+ writer.WriteString(/*pos=*/0,
BinaryString::FromString(meta.GetDataFileName(), pool));
+ writer.WriteInt(/*pos=*/1, meta.GetOffset());
+ writer.WriteInt(/*pos=*/2, meta.GetLength());
+ if (meta.GetCardinality()) {
+ writer.WriteLong(/*pos=*/3, meta.GetCardinality().value());
+ } else {
+ writer.SetNullAt(3);
+ }
+ writer.Complete();
+ array_writer.WriteRow(pos++, dv_data);
+ }
+ array_writer.Complete();
+ return array;
+}
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_meta_serializer.h
b/src/paimon/core/index/index_file_meta_serializer.h
new file mode 100644
index 0000000..6cac11a
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_serializer.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "paimon/common/data/binary_array.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/common/data/binary_string.h"
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/utils/object_serializer.h"
+#include "paimon/result.h"
+
+namespace paimon {
+class MemoryPool;
+class BinaryRowWriter;
+/// An `ObjectSerializer` for `IndexFileMeta`.
+class IndexFileMetaSerializer : public
ObjectSerializer<std::shared_ptr<IndexFileMeta>> {
+ public:
+ explicit IndexFileMetaSerializer(const std::shared_ptr<MemoryPool>& pool)
+ :
ObjectSerializer<std::shared_ptr<IndexFileMeta>>(IndexFileMeta::DataType(),
pool) {}
+
+ Result<BinaryRow> ToRow(const std::shared_ptr<IndexFileMeta>& meta) const
override;
+
+ Result<std::shared_ptr<IndexFileMeta>> FromRow(const InternalRow& row)
const override;
+
+ static void WriteIndexFileMeta(int32_t start_pos, const
std::shared_ptr<IndexFileMeta>& meta,
+ BinaryRowWriter* writer, MemoryPool* pool);
+
+ static BinaryArray DvRangesToRowArrayData(
+ const LinkedHashMap<std::string, DeletionVectorMeta>& dv_metas,
MemoryPool* pool);
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_meta_serializer_test.cpp
b/src/paimon/core/index/index_file_meta_serializer_test.cpp
new file mode 100644
index 0000000..4db2ce3
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_serializer_test.cpp
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/index_file_meta_serializer.h"
+
+#include <cstdlib>
+
+#include "gtest/gtest.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class IndexFileMetaSerializerTest : public testing::Test {
+ public:
+ void SetUp() override {
+ memory_pool_ = GetDefaultPool();
+ }
+
+ private:
+ std::shared_ptr<IndexFileMeta> GetRandomDeletionVectorIndexFile() {
+ LinkedHashMap<std::string, DeletionVectorMeta> deletion_vectors_ranges;
+ deletion_vectors_ranges.insert_or_assign(
+ "my_file_name1",
+ DeletionVectorMeta("my_file_name1", std::rand(), std::rand(),
std::nullopt));
+ deletion_vectors_ranges.insert_or_assign(
+ "my_file_name2",
+ DeletionVectorMeta("my_file_name2", std::rand(), std::rand(),
std::nullopt));
+ return std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "deletion_vectors_index_file_name" + std::to_string(std::rand()),
std::rand(), rand(),
+ deletion_vectors_ranges, /*external_path=*/std::nullopt);
+ }
+
+ const int32_t TRIES = 100;
+
+ std::shared_ptr<MemoryPool> memory_pool_;
+};
+
+TEST_F(IndexFileMetaSerializerTest, TestEqual) {
+ auto index_meta1 = GetRandomDeletionVectorIndexFile();
+ index_meta1->file_size_ = 10;
+ auto index_meta2 = GetRandomDeletionVectorIndexFile();
+ index_meta2->file_size_ = 20;
+ ASSERT_EQ(*index_meta1, *index_meta1);
+ ASSERT_TRUE(index_meta1->TEST_Equal(*index_meta1));
+ ASSERT_FALSE(*index_meta1 == *index_meta2);
+ ASSERT_FALSE(index_meta1->TEST_Equal(*index_meta2));
+}
+
+TEST_F(IndexFileMetaSerializerTest, TestToFromRow) {
+ IndexFileMetaSerializer serializer(memory_pool_);
+ for (int32_t i = 0; i < TRIES; i++) {
+ auto expected = GetRandomDeletionVectorIndexFile();
+ ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> actual,
serializer.FromRow(row));
+ ASSERT_EQ(expected->ToString(), actual->ToString());
+ ASSERT_EQ(*expected, *actual);
+ ASSERT_TRUE(expected->TEST_Equal(*actual));
+ }
+}
+
+TEST_F(IndexFileMetaSerializerTest, TestToFromRowWithNullDeletionVectorMetas) {
+ IndexFileMetaSerializer serializer(memory_pool_);
+ auto expected = std::make_shared<IndexFileMeta>(
+ std::string(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX),
+ "deletion_vectors_index_file_0", /*file_size=*/10,
+ /*row_count=*/5,
+ /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt);
+ ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> actual,
serializer.FromRow(row));
+ ASSERT_EQ(expected->ToString(), actual->ToString());
+ ASSERT_EQ(*expected, *actual);
+}
+
+TEST_F(IndexFileMetaSerializerTest, TestToFromRowWithGlobalIndex) {
+ auto bytes = std::make_shared<Bytes>("apple", memory_pool_.get());
+ IndexFileMetaSerializer serializer(memory_pool_);
+ GlobalIndexMeta global_index_meta(
+ /*row_range_start=*/10, /*row_range_end=*/50,
+ /*index_field_id=*/5,
/*extra_field_ids=*/std::optional<std::vector<int32_t>>({0, 1}),
+ bytes);
+ {
+ auto expected =
+ std::make_shared<IndexFileMeta>("bitmap", "bitmap_index_file_0",
/*file_size=*/10,
+ /*row_count=*/41,
/*dv_ranges=*/std::nullopt,
+ /*external_path=*/std::nullopt,
global_index_meta);
+ ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> actual,
serializer.FromRow(row));
+ ASSERT_EQ(expected->ToString(), actual->ToString());
+ ASSERT_EQ(*expected, *actual);
+ }
+ {
+ // test external path
+ auto expected = std::make_shared<IndexFileMeta>(
+ "bitmap", "bitmap_index_file_0", /*file_size=*/10,
+ /*row_count=*/41, /*dv_ranges=*/std::nullopt,
+ /*external_path=*/"FILE:/tmp/external/bitmap_index_file_0",
global_index_meta);
+ ASSERT_OK_AND_ASSIGN(BinaryRow row, serializer.ToRow(expected));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> actual,
serializer.FromRow(row));
+ ASSERT_EQ(expected->ToString(), actual->ToString());
+ ASSERT_EQ(*expected, *actual);
+ }
+}
+
+TEST_F(IndexFileMetaSerializerTest, TestSerialize) {
+ IndexFileMetaSerializer serializer(memory_pool_);
+ auto expected = GetRandomDeletionVectorIndexFile();
+ for (int32_t i = 0; i < TRIES; i++) {
+ MemorySegmentOutputStream out(1024, memory_pool_);
+ ASSERT_OK(serializer.Serialize(expected, &out));
+ PAIMON_UNIQUE_PTR<Bytes> bytes = MemorySegmentUtils::CopyToBytes(
+ out.Segments(), 0, out.CurrentSize(), memory_pool_.get());
+ auto input_stream =
std::make_shared<ByteArrayInputStream>(bytes->data(), bytes->size());
+ DataInputStream in(input_stream);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<IndexFileMeta> actual,
serializer.Deserialize(&in));
+ ASSERT_EQ(expected->ToString(), actual->ToString());
+ ASSERT_EQ(*expected, *actual);
+ }
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/index/index_file_meta_v1_deserializer.h
b/src/paimon/core/index/index_file_meta_v1_deserializer.h
new file mode 100644
index 0000000..8eb4f74
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_v1_deserializer.h
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/utils/object_serializer.h"
+
+namespace paimon {
+/// Serializer for `IndexFileMeta` with 0.9 version.
+class IndexFileMetaV1Deserializer : public
ObjectSerializer<std::shared_ptr<IndexFileMeta>> {
+ public:
+ static const std::shared_ptr<arrow::DataType>& DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_(
+ {arrow::field("_INDEX_TYPE", arrow::utf8(), false),
+ arrow::field("_FILE_NAME", arrow::utf8(), false),
+ arrow::field("_FILE_SIZE", arrow::int64(), false),
+ arrow::field("_ROW_COUNT", arrow::int64(), false),
+ arrow::field("_DELETIONS_VECTORS_RANGES",
+ arrow::list(arrow::field(
+ "item",
+ arrow::struct_({arrow::field("f0",
arrow::utf8(), false),
+ arrow::field("f1",
arrow::int32(), false),
+ arrow::field("f2",
arrow::int32(), false)}),
+ true)),
+ true)});
+ return schema;
+ }
+
+ explicit IndexFileMetaV1Deserializer(const std::shared_ptr<MemoryPool>&
pool)
+ : ObjectSerializer<std::shared_ptr<IndexFileMeta>>(DataType(), pool) {}
+
+ Result<BinaryRow> ToRow(const std::shared_ptr<IndexFileMeta>& meta) const
override {
+ assert(false);
+ return Status::Invalid("IndexFileMetaV1Deserializer to row is not
valid");
+ }
+
+ Result<std::shared_ptr<IndexFileMeta>> FromRow(const InternalRow& row)
const override {
+ auto file_type = row.GetString(0);
+ auto file_name = row.GetString(1);
+ auto file_size = row.GetLong(2);
+ auto row_count = row.GetLong(3);
+ std::optional<LinkedHashMap<std::string, DeletionVectorMeta>>
dv_ranges;
+ if (!row.IsNullAt(4)) {
+ dv_ranges = RowArrayDataToDvRanges(row.GetArray(4).get());
+ }
+ return std::make_shared<IndexFileMeta>(file_type.ToString(),
file_name.ToString(),
+ file_size, row_count, dv_ranges,
+ /*external_path=*/std::nullopt);
+ }
+
+ private:
+ static LinkedHashMap<std::string, DeletionVectorMeta>
RowArrayDataToDvRanges(
+ const InternalArray* array_data) {
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_metas;
+ for (int32_t i = 0; i < array_data->Size(); i++) {
+ auto row = array_data->GetRow(i, /*num_fields=*/3);
+ std::string file_name = row->GetString(0).ToString();
+ dv_metas.insert_or_assign(file_name, DeletionVectorMeta(file_name,
row->GetInt(1),
+
row->GetInt(2), std::nullopt));
+ }
+ return dv_metas;
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_meta_v2_deserializer.h
b/src/paimon/core/index/index_file_meta_v2_deserializer.h
new file mode 100644
index 0000000..a5acd4e
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_v2_deserializer.h
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/utils/object_serializer.h"
+
+namespace paimon {
+/// Serializer for `IndexFileMeta` with 1.2 version.
+class IndexFileMetaV2Deserializer : public
ObjectSerializer<std::shared_ptr<IndexFileMeta>> {
+ public:
+ static const std::shared_ptr<arrow::DataType>& DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_(
+ {arrow::field("_INDEX_TYPE", arrow::utf8(), false),
+ arrow::field("_FILE_NAME", arrow::utf8(), false),
+ arrow::field("_FILE_SIZE", arrow::int64(), false),
+ arrow::field("_ROW_COUNT", arrow::int64(), false),
+ arrow::field("_DELETIONS_VECTORS_RANGES",
+ arrow::list(arrow::field("item",
DeletionVectorMeta::DataType(), true)),
+ true)});
+ return schema;
+ }
+
+ explicit IndexFileMetaV2Deserializer(const std::shared_ptr<MemoryPool>&
pool)
+ : ObjectSerializer<std::shared_ptr<IndexFileMeta>>(DataType(), pool) {}
+
+ Result<BinaryRow> ToRow(const std::shared_ptr<IndexFileMeta>& meta) const
override {
+ assert(false);
+ return Status::Invalid("IndexFileMetaV2Deserializer to row is not
valid");
+ }
+
+ Result<std::shared_ptr<IndexFileMeta>> FromRow(const InternalRow& row)
const override {
+ auto file_type = row.GetString(0);
+ auto file_name = row.GetString(1);
+ auto file_size = row.GetLong(2);
+ auto row_count = row.GetLong(3);
+ std::optional<LinkedHashMap<std::string, DeletionVectorMeta>>
dv_ranges;
+ if (!row.IsNullAt(4)) {
+ dv_ranges = RowArrayDataToDvRanges(row.GetArray(4).get());
+ }
+ return std::make_shared<IndexFileMeta>(file_type.ToString(),
file_name.ToString(),
+ file_size, row_count, dv_ranges,
+ /*external_path=*/std::nullopt);
+ }
+
+ static LinkedHashMap<std::string, DeletionVectorMeta>
RowArrayDataToDvRanges(
+ const InternalArray* array_data) {
+ LinkedHashMap<std::string, DeletionVectorMeta> dv_metas;
+ for (int32_t i = 0; i < array_data->Size(); i++) {
+ auto row =
+ array_data->GetRow(i,
/*num_fields=*/DeletionVectorMeta::DataType()->num_fields());
+ std::string file_name = row->GetString(0).ToString();
+ std::optional<int64_t> cardinality =
+ row->IsNullAt(3) ? std::nullopt :
std::optional<int64_t>(row->GetLong(3));
+ dv_metas.insert_or_assign(file_name, DeletionVectorMeta(file_name,
row->GetInt(1),
+
row->GetInt(2), cardinality));
+ }
+ return dv_metas;
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_file_meta_v3_deserializer.h
b/src/paimon/core/index/index_file_meta_v3_deserializer.h
new file mode 100644
index 0000000..005da56
--- /dev/null
+++ b/src/paimon/core/index/index_file_meta_v3_deserializer.h
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <list>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "arrow/api.h"
+#include "paimon/common/data/internal_array.h"
+#include "paimon/common/data/internal_row.h"
+#include "paimon/common/utils/linked_hash_map.h"
+#include "paimon/core/index/deletion_vector_meta.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/index/index_file_meta_v2_deserializer.h"
+#include "paimon/core/utils/object_serializer.h"
+#include "paimon/result.h"
+
+struct ArrowArray;
+
+namespace paimon {
+class MemoryPool;
+
+class IndexFileMetaV3Deserializer : public
ObjectSerializer<std::shared_ptr<IndexFileMeta>> {
+ public:
+ static const std::shared_ptr<arrow::DataType>& DataType() {
+ static std::shared_ptr<arrow::DataType> schema = arrow::struct_({
+ arrow::field("_INDEX_TYPE", arrow::utf8(), false),
+ arrow::field("_FILE_NAME", arrow::utf8(), false),
+ arrow::field("_FILE_SIZE", arrow::int64(), false),
+ arrow::field("_ROW_COUNT", arrow::int64(), false),
+ arrow::field("_DELETIONS_VECTORS_RANGES",
+ arrow::list(arrow::field("item",
DeletionVectorMeta::DataType(), true)),
+ true),
+ arrow::field("_EXTERNAL_PATH", arrow::utf8(), true),
+ });
+ return schema;
+ }
+
+ explicit IndexFileMetaV3Deserializer(const std::shared_ptr<MemoryPool>&
pool)
+ : ObjectSerializer<std::shared_ptr<IndexFileMeta>>(DataType(), pool) {}
+
+ Result<BinaryRow> ToRow(const std::shared_ptr<IndexFileMeta>& meta) const
override {
+ assert(false);
+ return Status::Invalid("IndexFileMetaV3Deserializer to row is not
valid");
+ }
+
+ Result<std::shared_ptr<IndexFileMeta>> FromRow(const InternalRow& row)
const override {
+ auto file_type = row.GetString(0);
+ auto file_name = row.GetString(1);
+ auto file_size = row.GetLong(2);
+ auto row_count = row.GetLong(3);
+ std::optional<LinkedHashMap<std::string, DeletionVectorMeta>>
dv_ranges;
+ if (!row.IsNullAt(4)) {
+ dv_ranges =
IndexFileMetaV2Deserializer::RowArrayDataToDvRanges(row.GetArray(4).get());
+ }
+ std::optional<std::string> external_path;
+ if (!row.IsNullAt(5)) {
+ external_path = row.GetString(5).ToString();
+ }
+ return std::make_shared<IndexFileMeta>(file_type.ToString(),
file_name.ToString(),
+ file_size, row_count,
dv_ranges, external_path);
+ }
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_in_data_file_dir_path_factory.h
b/src/paimon/core/index/index_in_data_file_dir_path_factory.h
new file mode 100644
index 0000000..7339ddf
--- /dev/null
+++ b/src/paimon/core/index/index_in_data_file_dir_path_factory.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+
+#include "paimon/common/utils/path_util.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/index/index_path_factory.h"
+#include "paimon/core/io/data_file_path_factory.h"
+
+namespace paimon {
+/// Path factory to create an index path.
+class IndexInDataFileDirPathFactory : public IndexPathFactory {
+ public:
+ IndexInDataFileDirPathFactory(
+ const std::string& uuid, const std::shared_ptr<std::atomic<int32_t>>&
index_file_count,
+ const std::shared_ptr<DataFilePathFactory>& data_file_path_factory)
+ : uuid_(uuid),
+ index_file_count_(index_file_count),
+ data_file_path_factory_(data_file_path_factory) {}
+
+ std::string NewPath() const override {
+ std::string name = IndexPathFactory::INDEX_PREFIX + uuid_ + "-" +
+ std::to_string(index_file_count_->fetch_add(1));
+ return data_file_path_factory_->NewPathFromName(name);
+ }
+
+ std::string ToPath(const std::shared_ptr<IndexFileMeta>& file) const
override {
+ if (file->ExternalPath() != std::nullopt) {
+ return file->ExternalPath().value();
+ }
+ return PathUtil::JoinPath(data_file_path_factory_->Parent(),
file->FileName());
+ }
+
+ std::string ToPath(const std::string& file_name) const override {
+ return data_file_path_factory_->NewPathFromName(file_name);
+ }
+
+ bool IsExternalPath() const override {
+ return data_file_path_factory_->IsExternalPath();
+ }
+
+ private:
+ std::string uuid_;
+ std::shared_ptr<std::atomic<int32_t>> index_file_count_;
+ std::shared_ptr<DataFilePathFactory> data_file_path_factory_;
+};
+
+} // namespace paimon
diff --git a/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp
b/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp
new file mode 100644
index 0000000..fe49a46
--- /dev/null
+++ b/src/paimon/core/index/index_in_data_file_dir_path_factory_test.cpp
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/index/index_in_data_file_dir_path_factory.h"
+
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "paimon/common/fs/external_path_provider.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(IndexInDataFileDirPathFactoryTest, TestSimple) {
+ auto count = std::make_shared<std::atomic<int32_t>>();
+ count->store(0);
+
+ auto data_file_path_factory = std::make_shared<DataFilePathFactory>();
+
ASSERT_OK(data_file_path_factory->Init(/*parent=*/"/tmp/p0=1/p1=0/bucket-0",
+ /*format_identifier=*/"txt",
+ /*data_file_prefix=*/"data-",
+
/*external_path_provider=*/nullptr));
+ IndexInDataFileDirPathFactory factory(/*uuid=*/"uuid", count,
+ std::move(data_file_path_factory));
+
+ ASSERT_EQ(factory.NewPath(), "/tmp/p0=1/p1=0/bucket-0/index-uuid-0");
+ // test ToPath with IndexFileMeta
+ auto meta =
+ std::make_shared<IndexFileMeta>(/*index_type=*/"DELETION_VECTOR",
"deletion_file",
+ /*file_size=*/500, /*row_count=*/1,
+ /*dv_ranges=*/std::nullopt,
/*external_path=*/std::nullopt);
+ ASSERT_EQ(factory.ToPath(meta), "/tmp/p0=1/p1=0/bucket-0/deletion_file");
+
+ // test ToPath with file_name
+ ASSERT_EQ(factory.ToPath("bitmap.index"),
"/tmp/p0=1/p1=0/bucket-0/bitmap.index");
+ // test external path
+ ASSERT_FALSE(factory.IsExternalPath());
+}
+
+TEST(IndexInDataFileDirPathFactoryTest, TestWithExternalPath) {
+ auto count = std::make_shared<std::atomic<int32_t>>();
+ count->store(1);
+
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<ExternalPathProvider> external_path_provider,
+ ExternalPathProvider::Create({"/tmp/external_path/"},
"p0=1/p1=0/bucket-0"));
+
+ auto data_file_path_factory = std::make_shared<DataFilePathFactory>();
+ ASSERT_OK(
+ data_file_path_factory->Init(/*parent=*/"/tmp",
/*format_identifier=*/"txt",
+ /*data_file_prefix=*/"data-",
+
/*external_path_provider=*/std::move(external_path_provider)));
+ IndexInDataFileDirPathFactory factory(/*uuid=*/"uuid", count,
+ std::move(data_file_path_factory));
+ std::string external_path =
"/tmp/external_path/p0=1/p1=0/bucket-0/index-uuid-1";
+ ASSERT_EQ(factory.NewPath(), external_path);
+ auto meta = std::make_shared<IndexFileMeta>(
+ /*index_type=*/"DELETION_VECTOR", "deletion_file",
+ /*file_size=*/500, /*row_count=*/1,
+ /*dv_ranges=*/std::nullopt, external_path);
+ ASSERT_EQ(factory.ToPath(meta), external_path);
+ ASSERT_TRUE(factory.IsExternalPath());
+}
+
+} // namespace paimon::test
diff --git a/src/paimon/core/index/index_path_factory.h
b/src/paimon/core/index/index_path_factory.h
new file mode 100644
index 0000000..3cf7063
--- /dev/null
+++ b/src/paimon/core/index/index_path_factory.h
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "paimon/core/index/index_file_meta.h"
+
+namespace paimon {
+/// Path factory to create an index path.
+class IndexPathFactory {
+ public:
+ virtual ~IndexPathFactory() = default;
+
+ virtual std::string NewPath() const = 0;
+ virtual std::string ToPath(const std::string& file_name) const = 0;
+ virtual std::string ToPath(const std::shared_ptr<IndexFileMeta>& file)
const = 0;
+ virtual bool IsExternalPath() const = 0;
+
+ static inline const char INDEX_PREFIX[] = "index-";
+};
+} // namespace paimon
