hemantk-12 commented on code in PR #6182:
URL: https://github.com/apache/ozone/pull/6182#discussion_r1498455383
##########
hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch:
##########
@@ -16,592 +16,557 @@
* limitations under the License.
*/
-diff --git a/include/rocksdb/sst_dump_tool.h b/include/rocksdb/sst_dump_tool.h
-index 9261ba47d..1e62b88a3 100644
---- a/include/rocksdb/sst_dump_tool.h
-+++ b/include/rocksdb/sst_dump_tool.h
-@@ -11,7 +11,8 @@ namespace ROCKSDB_NAMESPACE {
-
- class SSTDumpTool {
- public:
-- int Run(int argc, char const* const* argv, Options options = Options());
-+ int Run(int argc, char const* const* argv, Options options = Options(),
-+ FILE* out = stdout, FILE* err = stderr);
- };
-
- } // namespace ROCKSDB_NAMESPACE
-diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc
-index eefbaaeee..734a2f0dd 100644
---- a/table/sst_file_dumper.cc
-+++ b/table/sst_file_dumper.cc
-@@ -45,7 +45,7 @@ SstFileDumper::SstFileDumper(const Options& options,
- Temperature file_temp, size_t readahead_size,
- bool verify_checksum, bool output_hex,
- bool decode_blob_index, const EnvOptions&
soptions,
-- bool silent)
-+ bool silent, FILE* out, FILE* err)
- : file_name_(file_path),
- read_num_(0),
- file_temp_(file_temp),
-@@ -57,10 +57,13 @@ SstFileDumper::SstFileDumper(const Options& options,
- ioptions_(options_),
- moptions_(ColumnFamilyOptions(options_)),
- read_options_(verify_checksum, false),
-- internal_comparator_(BytewiseComparator()) {
-+ internal_comparator_(BytewiseComparator()),
-+ out_(out),
-+ err_(err)
-+ {
- read_options_.readahead_size = readahead_size;
- if (!silent_) {
-- fprintf(stdout, "Process %s\n", file_path.c_str());
-+ fprintf(out_, "Process %s\n", file_path.c_str());
- }
- init_result_ = GetTableReader(file_name_);
- }
-@@ -253,17 +256,17 @@ Status SstFileDumper::ShowAllCompressionSizes(
- int32_t compress_level_from, int32_t compress_level_to,
- uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes,
- uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer) {
-- fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
-+ fprintf(out_, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
- for (auto& i : compression_types) {
- if (CompressionTypeSupported(i.first)) {
-- fprintf(stdout, "Compression: %-24s\n", i.second);
-+ fprintf(out_, "Compression: %-24s\n", i.second);
- CompressionOptions compress_opt;
- compress_opt.max_dict_bytes = max_dict_bytes;
- compress_opt.zstd_max_train_bytes = zstd_max_train_bytes;
- compress_opt.max_dict_buffer_bytes = max_dict_buffer_bytes;
- compress_opt.use_zstd_dict_trainer = use_zstd_dict_trainer;
- for (int32_t j = compress_level_from; j <= compress_level_to; j++) {
-- fprintf(stdout, "Compression level: %d", j);
-+ fprintf(out_, "Compression level: %d", j);
- compress_opt.level = j;
- Status s = ShowCompressionSize(block_size, i.first, compress_opt);
- if (!s.ok()) {
-@@ -271,7 +274,7 @@ Status SstFileDumper::ShowAllCompressionSizes(
- }
- }
- } else {
-- fprintf(stdout, "Unsupported compression type: %s.\n", i.second);
-+ fprintf(err_, "Unsupported compression type: %s.\n", i.second);
- }
- }
- return Status::OK();
-@@ -307,9 +310,9 @@ Status SstFileDumper::ShowCompressionSize(
- }
-
- std::chrono::steady_clock::time_point end =
std::chrono::steady_clock::now();
-- fprintf(stdout, " Size: %10" PRIu64, file_size);
-- fprintf(stdout, " Blocks: %6" PRIu64, num_data_blocks);
-- fprintf(stdout, " Time Taken: %10s microsecs",
-+ fprintf(out_, " Size: %10" PRIu64, file_size);
-+ fprintf(out_, " Blocks: %6" PRIu64, num_data_blocks);
-+ fprintf(out_, " Time Taken: %10s microsecs",
- std::to_string(
- std::chrono::duration_cast<std::chrono::microseconds>(end -
start)
- .count())
-@@ -342,11 +345,11 @@ Status SstFileDumper::ShowCompressionSize(
- : ((static_cast<double>(not_compressed_blocks) /
- static_cast<double>(num_data_blocks)) *
- 100.0);
-- fprintf(stdout, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks,
-+ fprintf(out_, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks,
- compressed_pcnt);
-- fprintf(stdout, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)",
-+ fprintf(out_, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)",
- ratio_not_compressed_blocks, ratio_not_compressed_pcnt);
-- fprintf(stdout, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n",
-+ fprintf(out_, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n",
- not_compressed_blocks, not_compressed_pcnt);
- return Status::OK();
- }
-@@ -362,7 +365,7 @@ Status SstFileDumper::ReadTableProperties(uint64_t
table_magic_number,
- /* memory_allocator= */ nullptr, prefetch_buffer);
- if (!s.ok()) {
- if (!silent_) {
-- fprintf(stdout, "Not able to read table properties\n");
-+ fprintf(err_, "Not able to read table properties\n");
- }
- }
- return s;
-@@ -410,7 +413,7 @@ Status SstFileDumper::SetTableOptionsByMagicNumber(
-
- options_.table_factory.reset(NewPlainTableFactory(plain_table_options));
- if (!silent_) {
-- fprintf(stdout, "Sst file format: plain table\n");
-+ fprintf(out_, "Sst file format: plain table\n");
- }
- } else {
- char error_msg_buffer[80];
-@@ -427,15 +430,56 @@ Status SstFileDumper::SetOldTableOptions() {
- assert(table_properties_ == nullptr);
- options_.table_factory = std::make_shared<BlockBasedTableFactory>();
- if (!silent_) {
-- fprintf(stdout, "Sst file format: block-based(old version)\n");
-+ fprintf(out_, "Sst file format: block-based(old version)\n");
- }
-
- return Status::OK();
- }
-
-+void write(int value, FILE* file) {
-+ char b[4];
-+ b[3] = value & 0x000000ff;
-+ b[2] = (value & 0x0000ff00) >> 8;
-+ b[1] = (value & 0x00ff0000) >> 16;
-+ b[0] = (value & 0xff000000) >> 24;
-+ std::fwrite(b, 4, 1, file);
+diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h
+new file mode 100644
+index 000000000..ec3c05d6d
+--- /dev/null
++++ b/include/rocksdb/raw_iterator.h
+@@ -0,0 +1,25 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++#pragma once
++#ifndef ROCKSDB_LITE
++
++
++#include "rocksdb/advanced_options.h"
++namespace ROCKSDB_NAMESPACE {
++
++class RawIterator {
++ public:
++ virtual ~RawIterator() {}
++ virtual bool has_next() const = 0;
++ virtual Slice getKey() const = 0;
++ virtual Slice getValue() const = 0;
++ virtual uint64_t getSequenceNumber() const = 0;
++ virtual uint32_t getType() const = 0;
++ virtual void next() = 0;
++};
++
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/include/rocksdb/raw_sst_file_reader.h
b/include/rocksdb/raw_sst_file_reader.h
+new file mode 100644
+index 000000000..266a0c1c8
+--- /dev/null
++++ b/include/rocksdb/raw_sst_file_reader.h
+@@ -0,0 +1,62 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++#pragma once
++#ifndef ROCKSDB_LITE
++
++#include <memory>
++#include <string>
++
++#include "rocksdb/raw_iterator.h"
++#include "rocksdb/advanced_options.h"
++#include "rocksdb/options.h"
++
++
++
++namespace ROCKSDB_NAMESPACE {
++
++class RawSstFileReader {
++ public:
++
++ RawSstFileReader(const Options& options, const std::string& file_name,
++ size_t readahead_size, bool verify_checksum,
++ bool silent = false);
++ ~RawSstFileReader();
++
++ RawIterator* newIterator(bool has_from, Slice* from,
++ bool has_to, Slice *to);
++ Status getStatus() { return init_result_; }
++
++ private:
++ // Get the TableReader implementation for the sst file
++ Status GetTableReader(const std::string& file_path);
++ Status ReadTableProperties(uint64_t table_magic_number,
++ uint64_t file_size);
++
++ Status SetTableOptionsByMagicNumber(uint64_t table_magic_number);
++ Status SetOldTableOptions();
++
++ // Helper function to call the factory with settings specific to the
++ // factory implementation
++ Status NewTableReader(uint64_t file_size);
++
++ std::string file_name_;
++ Temperature file_temp_;
++
++ // less verbose in stdout/stderr
++ bool silent_;
++
++ // options_ and internal_comparator_ will also be used in
++ // ReadSequential internally (specifically, seek-related operations)
++ Options options_;
++
++ Status init_result_;
++
++ struct Rep;
++ std::unique_ptr<Rep> rep_;
++};
++
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/src.mk b/src.mk
+index b94bc43ca..95bfff274 100644
+--- a/src.mk
++++ b/src.mk
+@@ -343,6 +343,8 @@ TOOL_LIB_SOURCES =
\
+ tools/ldb_tool.cc \
+ tools/sst_dump_tool.cc \
+ utilities/blob_db/blob_dump_tool.cc \
++ tools/raw_sst_file_reader.cc \
++ tools/raw_sst_file_reader_iterator.cc \
+
+ ANALYZER_LIB_SOURCES = \
+ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \
+diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc
+new file mode 100644
+index 000000000..1693bd1e6
+--- /dev/null
++++ b/tools/raw_sst_file_reader.cc
+@@ -0,0 +1,285 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++//
++#ifndef ROCKSDB_LITE
++
++#include "rocksdb/raw_sst_file_reader.h"
++
++#include <chrono>
++#include <cinttypes>
++#include <iostream>
++#include <map>
++#include <memory>
++#include <sstream>
++#include <vector>
++
++#include "db/blob/blob_index.h"
++#include "db/memtable.h"
++#include "db/write_batch_internal.h"
++#include "options/cf_options.h"
++#include "port/port.h"
++#include "rocksdb/db.h"
++#include "rocksdb/env.h"
++#include "rocksdb/iterator.h"
++#include "rocksdb/slice_transform.h"
++#include "rocksdb/status.h"
++#include "rocksdb/table_properties.h"
++#include "rocksdb/utilities/ldb_cmd.h"
++#include "table/block_based/block.h"
++#include "table/block_based/block_based_table_builder.h"
++#include "table/block_based/block_based_table_factory.h"
++#include "table/block_based/block_builder.h"
++#include "table/format.h"
++#include "table/meta_blocks.h"
++#include "table/plain/plain_table_factory.h"
++#include "table/table_reader.h"
++#include "tools/raw_sst_file_reader_iterator.h"
++#include "util/compression.h"
++#include "util/random.h"
++#include "db/dbformat.h"
++#include "file/writable_file_writer.h"
++#include "options/cf_options.h"
++
++namespace ROCKSDB_NAMESPACE {
++
++struct RawSstFileReader::Rep {
++ Options options;
++ EnvOptions soptions_;
++ ReadOptions read_options_;
++ ImmutableOptions ioptions_;
++ MutableCFOptions moptions_;
++ InternalKeyComparator internal_comparator_;
++ std::unique_ptr<TableProperties> table_properties_;
++ std::unique_ptr<TableReader> table_reader_;
++ std::unique_ptr<RandomAccessFileReader> file_;
++
++ Rep(const Options& opts, bool verify_checksum, size_t readahead_size)
++ : options(opts),
++ soptions_(EnvOptions()),
++ read_options_(verify_checksum, false),
++ ioptions_(options),
++ moptions_(ColumnFamilyOptions(options)),
++ internal_comparator_(InternalKeyComparator(BytewiseComparator())) {
++ read_options_.readahead_size = readahead_size;
++ }
++};
++
++RawSstFileReader::RawSstFileReader(const Options& options,
++ const std::string& file_name,
++ size_t readahead_size,
++ bool verify_checksum,
++ bool silent) :rep_(new Rep(options,
++
verify_checksum,
++
readahead_size)) {
++ file_name_ = file_name;
++ silent_ = silent;
++ options_ = options;
++ file_temp_ = Temperature::kUnknown;
++ init_result_ = GetTableReader(file_name_);
++}
++
++RawSstFileReader::~RawSstFileReader() {}
++
++
++
++extern const uint64_t kBlockBasedTableMagicNumber;
++extern const uint64_t kLegacyBlockBasedTableMagicNumber;
++extern const uint64_t kPlainTableMagicNumber;
++extern const uint64_t kLegacyPlainTableMagicNumber;
++
++Status RawSstFileReader::GetTableReader(const std::string& file_path) {
++ // Warning about 'magic_number' being uninitialized shows up only in UBsan
++ // builds. Though access is guarded by 's.ok()' checks, fix the issue to
++ // avoid any warnings.
++ uint64_t magic_number = Footer::kNullTableMagicNumber;
++
++ // read table magic number
++ Footer footer;
++
++ const auto& fs = options_.env->GetFileSystem();
++ std::unique_ptr<FSRandomAccessFile> file;
++ uint64_t file_size = 0;
++ FileOptions fopts = rep_->soptions_;
++ fopts.temperature = file_temp_;
++ Status s = fs->NewRandomAccessFile(file_path, fopts, &file, nullptr);
++ if (s.ok()) {
++ s = fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
++ }
++
++ // check empty file
++ // if true, skip further processing of this file
++ if (file_size == 0) {
++ return Status::Aborted(file_path, "Empty file");
++ }
++
++ rep_->file_.reset(new RandomAccessFileReader(std::move(file), file_path));
++
++ FilePrefetchBuffer prefetch_buffer(
++ 0 /* readahead_size */, 0 /* max_readahead_size */, true /* enable */,
++ false /* track_min_offset */);
++ if (s.ok()) {
++ const uint64_t kSstDumpTailPrefetchSize = 512 * 1024;
++ uint64_t prefetch_size = (file_size > kSstDumpTailPrefetchSize)
++ ? kSstDumpTailPrefetchSize
++ : file_size;
++ uint64_t prefetch_off = file_size - prefetch_size;
++ IOOptions opts;
++ s = prefetch_buffer.Prefetch(opts, rep_->file_.get(), prefetch_off,
++ static_cast<size_t>(prefetch_size),
++ Env::IO_TOTAL /* rate_limiter_priority */);
++
++ s = ReadFooterFromFile(opts, rep_->file_.get(), &prefetch_buffer,
file_size,
++ &footer);
++ }
++ if (s.ok()) {
++ magic_number = footer.table_magic_number();
++ }
++
++ if (s.ok()) {
++ if (magic_number == kPlainTableMagicNumber ||
++ magic_number == kLegacyPlainTableMagicNumber) {
++ rep_->soptions_.use_mmap_reads = true;
++
++ fs->NewRandomAccessFile(file_path, fopts, &file, nullptr);
++ rep_->file_.reset(new RandomAccessFileReader(std::move(file),
file_path));
++ }
++
++ s = ROCKSDB_NAMESPACE::ReadTableProperties(
++ rep_->file_.get(), file_size, magic_number, rep_->ioptions_,
&(rep_->table_properties_),
++ /* memory_allocator= */ nullptr, (magic_number ==
kBlockBasedTableMagicNumber)
++ ? &prefetch_buffer
++ : nullptr);
++ if (!s.ok()) {
++ if (!silent_) {
++ fprintf(stderr, "Not able to read table properties\n");
++ }
++ }
++ // For old sst format, ReadTableProperties might fail but file can be read
++ if (s.ok()) {
++ s = SetTableOptionsByMagicNumber(magic_number);
++ if (s.ok()) {
++ if (rep_->table_properties_ &&
!rep_->table_properties_->comparator_name.empty()) {
++ ConfigOptions config_options;
++ const Comparator* user_comparator = nullptr;
++ s = Comparator::CreateFromString(config_options,
++
rep_->table_properties_->comparator_name,
++ &user_comparator);
++ if (s.ok()) {
++ assert(user_comparator);
++ rep_->internal_comparator_ =
InternalKeyComparator(user_comparator);
++ }
++ }
++ }
++ } else {
++ s = SetOldTableOptions();
++ }
++ options_.comparator = rep_->internal_comparator_.user_comparator();
++ }
++
++ if (s.ok()) {
++ s = NewTableReader(file_size);
++ }
++ return s;
+}
+
-+void write(const char* value, int length, FILE* file) {
-+ write(length, file);
-+ fwrite(value, length, 1, file);
++Status RawSstFileReader::NewTableReader(uint64_t file_size) {
++ auto t_opt =
++ TableReaderOptions(rep_->ioptions_, rep_->moptions_.prefix_extractor,
rep_->soptions_,
++ rep_->internal_comparator_, false /* skip_filters */,
++ false /* imortal */, true /* force_direct_prefetch
*/);
++ // Allow open file with global sequence number for backward compatibility.
++ t_opt.largest_seqno = kMaxSequenceNumber;
++
++ // We need to turn off pre-fetching of index and filter nodes for
++ // BlockBasedTable
++ if (options_.table_factory->IsInstanceOf(
++ TableFactory::kBlockBasedTableName())) {
++ return options_.table_factory->NewTableReader(t_opt,
std::move(rep_->file_),
++ file_size,
&(rep_->table_reader_),
++ /*enable_prefetch=*/false);
++ }
++
++ // For all other factory implementation
++ return options_.table_factory->NewTableReader(t_opt, std::move(rep_->file_),
++ file_size,
&(rep_->table_reader_));
++}
++
++Status RawSstFileReader::SetTableOptionsByMagicNumber(
++ uint64_t table_magic_number) {
++ assert(rep_->table_properties_);
++ if (table_magic_number == kBlockBasedTableMagicNumber ||
++ table_magic_number == kLegacyBlockBasedTableMagicNumber) {
++ BlockBasedTableFactory* bbtf = new BlockBasedTableFactory();
++ // To force tail prefetching, we fake reporting two useful reads of 512KB
++ // from the tail.
++ // It needs at least two data points to warm up the stats.
++ bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024);
++ bbtf->tail_prefetch_stats()->RecordEffectiveSize(512 * 1024);
++
++ options_.table_factory.reset(bbtf);
++ if (!silent_) {
++ fprintf(stdout, "Sst file format: block-based\n");
++ }
++
++ auto& props = rep_->table_properties_->user_collected_properties;
++ auto pos = props.find(BlockBasedTablePropertyNames::kIndexType);
++ if (pos != props.end()) {
++ auto index_type_on_file =
static_cast<BlockBasedTableOptions::IndexType>(
++ DecodeFixed32(pos->second.c_str()));
++ if (index_type_on_file ==
++ BlockBasedTableOptions::IndexType::kHashSearch) {
++ options_.prefix_extractor.reset(NewNoopTransform());
++ }
++ }
++ } else if (table_magic_number == kPlainTableMagicNumber ||
++ table_magic_number == kLegacyPlainTableMagicNumber) {
++ options_.allow_mmap_reads = true;
++
++ PlainTableOptions plain_table_options;
++ plain_table_options.user_key_len = kPlainTableVariableLength;
++ plain_table_options.bloom_bits_per_key = 0;
++ plain_table_options.hash_table_ratio = 0;
++ plain_table_options.index_sparseness = 1;
++ plain_table_options.huge_page_tlb_size = 0;
++ plain_table_options.encoding_type = kPlain;
++ plain_table_options.full_scan_mode = true;
++
++ options_.table_factory.reset(NewPlainTableFactory(plain_table_options));
++ if (!silent_) {
++ fprintf(stdout, "Sst file format: plain table\n");
++ }
++ } else {
++ char error_msg_buffer[80];
++ snprintf(error_msg_buffer, sizeof(error_msg_buffer) - 1,
++ "Unsupported table magic number --- %lx",
++ (long)table_magic_number);
++ return Status::InvalidArgument(error_msg_buffer);
++ }
++
++ return Status::OK();
++}
++
++Status RawSstFileReader::SetOldTableOptions() {
++ assert(rep_->table_properties_ == nullptr);
++ options_.table_factory = std::make_shared<BlockBasedTableFactory>();
++ if (!silent_) {
++ fprintf(stdout, "Sst file format: block-based(old version)\n");
++ }
++
++ return Status::OK();
+}
+
-+void write(const std::string& value, FILE* file) {
-+ write(value.data(), (int)value.length(), file);
++RawIterator* RawSstFileReader::newIterator(
++ bool has_from, Slice* from, bool has_to, Slice* to) {
++ InternalIterator* iter = rep_->table_reader_->NewIterator(
++ rep_->read_options_, rep_->moptions_.prefix_extractor.get(),
++ /*arena=*/nullptr, /*skip_filters=*/false,
++ TableReaderCaller::kSSTDumpTool);
++ return new RawSstFileReaderIterator(iter, has_from, from, has_to, to);
++
+}
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/tools/raw_sst_file_reader_iterator.cc
b/tools/raw_sst_file_reader_iterator.cc
+new file mode 100644
+index 000000000..39a2b84be
+--- /dev/null
++++ b/tools/raw_sst_file_reader_iterator.cc
+@@ -0,0 +1,94 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++//
++#ifndef ROCKSDB_LITE
++
++#include "tools/raw_sst_file_reader_iterator.h"
++
++#include <chrono>
++#include <cinttypes>
++#include <iostream>
++#include <map>
++#include <memory>
++#include <sstream>
++#include <vector>
++
++#include "db/blob/blob_index.h"
++#include "db/memtable.h"
++#include "db/write_batch_internal.h"
++#include "options/cf_options.h"
++#include "port/port.h"
++#include "rocksdb/db.h"
++#include "rocksdb/env.h"
++#include "rocksdb/iterator.h"
++#include "rocksdb/slice_transform.h"
++#include "rocksdb/status.h"
++#include "rocksdb/table_properties.h"
++#include "rocksdb/utilities/ldb_cmd.h"
++#include "table/block_based/block.h"
++#include "table/block_based/block_based_table_builder.h"
++#include "table/block_based/block_based_table_factory.h"
++#include "table/block_based/block_builder.h"
++#include "table/format.h"
++#include "table/meta_blocks.h"
++#include "table/plain/plain_table_factory.h"
++#include "table/table_reader.h"
++#include "util/compression.h"
++#include "util/random.h"
++
++namespace ROCKSDB_NAMESPACE {
+
-+void write(Slice &slice, FILE* file) {
-+ int size = (int)slice.size();
-+ write(slice.data(), size, file);
++RawSstFileReaderIterator::RawSstFileReaderIterator(InternalIterator* iterator,
++ bool has_from, Slice* from_key,
++ bool has_to, Slice* to_key)
++ : iter_(iterator),
++ ikey(new ParsedInternalKey()),
++ has_to_(has_to),
++ to_key_(to_key) {
++ if (has_from) {
++ InternalKey k;
++ k.SetMinPossibleForUserKey(*from_key);
++ iter_->Seek(k.Encode());
++ } else {
++ iter_->SeekToFirst();
++ }
++ initKey();
+}
+
-+void write(SequenceNumber sequenceNumber, FILE* file) {
++bool RawSstFileReaderIterator::has_next() const {
++ return iter_->Valid() && (!has_to_ ||
++ BytewiseComparator()->Compare(
++ getKey(), *to_key_) < 0);
++}
+
-+ char b[8];
-+ int idx = 7;
-+ while (idx >= 0) {
-+ b[idx] = sequenceNumber % 256;
-+ sequenceNumber /= 256;
-+ idx -= 1;
++void RawSstFileReaderIterator::initKey() {
++ if (iter_->Valid()) {
++ ParseInternalKey(iter_->key(), ikey, true /* log_err_key */);
+ }
-+ fwrite(b, 8, 1, file);
++}
++void RawSstFileReaderIterator::next() {
++ iter_->Next();
++ initKey();
++
++}
++
++Slice RawSstFileReaderIterator::getKey() const {
++ return ikey->user_key;
+}
+
-+void write(ParsedInternalKey &key, FILE* file) {
-+ write(key.user_key, file);
-+ write(key.sequence, file);
-+ write(static_cast<int>(key.type), file);
++uint64_t RawSstFileReaderIterator::getSequenceNumber() const {
++ return ikey->sequence;
+}
+
- Status SstFileDumper::ReadSequential(bool print_kv, uint64_t read_num,
-- bool has_from, const std::string&
from_key,
-- bool has_to, const std::string& to_key,
-+ bool has_from, const Slice& from_key,
-+ bool has_to, const Slice& to_key,
- bool use_from_as_prefix) {
- if (!table_reader_) {
- return init_result_;
-@@ -446,6 +490,7 @@ Status SstFileDumper::ReadSequential(bool print_kv,
uint64_t read_num,
- /*arena=*/nullptr, /*skip_filters=*/false,
- TableReaderCaller::kSSTDumpTool);
- uint64_t i = 0;
-+
- if (has_from) {
- InternalKey ikey;
- ikey.SetMinPossibleForUserKey(from_key);
-@@ -453,6 +498,7 @@ Status SstFileDumper::ReadSequential(bool print_kv,
uint64_t read_num,
- } else {
- iter->SeekToFirst();
- }
-+
- for (; iter->Valid(); iter->Next()) {
- Slice key = iter->key();
- Slice value = iter->value();
-@@ -478,22 +524,19 @@ Status SstFileDumper::ReadSequential(bool print_kv,
uint64_t read_num,
-
- if (print_kv) {
- if (!decode_blob_index_ || ikey.type != kTypeBlobIndex) {
-- fprintf(stdout, "%s => %s\n",
-- ikey.DebugString(true, output_hex_).c_str(),
-- value.ToString(output_hex_).c_str());
-+ write(ikey, out_);
-+ write(value, out_);
- } else {
- BlobIndex blob_index;
--
- const Status s = blob_index.DecodeFrom(value);
- if (!s.ok()) {
-- fprintf(stderr, "%s => error decoding blob index\n",
-- ikey.DebugString(true, output_hex_).c_str());
-+ write(ikey, err_);
-+ write("error decoding blob index", err_);
- continue;
- }
--
-- fprintf(stdout, "%s => %s\n",
-- ikey.DebugString(true, output_hex_).c_str(),
-- blob_index.DebugString(output_hex_).c_str());
-+ write(ikey, out_);
-+ std::string v = blob_index.DebugString(output_hex_);
-+ write(v, out_);
- }
- }
- }
-diff --git a/table/sst_file_dumper.h b/table/sst_file_dumper.h
-index 7be876390..768c5b1e2 100644
---- a/table/sst_file_dumper.h
-+++ b/table/sst_file_dumper.h
-@@ -22,11 +22,13 @@ class SstFileDumper {
- bool verify_checksum, bool output_hex,
- bool decode_blob_index,
- const EnvOptions& soptions = EnvOptions(),
-- bool silent = false);
-+ bool silent = false,
-+ FILE* out = stdout,
-+ FILE* err = stderr);
-
- Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
-- const std::string& from_key, bool has_to,
-- const std::string& to_key,
-+ const Slice& from_key, bool has_to,
-+ const Slice& to_key,
- bool use_from_as_prefix = false);
-
- Status ReadTableProperties(
-@@ -94,6 +96,8 @@ class SstFileDumper {
- ReadOptions read_options_;
- InternalKeyComparator internal_comparator_;
- std::unique_ptr<TableProperties> table_properties_;
-+ FILE* out_;
-+ FILE* err_;
- };
-
- } // namespace ROCKSDB_NAMESPACE
-diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc
-index 7053366e7..8f248ddf3 100644
---- a/tools/sst_dump_tool.cc
-+++ b/tools/sst_dump_tool.cc
-@@ -31,7 +31,7 @@ static const std::vector<std::pair<CompressionType, const
char*>>
-
- namespace {
-
--void print_help(bool to_stderr) {
-+void print_help(bool to_stderr, FILE* err_, FILE* out_) {
- std::string supported_compressions;
- for (CompressionType ct : GetSupportedCompressions()) {
- if (!supported_compressions.empty()) {
-@@ -43,7 +43,7 @@ void print_help(bool to_stderr) {
- supported_compressions += str;
- }
- fprintf(
-- to_stderr ? stderr : stdout,
-+ to_stderr ? err_ : out_,
- R"(sst_dump --file=<data_dir_OR_sst_file>
[--command=check|scan|raw|recompress|identify]
- --file=<data_dir_OR_sst_file>
- Path to SST file or directory containing SST files
-@@ -149,7 +149,13 @@ bool ParseIntArg(const char* arg, const std::string
arg_name,
- }
- } // namespace
-
--int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
-+Slice* AssignSlicePrependedWithLength(const char* buf) {
-+ long val = std::stol(buf);
-+ return reinterpret_cast<Slice *>(val);
++uint32_t RawSstFileReaderIterator::getType() const {
++ return static_cast<int>(ikey->type);
+}
+
-+int SSTDumpTool::Run(int argc, char const* const* argv, Options options,
-+ FILE* out, FILE* err) {
- std::string env_uri, fs_uri;
- const char* dir_or_file = nullptr;
- uint64_t read_num = std::numeric_limits<uint64_t>::max();
-@@ -170,8 +176,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- bool has_compression_level_from = false;
- bool has_compression_level_to = false;
- bool has_specified_compression_types = false;
-- std::string from_key;
-- std::string to_key;
-+ bool silent = false;
-+ Slice* from_key = nullptr;
-+ Slice* to_key = nullptr;
- std::string block_size_str;
- std::string compression_level_from_str;
- std::string compression_level_to_str;
-@@ -197,7 +204,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- int64_t tmp_val;
-
- for (int i = 1; i < argc; i++) {
-- if (strncmp(argv[i], "--env_uri=", 10) == 0) {
-+ if (strncmp(argv[i], "--silent", 8) == 0) {
-+ silent = true;
-+ } else if (strncmp(argv[i], "--env_uri=", 10) == 0) {
- env_uri = argv[i] + 10;
- } else if (strncmp(argv[i], "--fs_uri=", 9) == 0) {
- fs_uri = argv[i] + 9;
-@@ -217,13 +226,13 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- } else if (strncmp(argv[i], "--command=", 10) == 0) {
- command = argv[i] + 10;
- } else if (strncmp(argv[i], "--from=", 7) == 0) {
-- from_key = argv[i] + 7;
-+ from_key = AssignSlicePrependedWithLength(argv[i] + 7);
- has_from = true;
- } else if (strncmp(argv[i], "--to=", 5) == 0) {
-- to_key = argv[i] + 5;
-+ to_key = AssignSlicePrependedWithLength(argv[i] + 5);
- has_to = true;
- } else if (strncmp(argv[i], "--prefix=", 9) == 0) {
-- from_key = argv[i] + 9;
-+ from_key = AssignSlicePrependedWithLength( argv[i] + 9);
- use_from_as_prefix = true;
- } else if (strcmp(argv[i], "--show_properties") == 0) {
- show_properties = true;
-@@ -273,7 +282,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- std::cerr << pik_status.getState() << "\n";
- retc = -1;
- }
-- fprintf(stdout, "key=%s\n", ikey.DebugString(true, true).c_str());
-+ fprintf(out, "key=%s\n", ikey.DebugString(true, true).c_str());
- return retc;
- } else if (ParseIntArg(argv[i], "--compression_level_from=",
- "compression_level_from must be numeric",
-@@ -288,9 +297,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- "compression_max_dict_bytes must be numeric",
- &tmp_val)) {
- if (tmp_val < 0 || tmp_val > std::numeric_limits<uint32_t>::max()) {
-- fprintf(stderr, "compression_max_dict_bytes must be a uint32_t:
'%s'\n",
-+ fprintf(err, "compression_max_dict_bytes must be a uint32_t: '%s'\n",
- argv[i]);
-- print_help(/*to_stderr*/ true);
-+ print_help(/*to_stderr*/ true, err, out);
- return 1;
- }
- compression_max_dict_bytes = static_cast<uint32_t>(tmp_val);
-@@ -298,10 +307,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- "compression_zstd_max_train_bytes must be numeric",
- &tmp_val)) {
- if (tmp_val < 0 || tmp_val > std::numeric_limits<uint32_t>::max()) {
-- fprintf(stderr,
-+ fprintf(err,
- "compression_zstd_max_train_bytes must be a uint32_t: '%s'\n",
- argv[i]);
-- print_help(/*to_stderr*/ true);
-+ print_help(/*to_stderr*/ true, err, out);
- return 1;
- }
- compression_zstd_max_train_bytes = static_cast<uint32_t>(tmp_val);
-@@ -309,56 +318,56 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- "compression_max_dict_buffer_bytes must be
numeric",
- &tmp_val)) {
- if (tmp_val < 0) {
-- fprintf(stderr,
-+ fprintf(err,
- "compression_max_dict_buffer_bytes must be positive: '%s'\n",
- argv[i]);
-- print_help(/*to_stderr*/ true);
-+ print_help(/*to_stderr*/ true, err, out);
- return 1;
- }
- compression_max_dict_buffer_bytes = static_cast<uint64_t>(tmp_val);
- } else if (strcmp(argv[i], "--compression_use_zstd_finalize_dict") == 0) {
- compression_use_zstd_finalize_dict = true;
- } else if (strcmp(argv[i], "--help") == 0) {
-- print_help(/*to_stderr*/ false);
-+ print_help(/*to_stderr*/ false, err, out);
- return 0;
- } else if (strcmp(argv[i], "--version") == 0) {
- printf("%s\n", GetRocksBuildInfoAsString("sst_dump").c_str());
- return 0;
- } else {
-- fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]);
-- print_help(/*to_stderr*/ true);
-+ fprintf(err, "Unrecognized argument '%s'\n\n", argv[i]);
-+ print_help(/*to_stderr*/ true, err, out);
- return 1;
- }
- }
-
- if(has_compression_level_from && has_compression_level_to) {
- if(!has_specified_compression_types || compression_types.size() != 1) {
-- fprintf(stderr, "Specify one compression type.\n\n");
-+ fprintf(err, "Specify one compression type.\n\n");
- exit(1);
- }
- } else if(has_compression_level_from || has_compression_level_to) {
-- fprintf(stderr, "Specify both --compression_level_from and "
-+ fprintf(err, "Specify both --compression_level_from and "
- "--compression_level_to.\n\n");
- exit(1);
- }
-
- if (use_from_as_prefix && has_from) {
-- fprintf(stderr, "Cannot specify --prefix and --from\n\n");
-+ fprintf(err, "Cannot specify --prefix and --from\n\n");
- exit(1);
- }
-
- if (input_key_hex) {
- if (has_from || use_from_as_prefix) {
-- from_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(from_key);
-+ *from_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(from_key ->
ToString());
- }
- if (has_to) {
-- to_key = ROCKSDB_NAMESPACE::LDBCommand::HexToString(to_key);
-+ *to_key =
ROCKSDB_NAMESPACE::LDBCommand::HexToString(to_key->ToString());
- }
- }
-
- if (dir_or_file == nullptr) {
-- fprintf(stderr, "file or directory must be specified.\n\n");
-- print_help(/*to_stderr*/ true);
-+ fprintf(err, "file or directory must be specified.\n\n");
-+ print_help(/*to_stderr*/ true, err, out);
- exit(1);
- }
-
-@@ -373,10 +382,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- Status s = Env::CreateFromUri(config_options, env_uri, fs_uri,
&options.env,
- &env_guard);
- if (!s.ok()) {
-- fprintf(stderr, "CreateEnvFromUri: %s\n", s.ToString().c_str());
-+ fprintf(err, "CreateEnvFromUri: %s\n", s.ToString().c_str());
- exit(1);
-- } else {
-- fprintf(stdout, "options.env is %p\n", options.env);
-+ } else if (!silent){
-+ fprintf(out, "options.env is %p\n", options.env);
- }
- }
-
-@@ -390,7 +399,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- Status s = env->FileExists(dir_or_file);
- // dir_or_file does not exist
- if (!s.ok()) {
-- fprintf(stderr, "%s%s: No such file or directory\n",
s.ToString().c_str(),
-+ fprintf(err, "%s%s: No such file or directory\n", s.ToString().c_str(),
- dir_or_file);
- return 1;
- }
-@@ -421,10 +430,11 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
-
- ROCKSDB_NAMESPACE::SstFileDumper dumper(
- options, filename, Temperature::kUnknown, readahead_size,
-- verify_checksum, output_hex, decode_blob_index);
-+ verify_checksum, output_hex, decode_blob_index, EnvOptions(),
-+ silent, out, err);
- // Not a valid SST
- if (!dumper.getStatus().ok()) {
-- fprintf(stderr, "%s: %s\n", filename.c_str(),
-+ fprintf(err, "%s: %s\n", filename.c_str(),
- dumper.getStatus().ToString().c_str());
- continue;
- } else {
-@@ -433,10 +443,11 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- // where there is at least one valid SST
- if (valid_sst_files.size() == 1) {
- // from_key and to_key are only used for "check", "scan", or ""
-- if (command == "check" || command == "scan" || command == "") {
-- fprintf(stdout, "from [%s] to [%s]\n",
-- ROCKSDB_NAMESPACE::Slice(from_key).ToString(true).c_str(),
-- ROCKSDB_NAMESPACE::Slice(to_key).ToString(true).c_str());
-+ if (!silent && (command == "check" || command == "scan" ||
-+ command == "")) {
-+ fprintf(out, "from [%s] to [%s]\n",
-+ from_key->ToString(true).c_str(),
-+ to_key->ToString(true).c_str());
- }
- }
- }
-@@ -449,7 +460,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- compression_zstd_max_train_bytes, compression_max_dict_buffer_bytes,
- !compression_use_zstd_finalize_dict);
- if (!st.ok()) {
-- fprintf(stderr, "Failed to recompress: %s\n", st.ToString().c_str());
-+ fprintf(err, "Failed to recompress: %s\n", st.ToString().c_str());
- exit(1);
- }
- return 0;
-@@ -461,10 +472,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
-
- st = dumper.DumpTable(out_filename);
- if (!st.ok()) {
-- fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str());
-+ fprintf(err, "%s: %s\n", filename.c_str(), st.ToString().c_str());
- exit(1);
- } else {
-- fprintf(stdout, "raw dump written to file %s\n", &out_filename[0]);
-+ fprintf(out, "raw dump written to file %s\n", &out_filename[0]);
- }
- continue;
- }
-@@ -473,10 +484,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- if (command == "" || command == "scan" || command == "check") {
- st = dumper.ReadSequential(
- command == "scan", read_num > 0 ? (read_num - total_read) :
read_num,
-- has_from || use_from_as_prefix, from_key, has_to, to_key,
-+ has_from || use_from_as_prefix, *from_key, has_to, *to_key,
- use_from_as_prefix);
- if (!st.ok()) {
-- fprintf(stderr, "%s: %s\n", filename.c_str(),
-+ fprintf(err, "%s: %s\n", filename.c_str(),
- st.ToString().c_str());
- }
- total_read += dumper.GetReadNumber();
-@@ -488,10 +499,10 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- if (command == "verify") {
- st = dumper.VerifyChecksum();
- if (!st.ok()) {
-- fprintf(stderr, "%s is corrupted: %s\n", filename.c_str(),
-+ fprintf(err, "%s is corrupted: %s\n", filename.c_str(),
- st.ToString().c_str());
- } else {
-- fprintf(stdout, "The file is ok\n");
-+ fprintf(out, "The file is ok\n");
- }
- continue;
- }
-@@ -503,15 +514,15 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- table_properties_from_reader;
- st = dumper.ReadTableProperties(&table_properties_from_reader);
- if (!st.ok()) {
-- fprintf(stderr, "%s: %s\n", filename.c_str(), st.ToString().c_str());
-- fprintf(stderr, "Try to use initial table properties\n");
-+ fprintf(err, "%s: %s\n", filename.c_str(), st.ToString().c_str());
-+ fprintf(err, "Try to use initial table properties\n");
- table_properties = dumper.GetInitTableProperties();
- } else {
- table_properties = table_properties_from_reader.get();
- }
- if (table_properties != nullptr) {
- if (show_properties) {
-- fprintf(stdout,
-+ fprintf(out,
- "Table Properties:\n"
- "------------------------------\n"
- " %s",
-@@ -523,18 +534,18 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- total_index_block_size += table_properties->index_size;
- total_filter_block_size += table_properties->filter_size;
- if (show_properties) {
-- fprintf(stdout,
-+ fprintf(out,
- "Raw user collected properties\n"
- "------------------------------\n");
- for (const auto& kv : table_properties->user_collected_properties) {
- std::string prop_name = kv.first;
- std::string prop_val = Slice(kv.second).ToString(true);
-- fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(),
-+ fprintf(out, " # %s: 0x%s\n", prop_name.c_str(),
- prop_val.c_str());
- }
- }
- } else {
-- fprintf(stderr, "Reader unexpectedly returned null properties\n");
-+ fprintf(err, "Reader unexpectedly returned null properties\n");
- }
- }
- }
-@@ -555,9 +566,9 @@ int SSTDumpTool::Run(int argc, char const* const* argv,
Options options) {
- // Exit with an error state
- if (dir) {
- fprintf(stdout, "------------------------------\n");
-- fprintf(stderr, "No valid SST files found in %s\n", dir_or_file);
-+ fprintf(err, "No valid SST files found in %s\n", dir_or_file);
- } else {
-- fprintf(stderr, "%s is not a valid SST file\n", dir_or_file);
-+ fprintf(err, "%s is not a valid SST file\n", dir_or_file);
- }
- return 1;
- } else {
++Slice RawSstFileReaderIterator::getValue() const {
++ return iter_->value();
++}
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/tools/raw_sst_file_reader_iterator.h
b/tools/raw_sst_file_reader_iterator.h
+new file mode 100644
+index 000000000..79313264f
+--- /dev/null
++++ b/tools/raw_sst_file_reader_iterator.h
Review Comment:
nit: let's just call it `raw_sst_file_iterator`. I think it is good enough
to say `raw_sst_file_reader` provides a `raw_sst_file_iterator`.
##########
hadoop-hdds/rocks-native/pom.xml:
##########
@@ -113,79 +111,80 @@
<build>
<plugins>
<plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
- <id>rocksdb source download</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
-
<url>https://github.com/facebook/rocksdb/archive/refs/tags/v${rocksdb.version}.tar.gz</url>
-
<outputFileName>rocksdb-v${rocksdb.version}.tar.gz</outputFileName>
-
<outputDirectory>${project.build.directory}/rocksdb</outputDirectory>
- </configuration>
- </execution>
- <execution>
- <id>zlib source download</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
- <url>${zlib.url}</url>
- <outputFileName>zlib-${zlib.version}.tar.gz</outputFileName>
-
<outputDirectory>${project.build.directory}/zlib</outputDirectory>
- </configuration>
- </execution>
- <execution>
- <id>bzip2 source download</id>
- <phase>generate-sources</phase>
+ <id>set-property</id>
+ <phase>initialize</phase>
<goals>
- <goal>wget</goal>
+ <goal>java</goal>
</goals>
<configuration>
- <url>${bzip2.url}</url>
-
<outputFileName>bzip2-v${bzip2.version}.tar.gz</outputFileName>
-
<outputDirectory>${project.build.directory}/bzip2</outputDirectory>
+
<mainClass>org.apache.hadoop.hdds.utils.db.managed.JniLibNamePropertyWriter</mainClass>
+ <arguments>
+
<argument>${project.build.directory}/propertyFile.txt</argument>
+ </arguments>
</configuration>
</execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <executions>
<execution>
- <id>lz4 source download</id>
- <phase>generate-sources</phase>
+ <id>read-property-from-file</id>
+ <phase>initialize</phase> <!-- Choose the appropriate phase -->
Review Comment:
Do you intend to change phase? If yes, please do it as part of this change
and remove the comment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
