hemantk-12 commented on code in PR #6182:
URL: https://github.com/apache/ozone/pull/6182#discussion_r1498440181
##########
hadoop-hdds/rocks-native/src/main/patches/rocks-native.patch:
##########
@@ -16,592 +16,557 @@
* limitations under the License.
*/
-diff --git a/include/rocksdb/sst_dump_tool.h b/include/rocksdb/sst_dump_tool.h
-index 9261ba47d..1e62b88a3 100644
---- a/include/rocksdb/sst_dump_tool.h
-+++ b/include/rocksdb/sst_dump_tool.h
-@@ -11,7 +11,8 @@ namespace ROCKSDB_NAMESPACE {
-
- class SSTDumpTool {
- public:
-- int Run(int argc, char const* const* argv, Options options = Options());
-+ int Run(int argc, char const* const* argv, Options options = Options(),
-+ FILE* out = stdout, FILE* err = stderr);
- };
-
- } // namespace ROCKSDB_NAMESPACE
-diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc
-index eefbaaeee..734a2f0dd 100644
---- a/table/sst_file_dumper.cc
-+++ b/table/sst_file_dumper.cc
-@@ -45,7 +45,7 @@ SstFileDumper::SstFileDumper(const Options& options,
- Temperature file_temp, size_t readahead_size,
- bool verify_checksum, bool output_hex,
- bool decode_blob_index, const EnvOptions&
soptions,
-- bool silent)
-+ bool silent, FILE* out, FILE* err)
- : file_name_(file_path),
- read_num_(0),
- file_temp_(file_temp),
-@@ -57,10 +57,13 @@ SstFileDumper::SstFileDumper(const Options& options,
- ioptions_(options_),
- moptions_(ColumnFamilyOptions(options_)),
- read_options_(verify_checksum, false),
-- internal_comparator_(BytewiseComparator()) {
-+ internal_comparator_(BytewiseComparator()),
-+ out_(out),
-+ err_(err)
-+ {
- read_options_.readahead_size = readahead_size;
- if (!silent_) {
-- fprintf(stdout, "Process %s\n", file_path.c_str());
-+ fprintf(out_, "Process %s\n", file_path.c_str());
- }
- init_result_ = GetTableReader(file_name_);
- }
-@@ -253,17 +256,17 @@ Status SstFileDumper::ShowAllCompressionSizes(
- int32_t compress_level_from, int32_t compress_level_to,
- uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes,
- uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer) {
-- fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
-+ fprintf(out_, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
- for (auto& i : compression_types) {
- if (CompressionTypeSupported(i.first)) {
-- fprintf(stdout, "Compression: %-24s\n", i.second);
-+ fprintf(out_, "Compression: %-24s\n", i.second);
- CompressionOptions compress_opt;
- compress_opt.max_dict_bytes = max_dict_bytes;
- compress_opt.zstd_max_train_bytes = zstd_max_train_bytes;
- compress_opt.max_dict_buffer_bytes = max_dict_buffer_bytes;
- compress_opt.use_zstd_dict_trainer = use_zstd_dict_trainer;
- for (int32_t j = compress_level_from; j <= compress_level_to; j++) {
-- fprintf(stdout, "Compression level: %d", j);
-+ fprintf(out_, "Compression level: %d", j);
- compress_opt.level = j;
- Status s = ShowCompressionSize(block_size, i.first, compress_opt);
- if (!s.ok()) {
-@@ -271,7 +274,7 @@ Status SstFileDumper::ShowAllCompressionSizes(
- }
- }
- } else {
-- fprintf(stdout, "Unsupported compression type: %s.\n", i.second);
-+ fprintf(err_, "Unsupported compression type: %s.\n", i.second);
- }
- }
- return Status::OK();
-@@ -307,9 +310,9 @@ Status SstFileDumper::ShowCompressionSize(
- }
-
- std::chrono::steady_clock::time_point end =
std::chrono::steady_clock::now();
-- fprintf(stdout, " Size: %10" PRIu64, file_size);
-- fprintf(stdout, " Blocks: %6" PRIu64, num_data_blocks);
-- fprintf(stdout, " Time Taken: %10s microsecs",
-+ fprintf(out_, " Size: %10" PRIu64, file_size);
-+ fprintf(out_, " Blocks: %6" PRIu64, num_data_blocks);
-+ fprintf(out_, " Time Taken: %10s microsecs",
- std::to_string(
- std::chrono::duration_cast<std::chrono::microseconds>(end -
start)
- .count())
-@@ -342,11 +345,11 @@ Status SstFileDumper::ShowCompressionSize(
- : ((static_cast<double>(not_compressed_blocks) /
- static_cast<double>(num_data_blocks)) *
- 100.0);
-- fprintf(stdout, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks,
-+ fprintf(out_, " Compressed: %6" PRIu64 " (%5.1f%%)", compressed_blocks,
- compressed_pcnt);
-- fprintf(stdout, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)",
-+ fprintf(out_, " Not compressed (ratio): %6" PRIu64 " (%5.1f%%)",
- ratio_not_compressed_blocks, ratio_not_compressed_pcnt);
-- fprintf(stdout, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n",
-+ fprintf(out_, " Not compressed (abort): %6" PRIu64 " (%5.1f%%)\n",
- not_compressed_blocks, not_compressed_pcnt);
- return Status::OK();
- }
-@@ -362,7 +365,7 @@ Status SstFileDumper::ReadTableProperties(uint64_t
table_magic_number,
- /* memory_allocator= */ nullptr, prefetch_buffer);
- if (!s.ok()) {
- if (!silent_) {
-- fprintf(stdout, "Not able to read table properties\n");
-+ fprintf(err_, "Not able to read table properties\n");
- }
- }
- return s;
-@@ -410,7 +413,7 @@ Status SstFileDumper::SetTableOptionsByMagicNumber(
-
- options_.table_factory.reset(NewPlainTableFactory(plain_table_options));
- if (!silent_) {
-- fprintf(stdout, "Sst file format: plain table\n");
-+ fprintf(out_, "Sst file format: plain table\n");
- }
- } else {
- char error_msg_buffer[80];
-@@ -427,15 +430,56 @@ Status SstFileDumper::SetOldTableOptions() {
- assert(table_properties_ == nullptr);
- options_.table_factory = std::make_shared<BlockBasedTableFactory>();
- if (!silent_) {
-- fprintf(stdout, "Sst file format: block-based(old version)\n");
-+ fprintf(out_, "Sst file format: block-based(old version)\n");
- }
-
- return Status::OK();
- }
-
-+void write(int value, FILE* file) {
-+ char b[4];
-+ b[3] = value & 0x000000ff;
-+ b[2] = (value & 0x0000ff00) >> 8;
-+ b[1] = (value & 0x00ff0000) >> 16;
-+ b[0] = (value & 0xff000000) >> 24;
-+ std::fwrite(b, 4, 1, file);
+diff --git a/include/rocksdb/raw_iterator.h b/include/rocksdb/raw_iterator.h
+new file mode 100644
+index 000000000..ec3c05d6d
+--- /dev/null
++++ b/include/rocksdb/raw_iterator.h
+@@ -0,0 +1,25 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++#pragma once
++#ifndef ROCKSDB_LITE
++
++
++#include "rocksdb/advanced_options.h"
++namespace ROCKSDB_NAMESPACE {
++
++class RawIterator {
++ public:
++ virtual ~RawIterator() {}
++ virtual bool has_next() const = 0;
++ virtual Slice getKey() const = 0;
++ virtual Slice getValue() const = 0;
++ virtual uint64_t getSequenceNumber() const = 0;
++ virtual uint32_t getType() const = 0;
++ virtual void next() = 0;
++};
++
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/include/rocksdb/raw_sst_file_reader.h
b/include/rocksdb/raw_sst_file_reader.h
+new file mode 100644
+index 000000000..266a0c1c8
+--- /dev/null
++++ b/include/rocksdb/raw_sst_file_reader.h
+@@ -0,0 +1,62 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++#pragma once
++#ifndef ROCKSDB_LITE
++
++#include <memory>
++#include <string>
++
++#include "rocksdb/raw_iterator.h"
++#include "rocksdb/advanced_options.h"
++#include "rocksdb/options.h"
++
++
++
++namespace ROCKSDB_NAMESPACE {
++
++class RawSstFileReader {
++ public:
++
++ RawSstFileReader(const Options& options, const std::string& file_name,
++ size_t readahead_size, bool verify_checksum,
++ bool silent = false);
++ ~RawSstFileReader();
++
++ RawIterator* newIterator(bool has_from, Slice* from,
++ bool has_to, Slice *to);
++ Status getStatus() { return init_result_; }
++
++ private:
++ // Get the TableReader implementation for the sst file
++ Status GetTableReader(const std::string& file_path);
++ Status ReadTableProperties(uint64_t table_magic_number,
++ uint64_t file_size);
++
++ Status SetTableOptionsByMagicNumber(uint64_t table_magic_number);
++ Status SetOldTableOptions();
++
++ // Helper function to call the factory with settings specific to the
++ // factory implementation
++ Status NewTableReader(uint64_t file_size);
++
++ std::string file_name_;
++ Temperature file_temp_;
++
++ // less verbose in stdout/stderr
++ bool silent_;
++
++ // options_ and internal_comparator_ will also be used in
++ // ReadSequential internally (specifically, seek-related operations)
++ Options options_;
++
++ Status init_result_;
++
++ struct Rep;
++ std::unique_ptr<Rep> rep_;
++};
++
++} // namespace ROCKSDB_NAMESPACE
++
++#endif // ROCKSDB_LITE
+diff --git a/src.mk b/src.mk
+index b94bc43ca..95bfff274 100644
+--- a/src.mk
++++ b/src.mk
+@@ -343,6 +343,8 @@ TOOL_LIB_SOURCES =
\
+ tools/ldb_tool.cc \
+ tools/sst_dump_tool.cc \
+ utilities/blob_db/blob_dump_tool.cc \
++ tools/raw_sst_file_reader.cc \
++ tools/raw_sst_file_reader_iterator.cc \
+
+ ANALYZER_LIB_SOURCES = \
+ tools/block_cache_analyzer/block_cache_trace_analyzer.cc \
+diff --git a/tools/raw_sst_file_reader.cc b/tools/raw_sst_file_reader.cc
+new file mode 100644
+index 000000000..1693bd1e6
+--- /dev/null
++++ b/tools/raw_sst_file_reader.cc
+@@ -0,0 +1,285 @@
++// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
++// This source code is licensed under both the GPLv2 (found in the
++// COPYING file in the root directory) and Apache 2.0 License
++// (found in the LICENSE.Apache file in the root directory).
++//
++#ifndef ROCKSDB_LITE
++
++#include "rocksdb/raw_sst_file_reader.h"
++
++#include <chrono>
++#include <cinttypes>
++#include <iostream>
++#include <map>
++#include <memory>
++#include <sstream>
++#include <vector>
++
++#include "db/blob/blob_index.h"
++#include "db/memtable.h"
++#include "db/write_batch_internal.h"
++#include "options/cf_options.h"
++#include "port/port.h"
++#include "rocksdb/db.h"
++#include "rocksdb/env.h"
++#include "rocksdb/iterator.h"
++#include "rocksdb/slice_transform.h"
++#include "rocksdb/status.h"
++#include "rocksdb/table_properties.h"
++#include "rocksdb/utilities/ldb_cmd.h"
++#include "table/block_based/block.h"
++#include "table/block_based/block_based_table_builder.h"
++#include "table/block_based/block_based_table_factory.h"
++#include "table/block_based/block_builder.h"
++#include "table/format.h"
++#include "table/meta_blocks.h"
++#include "table/plain/plain_table_factory.h"
++#include "table/table_reader.h"
++#include "tools/raw_sst_file_reader_iterator.h"
++#include "util/compression.h"
++#include "util/random.h"
++#include "db/dbformat.h"
++#include "file/writable_file_writer.h"
++#include "options/cf_options.h"
++
++namespace ROCKSDB_NAMESPACE {
++
++struct RawSstFileReader::Rep {
++ Options options;
++ EnvOptions soptions_;
++ ReadOptions read_options_;
++ ImmutableOptions ioptions_;
++ MutableCFOptions moptions_;
++ InternalKeyComparator internal_comparator_;
++ std::unique_ptr<TableProperties> table_properties_;
++ std::unique_ptr<TableReader> table_reader_;
++ std::unique_ptr<RandomAccessFileReader> file_;
++
++ Rep(const Options& opts, bool verify_checksum, size_t readahead_size)
++ : options(opts),
++ soptions_(EnvOptions()),
++ read_options_(verify_checksum, false),
Review Comment:
You can providing an options to verify checksum but it is never used as it
is in
[sst_file_dumper](https://github.com/facebook/rocksdb/blob/main/table/sst_file_dumper.cc#L202-L207).
Is my understanding correct, if so, we should add that or remove the option.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
