lordgamez commented on code in PR #1586:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1586#discussion_r1258201957
##########
extensions/aws/processors/PutS3Object.h:
##########
@@ -97,54 +111,61 @@ class PutS3Object : public S3Processor {
class ReadCallback {
public:
- static constexpr uint64_t MAX_SIZE = 5_GiB;
- static constexpr uint64_t BUFFER_SIZE = 4_KiB;
-
- ReadCallback(uint64_t flow_size, const
minifi::aws::s3::PutObjectRequestParameters& options, aws::s3::S3Wrapper&
s3_wrapper)
- : flow_size_(flow_size)
- , options_(options)
- , s3_wrapper_(s3_wrapper) {
+ ReadCallback(uint64_t flow_size, const
minifi::aws::s3::PutObjectRequestParameters& options, aws::s3::S3Wrapper&
s3_wrapper,
+ uint64_t multipart_threshold, uint64_t multipart_size,
core::logging::Logger& logger)
+ : flow_size_(flow_size),
+ options_(options),
+ s3_wrapper_(s3_wrapper),
+ multipart_threshold_(multipart_threshold),
+ multipart_size_(multipart_size),
+ logger_(logger) {
}
int64_t operator()(const std::shared_ptr<io::InputStream>& stream) {
- if (flow_size_ > MAX_SIZE) {
- return -1;
- }
- std::vector<std::byte> buffer;
- buffer.resize(BUFFER_SIZE);
- auto data_stream = std::make_shared<std::stringstream>();
- read_size_ = 0;
- while (read_size_ < flow_size_) {
- const auto next_read_size = (std::min)(flow_size_ - read_size_,
BUFFER_SIZE);
- const auto read_ret = stream->read(gsl::make_span(buffer).subspan(0,
next_read_size));
- if (io::isError(read_ret)) {
- return -1;
- }
- if (read_ret > 0) {
- data_stream->write(reinterpret_cast<char*>(buffer.data()),
gsl::narrow<std::streamsize>(next_read_size));
- read_size_ += read_ret;
+ try {
+ if (flow_size_ <= multipart_threshold_) {
+ logger_.log_info("Uploading S3 Object '%s' in a single upload",
options_.object_key);
Review Comment:
Updated in e5b30fa6df2d0fbd54a85e74dd3ab9995dafe4c9
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
