Hello, please help with the following problem.
I use CodedOutputStream to write a message's length and message itself to a
file.
When I later read from generated file, it turns out that file contains less
data.
I provide source code for simple test: write 1000 messages to a file and
then read this file.
Output is the following:
[aagenosov@black build]$ bin/test_coded_stream
written 1000 records, total bytes: 16384
test successfull
total length: 13890, consumed bytes: 13890, messages count: 594
I tried to forcedly flush buffers, but there was no effect.
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/protobuf.
For more options, visit https://groups.google.com/d/optout.
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <memory>
#include <fcntl.h>
#include <errno.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
static const uint32_t records_limit = 1000;
int main()
{
using namespace google::protobuf::io;
const char * file_path = "/tmp/coded_stream_test.log";
{
int file_handle = open(file_path,
O_WRONLY | O_CREAT,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
);
if (file_handle == -1)
{
std::cerr << "can't create file " << file_path << ", errno: " << errno << std::endl;
return -1;
}
std::auto_ptr<FileOutputStream> file_stream;
std::auto_ptr<CodedOutputStream> coded_output;
file_stream.reset(new FileOutputStream(file_handle));
file_stream->SetCloseOnDelete(true);
coded_output.reset(new CodedOutputStream(file_stream.get()));
uint32_t idx = 0;
for (; idx < records_limit; ++idx)
{
std::ostringstream data;
data << "iteration " << idx;
const std::string & str = data.str();
int coded_size = CodedOutputStream::VarintSize32(str.length()) + str.length();
uint8_t * buffer = coded_output->GetDirectBufferForNBytesAndAdvance(coded_size);
if (buffer != NULL)
{
buffer = CodedOutputStream::WriteVarint32ToArray(str.length(), buffer);
buffer = CodedOutputStream::WriteRawToArray(str.c_str(), str.length(), buffer);
}
else
{
coded_output->WriteVarint32(str.length());
coded_output->WriteString(str);
}
if (coded_output->HadError())
{
std::cerr << "error on iteration " << idx << std::endl;
break;
}
}
/*coded_output.reset();
if (!file_stream->Close())
{
std::cerr << "error " << file_stream->GetErrno() << " on close file" << std::endl;
}*/
std::cout << "written " << idx << " records, total bytes: " << file_stream->ByteCount() << std::endl;
}
std::fstream input_file(file_path, std::ios_base::in | std::ios_base::binary);
input_file.seekg(0, input_file.end);
size_t total_length = input_file.tellg();
input_file.seekg(0, input_file.beg);
std::auto_ptr<ZeroCopyInputStream> raw_input(new IstreamInputStream(&input_file));
std::auto_ptr<CodedInputStream> coded_input(new CodedInputStream(raw_input.get()));
uint32_t message_count = 0;
int error = 0;
while (!coded_input->ExpectAtEnd() && static_cast<size_t>(raw_input->ByteCount()) != total_length)
{
uint32_t msg_len;
if (!coded_input->ReadVarint32(&msg_len))
{
std::cerr << "failed to get length of next message" << std::endl;
error = -1;
break;
}
CodedInputStream::Limit limit = coded_input->PushLimit(msg_len);
std::string buf;
if (!coded_input->ReadString(&buf, msg_len))
{
std::cerr << "failed to parse message" << std::endl;
error = -1;
break;
}
message_count++;
//std::cout << buf << std::endl;
coded_input->PopLimit(limit);
}
if (error == 0)
{
std::cout << "test successfull" << std::endl;
}
std::cout << "total length: " << total_length
<< ", consumed bytes: " << raw_input->ByteCount()
<< ", messages count: " << message_count
<< std::endl;
input_file.close();
return error;
}
