[protobuf] Re: Reading and Writing using CodedOutputStream and CodedInputStream
I resolved this issue by slightly adjusting my logic, and changing the
way in which i deal with file handling.
Now I just have to get it to honor my append parameter so I can expand
the file, and eventually truncate it
Here is the current code:
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY | O_APPEND;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
FILE* fd = fopen(location.c_str(), w+);
go::FileOutputStream* zos = new go::FileOutputStream(fileno(fd));
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
const char* carr = data.c_str();
cos-WriteVarint32(1);
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
delete cos;
delete zos;
close(fileno(fd));
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY | O_APPEND;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
FILE* fd = fopen(location.c_str(), w+);
go::FileOutputStream* zos = new go::FileOutputStream(fileno(fd));
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
int ds = data.size();
cos-WriteVarint32(ds);
for (int i = 0; i ds; ++i) {
string st = data[i];
const char* carr = st.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
}
delete cos;
delete zos;
close(fileno(fd));
}
vectorstring FileLogger::read(const string location, const bool
purge) {
FILE* fd = fopen(location.c_str(), r);
go::FileInputStream* zis = new go::FileInputStream(fileno(fd));
go::CodedInputStream* cis = new go::CodedInputStream(zis);
bool flag = true;
vector string ret;
uint32_t num;
cis-ReadVarint32(num);
for (int i = 0; i num; i++) {
uint32_t size;
cis-ReadVarint32(size);
char* text = new char[size];
flag = cis-ReadRaw(text, size);
ret.push_back(string(text));
free(text);
}
delete cis;
delete zis;
close(fileno(fd));
return ret;
}
On Aug 29, 1:30 pm, Birch moa...@gmail.com wrote:
I am having some issues with the following three methods. I need the
ability to read and write any arbitrary protobuf message, that has
already been deserialized. It is assumed that I will always only read
or write the same type. So I have three methods; write(location,
data), write(location, vectordata), vectordataread(location).
Below is the code. If I call it with a single string, the first write
method, it seems to work ok. But when I call it with a vector, then
it failes saying the first readVarInt returns 20+.
Am I missing something obvious?
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
const char* carr = data.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
delete cos;
delete zos;
close(fd);
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
int ds = data.size();
for (int i = 0; i ds; ++i) {
string st = data[i];
const char* carr = st.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
}
delete cos;
delete zos;
close(fd);
}
vectorstring FileLogger::read(const string location, const bool
purge) {
int fd = open(location.c_str(), O_RDONLY);
go::FileInputStream* zis = new go::FileInputStream(fd);
go::CodedInputStream* cis = new go::CodedInputStream(zis);
bool flag = true;
vector string ret;
while (flag) {
uint32_t size;
cis-ReadVarint32(size);
if(0 ==
[protobuf] Re: Reading and Writing using CodedOutputStream and CodedInputStream
Ok, here is the final working code which handles the append and purge
boolean parameters.
If anyone can provide feedback, comments, suggestions, snotty remarks,
or anything, I would love to hear them. Especially love to know if
you have a more efficient way.
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
const char* carr = data.c_str();
cos-WriteVarint32(1);
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
delete cos;
delete zos;
close(fd);
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
int ds = data.size();
cos-WriteVarint32(ds);
for (int i = 0; i ds; ++i) {
string st = data[i];
const char* carr = st.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
}
delete cos;
delete zos;
close(fd);
}
vectorstring FileLogger::read(const string location, const bool
purge) {
int perms = O_RDONLY | O_CREAT | O_FSYNC | O_EXLOCK;
int fd = open(location.c_str(), perms);
go::FileInputStream* zis = new go::FileInputStream(fd);
go::CodedInputStream* cis = new go::CodedInputStream(zis);
bool flag = true;
vector string ret;
uint32_t num;
cis-ReadVarint32(num);
for (int i = 0; i num; i++) {
uint32_t size;
cis-ReadVarint32(size);
char* text = new char[size];
flag = cis-ReadRaw(text, size);
ret.push_back(string(text));
free(text);
}
delete cis;
delete zis;
close(fd);
if (purge) {
int perms = O_TRUNC | O_CREAT | O_FSYNC | O_EXLOCK;
int fd = open(location.c_str(), perms);
}
return ret;
}
On Aug 29, 1:30 pm, Birch moa...@gmail.com wrote:
I am having some issues with the following three methods. I need the
ability to read and write any arbitrary protobuf message, that has
already been deserialized. It is assumed that I will always only read
or write the same type. So I have three methods; write(location,
data), write(location, vectordata), vectordataread(location).
Below is the code. If I call it with a single string, the first write
method, it seems to work ok. But when I call it with a vector, then
it failes saying the first readVarInt returns 20+.
Am I missing something obvious?
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
const char* carr = data.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
delete cos;
delete zos;
close(fd);
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_CREAT;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
int ds = data.size();
for (int i = 0; i ds; ++i) {
string st = data[i];
const char* carr = st.c_str();
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
}
delete cos;
delete zos;
close(fd);
}
vectorstring FileLogger::read(const string location, const bool
purge) {
int fd = open(location.c_str(), O_RDONLY);
go::FileInputStream* zis = new go::FileInputStream(fd);
[protobuf] Re: Reading and Writing using CodedOutputStream and CodedInputStream
Ok, I incorporated some suggestions
from a someone who replied, but I still core dump in the following
method:
bool flag = cis-ReadRaw(ret.back(), size);
Any ideas?
Code is below.
Thanks!
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms, 0660);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
cos-WriteLittleEndian32(1);
root.debug(number of messages to write 1 byte);
root.debug(writing ...);
cos-WriteVarint32(data.size());
Utilities::Utilities* u = Utilities::getInstance();
root.debug(writing varint32 + u-intToString(data.size()));
cos-WriteRaw(data.data(), data.size());
delete cos;
delete zos;
close(fd);
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms, 0660);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
uint32_t ds = data.size();
cos-WriteLittleEndian32(ds);
Utilities::Utilities* u = Utilities::getInstance();
root.debug(number of messages to write + u-intToString(ds) +
byte(s));
for (int i = 0; i ds; ++i) {
string st = data[i];
root.debug(writing ...);
cos-WriteVarint32(st.size());
root.debug(writing varint32 + u-intToString(st.size()));
cos-WriteRaw(st, st.size());
}
delete cos;
delete zos;
close(fd);
}
vectorstring FileLogger::read(const string location, const bool
purge) {
int perms = O_RDONLY | O_CREAT | O_FSYNC | O_EXLOCK;
int fd = open(location.c_str(), perms, 0660);
go::FileInputStream* zis = new go::FileInputStream(fd);
go::CodedInputStream* cis = new go::CodedInputStream(zis);
vector string ret;
uint32_t num;
cis-ReadLittleEndian32(num);
Utilities::Utilities* u = Utilities::getInstance();
root.debug(number of messages to read + u-intToString(num));
for (int i = 0; i num; i++) {
uint32_t size;
root.debug(reading...);
cis-ReadVarint32(size);
root.debug(read readvarint32 + u-intToString(size));
bool flag = cis-ReadRaw(ret.back(), size);
root.debug(read readraw + flag);
}
delete cis;
delete zis;
close(fd);
if (purge) {
int perms = O_TRUNC | O_CREAT | O_FSYNC | O_EXLOCK;
int fd = open(location.c_str(), perms);
}
return ret;
}
On Aug 29, 3:20 pm, Birch moa...@gmail.com wrote:
Ok, here is the final working code which handles the append and purge
boolean parameters.
If anyone can provide feedback, comments, suggestions, snotty remarks,
or anything, I would love to hear them. Especially love to know if
you have a more efficient way.
Birch
void FileLogger::write(const string location, const string data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
const char* carr = data.c_str();
cos-WriteVarint32(1);
cos-WriteVarint32(strlen(carr));
cos-WriteRaw(carr, strlen(carr));
delete cos;
delete zos;
close(fd);
}
void FileLogger::write(const string location, const vectorstring
data,
const bool append) {
int perms = O_WRONLY | O_CREAT | O_FSYNC | O_EXLOCK;
if (append) {
perms |= O_APPEND;
} else {
perms |= O_TRUNC;
}
int fd = open(location.c_str(), perms);
go::FileOutputStream* zos = new go::FileOutputStream(fd);
go::CodedOutputStream* cos = new go::CodedOutputStream(zos);
int ds = data.size();
cos-WriteVarint32(ds);
for (int i = 0; i ds; ++i) {
string st = data[i];
const char* carr = st.c_str();
cos-WriteVarint32(strlen(carr));
