Dear, I am working on a project about Serialization/Deserialisation using Avro Java and Avro C++>
I start serializing a file from .JSON and .ascv using Avro Java into a
binary file : .avro file.
What i want to do now is to store schema - keys - values from this binary
file and to use them after in a matlab program to create new object with
these keys and values.
I've my avro file ("avroProgram2.avro"), i have made a little program to
recuperate schema, keys ("user_final.cpp") and i can access to datas with
the automatic file .hh generated by avrogen and use for example c::User C;
C.values[] ...
Ther final output : "record_final.txt"
But this way is no generical and i want to know what functions i can use to
better parse my file.
With just the binary .avro file, i would like to know what function can
find the schema and return it, if its possible to use a function for having
keys (in my case, i would like: name color number1 number2 number3 number4
), how i can access to datas (in the end of the file) without C.value1 ,
C.value2 ... but maybe with an array of value ?
And in a case i'm using more records, it's that possible to know how many
records there are in the file (information at the beginning of the avro
file) ?
Thanks for your understanding,
Regards,
Nicolas Ranc
{"type": "record",
"name": "User",
"namespace": "firstProgram.avro",
"fields": [
{"name": "name", "type": "string"},
{"name": "color", "type": "string"},
{"name": "number1", "type": "int"},
{"name": "number2", "type": "int"},
{"name": "number3", "type": "int"},
{"name": "number4", "type": "int"}
]
}
Keys are: name color number1 number2 number3 number4 #include "user.hh"
#include <fstream>
#include <stdio.h>
#include <stdlib.h>
#include <vector>
#include "avro/Encoder.hh"
#include "avro/Decoder.hh"
#include "avro/ValidSchema.hh"
#include "avro/Specific.hh"
#include "avro/Compiler.hh"
#include "avro/DataFile.hh"
#include <rapidjson/document.h>
#include <rapidjson/filereadstream.h>
using namespace rapidjson;
avro::ValidSchema loadSchema(const char* filename)
{
std::ifstream ifs(filename);
avro::ValidSchema result;
avro::compileJsonSchema(ifs, result);
return result;
}
int main(){
avro::ValidSchema mySchema = loadSchema("user.json");
rapidjson::Document document;
document.Parse("user.json");
std::string div,ligne;
char c;
bool IN=false,FIELDS=false,BETWEEN=false,IN2=false,FIELDS2=false;
std::vector<std::string> key,value;
std::vector<int>::iterator keyit,valueit;
char noend,end;
//Bin File
std::ifstream myfile("avroProgram2.avro");
std::string line;
if (myfile.is_open())
{
while ( myfile.get(c) )
{
if(c=='{')
IN=true;
if(IN==true)
{
if(c=='}')
noend='}';
if(c==']')
{
if(noend=='}')
{
div=div+'\n'+'\t';
}
}
if(c==']')
end=c;
if(c=='}')
{
if(end==']')
{
div=div+'\n';
IN=false;
}
}
div=div+c; //Char add
if(c==':')
div=div+' ';
if(c=='[')
{
div=div+'\n'+'\t';
FIELDS=true;
}
if(FIELDS==true)
{
if(c=='{')
BETWEEN=true;
if(c=='}')
BETWEEN=false;
}
if(BETWEEN==true)
{
if(c==',')
{
div=div+' ';
}
}
if(c==',')
{
if(BETWEEN==false)
div=div+'\n';
if(FIELDS==true && BETWEEN==false)
div=div+'\t';
}
}
}
std::cout<<div<<std::endl;
std::ofstream out("record.avro");
out<<div<<std::endl;
out.close();
std::ifstream record("record.avro");
while(record.get(c) )
{
std::string word;
if(c=='{')
IN2=true;
if(IN2==true)
{
if(c=='[')
FIELDS2=true;
if(FIELDS2==true)
{
if(c=='{')
{
record.ignore(5,'"');record.ignore(5,'"');record.ignore(5,'"');
while(record.get(c))
{
if(c!='"')
{word=word+c;}
if(c=='"')
{
key.push_back(word);
break;
}
}
}
}
}
}
std::cout<<"Keys are: ";
for (unsigned i=0; i<key.size(); i++)
{
std::cout<<key[i]<<' ';
}
myfile.close();
}
while(myfile)
{
std::string strInput;
myfile >> strInput;
}
//Read file
avro::DataFileReader<c::User> dfr("avroProgram2.avro",mySchema);
c::User u1;
/*while(dfr.read(u1)){
std::cout<<"( name :"<<u1.name<<" ),"<<std::endl;
std::cout<<"( color :"<<u1.color<<" ),"<<std::endl;
std::cout<<"( number1 :"<<u1.number1<<" ),"<<std::endl;
std::cout<<"( number2 :"<<u1.number2<<" ),"<<std::endl;
std::cout<<"( number3 :"<<u1.number3<<" ),"<<std::endl;
std::cout<<"( number4 :"<<u1.number4<<" )"<<std::endl;
}*/
return 0;
}
avroProgram2.avro
Description: Binary data
