Using python 2.7.6 on darwin and windows, both using protoc 2.5.0 and libs from 2.5.0, the python code on windows (see test.py in the attached zip) contains two extra bytes valued 0x0D in the message compared to the darwin version. These extra bytes are preventing c++ libprotobuf from parsing the file. Just to be sure it wasn't some null termination causing the parse to fail, I'm using ParsePartialFromArray in the exe. The same code run on darwin using the same version of python (CPython 2.7.6) and protobufs (2.5.0) generates the correct code and can be parsed with the windows and python c++ code. It appears only the python code on windows is producing incompatible messages.
Attached is minimal code and setup for reproducing the problem: Entity.proto (protobuf def) main.cpp (c++ parsing code) test.py (python code generating protobuf message) test_Darwin.pb (the output from the python file on darwin) test_Windows.pb (the output from the python file on windows) wscript (simple waf script for building these files) Any help would be greatly appreciated since this is a major issue for my project. Thanks! /andrew -- You received this message because you are subscribed to the Google Groups "Protocol Buffers" group. To unsubscribe from this group and stop receiving emails from it, send an email to protobuf+unsubscr...@googlegroups.com. To post to this group, send email to protobuf@googlegroups.com. Visit this group at http://groups.google.com/group/protobuf. For more options, visit https://groups.google.com/groups/opt_out.
test_Windows.pb
Description: Binary data
Entity.proto
Description: Binary data
#include "Entity.pb.h"
#define ASSERT(condition, message) { if ( !(condition) ) { printf("`" #condition "` failed in %s line %d : `" #message "`\n", __FILE__, __LINE__ ); std::exit(EXIT_FAILURE); } }
char* getFileString(const char* filename, size_t& size)
{
FILE* pFile = fopen(filename, "rb");
ASSERT( pFile != NULL, "Error opening file!" );
fseek (pFile , 0 , SEEK_END);
size = ftell (pFile);
rewind (pFile);
char* buffer = (char*) malloc ( sizeof(char) * size );
int result = fread( buffer, 1, size, pFile );
ASSERT( result == size, "Error reading file!" );
fclose (pFile);
return buffer;
}
int main(int argc, char **argv)
{
int res = 0;
if ( argc < 2 )
{
printf("Run with a file to run the test!\n");
return -1;
}
else
{
size_t size;
char* data = getFileString(argv[1], size);
Entity::Entities entitiesMsg;
bool res = entitiesMsg.ParsePartialFromArray(data, size);
if ( res )
{
printf("Succeeeded!\n");
}
else
{
printf("failed!\n");
return -1;
}
const google::protobuf::RepeatedPtrField<Entity::Entity>& entities = entitiesMsg.entities();
for (google::protobuf::RepeatedPtrField<Entity::Entity>::const_iterator entIT = entities.begin(); entIT != entities.end(); ++entIT )
{
const Entity::Entity& entityMsg = *entIT;
unsigned long long entityID = entityMsg.entityid();
printf("Entity %llu\n", entityID);
}
}
return 0;
}import sys
import Entity_pb2
def parse_list(values,message):
'''parse list to protobuf message'''
if len(values) > 0 and isinstance(values[0],dict):#value needs to be further parsed
for v in values:
cmd = message.add()
parse_dict(v,cmd)
else:#value can be set
message.extend(values)
def parse_dict(values,message):
for k,v in values.iteritems():
if isinstance(v,dict):#value needs to be further parsed
parse_dict(v,getattr(message,k))
elif isinstance(v,list):
parse_list(v,getattr(message,k))
else:#value can be set
try:
setattr(message, k, v)
except AttributeError:
logging.basicConfig()
l.warning('try to access invalid attributes %r.%r = %r',message,k,v)
def dict_to_protobuf(value,message):
parse_dict(value,message)
return message
def main():
data = {
"entities": [
{
"entityID": 1,
"components": [
{
"name": "ABC",
"data": ""
}
]
}
]
}
msg = dict_to_protobuf(data, Entity_pb2.Entities())
f = open(sys.argv[1], "w")
f.write(msg.SerializeToString())
f.close()
try:
main()
sys.exit(0)
except:
sys.exit(-1)
test_Darwin.pb
Description: Binary data
import os
import fnmatch
import subprocess
import sys
from contextlib import contextmanager
import copy
import sys
top = '.'
out = 'build'
APPNAME = 'testProtobufs'
VERSION = '0.0'
# For OSX (clang++):
"""
CXX=clang++ ./waf configure
./waf
./waf makeFromPython
./waf runTests
"""
# For windows (MSVC x86):
"""
python waf configure --msvc_targets="x86"
python waf
python waf makeFromPython
python waf runTets
"""
def options(opt):
opt.load('compiler_cxx')
def LoadLibs(cnf, libDict):
cnf.env.append_value("LibsToUse", libDict.keys())
for libUseName, libName in libDict.iteritems():
cnf.check(compiler='cxx',
lib=libName,
mandatory=True,
uselib_store=libUseName)
def winConfig(cnf):
cnf.load('compiler_cxx')
libs = {
"PROTOBUF": "libprotobuf",
}
LoadLibs(cnf, libs)
cnf.env.append_value('CXXFLAGS', ['/EHsc', '/MT'])
cnf.env.append_value('LINKFLAGS', [])
def darwinConfig(cnf):
cnf.load('compiler_cxx')
libs = {
"PROTOBUF": "protobuf",
}
LoadLibs(cnf, libs)
baseFlags = ['-stdlib=libstdc++', '-D', 'USE_JEM']
baseFlags.extend(['-O3'])
compileFlags = copy.copy(baseFlags)
compileFlags.extend(['-o'])
cnf.env.append_value('CXXFLAGS', compileFlags)
cnf.env.append_value('LINKFLAGS', baseFlags)
CONFIGS = {
"win32": winConfig,
"darwin": darwinConfig
}
def configure(cnf):
CONFIGS[sys.platform](cnf)
def RunAllInDir(directory, extensions, runInDir, excludes=[]):
excludedList = []
for root, dirs, files in os.walk(directory):
for extension in extensions:
for filename in fnmatch.filter(files, '*.' + extension):
fullName = os.path.join(root, filename)
if filename not in excludes:
runInDir(fullName)
else:
excludedList.append(fullName)
return excludedList
@contextmanager
def pushd(newDir):
previousDir = os.getcwd()
os.chdir(newDir)
yield
os.chdir(previousDir)
def build(bld):
subprocess.call(["protoc", "--version"])
protoBufBuild = ["protoc", "--cpp_out=.", "--python_out=.", "Entity.proto"]
exitCode = subprocess.call(protoBufBuild)
if exitCode != 0:
print "Proto buffers build step failed in dir " + dirName
sys.exit(1)
libs = bld.env.LibsToUse
bld.program(features=['cxx'],
source=["main.cpp", "Entity.pb.cc"],
includes=["protobufIncludes"],
target='testProtobufs',
use=libs,
)
def makeFromPython(bld):
subprocess.call(["python", "test.py", "test.pb"])
def runTests(bld):
if sys.platform == "win32":
subprocess.call([".\\build\\testProtobufs", "test.pb"])
else:
subprocess.call(["./build/testProtobufs", "test.pb"])
