Update: so AMQ can support binary data if its content-length field in the
headers is used.
If I take the stomp client's string value and convert it to a buffer using
a "binary" encoding (rather than the default utf-8/ucs-2), then I seem to
be able to create the proper GPB object.
sample code (replace utils with console.log):
gpbencodedecodetest.js:
let fs = require('fs');
const path = require('path');
var protobuf = require("protobufjs");
const utils = require('../utils/testutils'); << just use console.log for
these
let TEST_PROTO_FILE_NAME = 'Test.proto';
let protoDir = path.normalize(__dirname);
console.log(`Proto dir [${protoDir}] exists? ${fs.existsSync(protoDir)}`);
let TestRespGPBDef = null;
(async ()=> {
try {
TestRespGPBDef = await asyncLoadGBP(protoDir,
TEST_PROTO_FILE_NAME,'testt.TestResponse');
//build payload object with just 15 of the fields
let payload = {
myvaluea1: 'A',
myvalueb2: 'B',
myvaluec3: 'C',
myvalued4: 'D',
myvaluee5: 'E',
myvaluef6: 'F',
myvalueg7: 'G',
myvalueh8: 'H',
myvaluei9: 'I',
myvaluej10: 'J',
myvaluek11: 'K',
myvaluel12: 'L',
myvaluem13: 'M',
myvaluen14: 'N',
myvalueo15: 'O',
//myvaluep16: 'P',
};
testEncodeAndDecodeToFromGPB(payload);
console.log('++++++++++++++++++++++++++++++++++++++++++++++++++++++++');
// repeat, but with 16 fields set, this one fails
payload.myvaluep16 = 'P';;
testEncodeAndDecodeToFromGPB(payload);
} catch (exc) {
console.log(`ERROR`,exc);
}
})();
/**
* This function takes a payload object literal and converts it into
* a GPB object.
*
* It then encodes the GPB object into a buffer and then into a
* string (that string would normally then be
* sent over the AMQ stomp client).
*
* It then takes the string and tries to rebuild
* the GPB object by converting the string back into a buffer.
* The buffer is then decoded into a GPB object.
*
* @param {*} payload object to be converted to a GPB
*/
function testEncodeAndDecodeToFromGPB(payload) {
utils.printObj(`payload`,payload);
// verify that payload is legit
let verifyErr = TestRespGPBDef.verify(payload);
if( verifyErr ) {
console.log(`GPB OBJECT/PAYLOAD ERROR`, {verifyErr});
process.exit(-1);
}
// create GPB object from payload
let gpbObj1 = TestRespGPBDef.create(payload);
utils.printObj(`Gpb obj1`,gpbObj1);
// encode GPB into a byte buffer
let buff1 = TestRespGPBDef.encode(gpbObj1).finish();
utils.printObj(`\ngmsg encodded as buff (len ${buff1.length})`, buff1);
// we should be able to decode buffer back into a GPB objerct
let gpbObj2 = TestRespGPBDef.decode(buff1);
utils.printObj(`sanity check, Gpb obj2`, gpbObj2);
// we need to convert to a string before sending to an AMQ stream
// supposed to encode an array to utf8 string, but eh not completely
let gmsgAsStr = buff1.toString('binary');
//let gmsgAsStr = buff1.toString('utf8');
console.log(`\n>>>>>>>gmsg as string (len ${gmsgAsStr.length})
[${gmsgAsStr}]`);
utils.printObj(`gmsg as string (len ${gmsgAsStr.length})`,gmsgAsStr);
let teArray = new TextEncoder().encode(gmsgAsStr);
console.log(`gmsg str encoded by node (len ${teArray.length})`,teArray);
//let barray = utils.stringToBinArray(gmsgAsStr);
//console.log(`gmsg as supposed binary array (len ${barray.length})`,
barray);
// now lets take the string and try to reconstruct the GPB object
// from a string, first we need to create a buffer
let buff2 = Buffer.from(gmsgAsStr,'binary'); // ERROR RangeError: index out
of range: 47 + 4 > 49
//let buff2 = Buffer.from(gmsgAsStr); // ERROR Error: invalid wire type 7
at offset 49
utils.printObj(`\nbuff2 from string (len ${buff2.length})`, buff2);
// and from the buffer we should be able to re-create gpb obj
let gpbObj3 = TestRespGPBDef.decode(buff2);
utils.printObj(`\nsanity check 3, resulting GPB`,gpbObj3);
console.log('w00t!');
}
/**
* Load GPB proto file into a defintion object
* @param protoDir directory name of protofile
* @param protoFile name of protofile
* @param message definition name of proto you are interested in from within
the file
* @return Promise
* @resolves GPB object
* @rejects if file cannot be read/parsed/loaded
*/
async function asyncLoadGBP(protoDir, protoFile, protoMsgName) {
return new Promise( ((resolve,reject) => {
let retGpbTypeDef = null;
let gpbAbsFName = protoDir + '/' + protoFile;
protobuf.load(gpbAbsFName, function(err, root) {
if (err) {
let e = `ERROR LOADING GBP [${gpbAbsFName}] +
${utils.getObjAsString(err)}]`;
console.log(e);
reject(e);
}
console.log(`GPB [${gpbAbsFName}] loaded ok.`);
try {
retGpbTypeDef = root.lookupType(protoMsgName);
console.log(`GPB Message Type [${protoMsgName}] look up ok.`);
} catch( luExc) {
let e = `ERROR LOOKING UP GBP TYPE [${protoMsgName}]
[${utils.getObjAsString(luExc)}]`;
console.log(e);
reject(e);
}
resolve(retGpbTypeDef);
});
})); // end promise
}
Test.proto:
syntax = "proto2";
package testt;
message TestResponse
{
//optional Monkey header = 16;
optional string myvaluea1 = 1;
optional string myvalueb2 = 2;
optional string myvaluec3 = 3;
optional string myvalued4 = 4;
optional string myvaluee5 = 5;
optional string myvaluef6 = 6;
optional string myvalueg7 = 7;
optional string myvalueh8 = 8;
optional string myvaluei9 = 9;
optional string myvaluej10 = 10;
optional string myvaluek11 = 11;
optional string myvaluel12 = 12;
optional string myvaluem13 = 13;
optional string myvaluen14 = 14;
optional string myvalueo15 = 15;
optional string myvaluep16 = 16;
}
On Monday, January 11, 2021 at 6:51:32 AM UTC-7 peo stri tise safe wrote:
> Yes, I agree, thank you very much for your input. The incoming data is
> being treated as a string by the node AMQ stomp client library, and,
> attempting to convert that string to a binary array is what is not working,
> (thus the funky 65533 value). I either need to find a way for the AMQ
> client to accept in-coming data as binary or over-ride the charCodeAt to
> truly return a data byte. Eesh. Thanks again!
>
> On Monday, January 11, 2021 at 2:13:59 AM UTC-7 [email protected] wrote:
>
>> hex "82 01" is indeed the expected field header for field 16,
>> length-prefixed (i.e. wire type 2); further, the wider hex "12 02 68 69 1a
>> 05 74 68 65 72 65 82 01 0c 0a 0a 6d 6f 6e 6b 65 79 62 75 74 74" is
>> perfectly well-formed:
>>
>> [image: image.png]
>>
>> (via https://protogen.marcgravell.com/decode)
>>
>> So: if it isn't working, there are two likely scenarios:
>>
>> 1. the serializer has a bug
>> 2. the data is being corrupted during processing, perhaps (usually) by
>> treating it as text data rather than binary (meaning: the buffer you're
>> passing to the serializer doesn't actually contain hex "12 02 68 69 1a 05
>> 74 68 65 72 65 82 01 0c 0a 0a 6d 6f 6e 6b 65 79 62 75 74 74"
>>
>> In my experience, option 2 is *vastly* more common than option 1
>>
>> Marc
>>
>> On Fri, 8 Jan 2021 at 15:06, peo stri tise safe <[email protected]>
>> wrote:
>>
>>> More info-- when the "header" is set with an index of >= 16, the
>>> encoding process appears to insert an extra byte in the encoded value, and
>>> a decode attempt will fail, here is an example where an index of 15 can
>>> allow for an encode and subsequent decode, but an index of 16 fails at the
>>> subsequent decode:
>>>
>>> using a header with an index = 15
>>> obj to publish (typeof [object]): [TestResponse {
>>> header: { timestamp: 'monkeybutt' },
>>> mykey: 'hi',
>>> myvalue: 'there'
>>> }]
>>> encodded as buff (typeof [object]): [<Buffer 12 02 68 69 1a 05 74 68 65
>>> 72 65 7a 0c 0a 0a 6d 6f 6e 6b 65 79 62 75 74 74>]
>>> encoded as str (typeof [string]):
>>> ['\u0012\u0002hi\u001a\u0005therez\f\n\nmonkeybutt']
>>> sanBinArray (typeof [object]): [[
>>> 18, 2, 104,
>>> 105, 26, 5,
>>> 116, 104, 101,
>>> 114, 101, 122,
>>> 12, 10, 10,
>>> 109, 111, 110,
>>> 107, 101, 121,
>>> 98, 117, 116,
>>> 116, [length]: 25
>>> ]]
>>> Sanity check, decoding encoded msg: (typeof [object]): [TestResponse {
>>> mykey: 'hi',
>>> myvalue: 'there',
>>> header: Monkey { timestamp: 'monkeybutt' }
>>> }]
>>> pinging
>>> CONSUMER << RXD [25] elements (typeof [string]):
>>> ['\u0012\u0002hi\u001a\u0005therez\f\n\nmonkeybutt']
>>> As binArray (typeof [object]): [[
>>> 18, 2, 104,
>>> 105, 26, 5,
>>> 116, 104, 101,
>>> 114, 101, 122,
>>> 12, 10, 10,
>>> 109, 111, 110,
>>> 107, 101, 121,
>>> 98, 117, 116,
>>> 116, [length]: 25
>>> ]]
>>> (typeof [object]): [TestResponse {
>>> mykey: 'hi',
>>> myvalue: 'there',
>>> header: Monkey { timestamp: 'monkeybutt' }
>>> }]
>>>
>>>
>>> using a header with index = 16
>>> obj to publish (typeof [object]): [TestResponse {
>>> header: { timestamp: 'monkeybutt' },
>>> mykey: 'hi',
>>> myvalue: 'there'
>>> }]
>>> encodded as buff (typeof [object]): [<Buffer 12 02 68 69 1a 05 74 68 65
>>> 72 65 82 01 0c 0a 0a 6d 6f 6e 6b 65 79 62 75 74 74>]
>>>
>>> == == <-- two bytes
>>>
>>> encoded as str (typeof [string]):
>>> ['\u0012\u0002hi\u001a\u0005there�\u0001\f\n\nmonkeybutt']
>>> sanBinArray (typeof [object]): [[
>>> 18, 2, 104,
>>> 105, 26, 5,
>>> 116, 104, 101,
>>> 114, 101, 65533, <---- suspicious!
>>> 1, 12, 10,
>>> 10, 109, 111,
>>> 110, 107, 101,
>>> 121, 98, 117,
>>> 116, 116, [length]: 26
>>> ]]
>>> Sanity check error: Could not decode msg {
>>> decodeExc: Error: invalid wire type 7 at offset 18
>>> at Reader.skipType (node_modules/protobufjs/src/reader.js:377:19)
>>> at Type.TestResponse$decode [as decode] (eval at Codegen
>>> (node_modules/@protobufjs/codegen/index.js:50:33), <anonymous>:20:5)
>>> at buildEncodedStr (test/utils/amqprototest.js:94:34)
>>> at Timeout._onTimeout (test/utils/amqprototest.js:57:17)
>>> at listOnTimeout (internal/timers.js:531:17)
>>> at processTimers (internal/timers.js:475:7)
>>> }
>>>
>>> On Friday, January 8, 2021 at 6:25:41 AM UTC-7 peo stri tise safe wrote:
>>>
>>>> Gotcha, so the index value of 100 is not a non-standard value.
>>>>
>>>> So I'm seeing "invalid wire type" when using the decode method of
>>>> protobuf.js of an AMQ body. This is in a Node using the protobuf.js,
>>>> stomp-client.js and an AMQ broker.
>>>>
>>>> The GPB that I'm using has a basic format of this:
>>>>
>>>> syntax = "proto2";
>>>> package testt;
>>>> message Monkey
>>>> {
>>>> optional string timestamp = 1;
>>>> }
>>>> message TestResponse
>>>> {
>>>> optional Monkey header = 100;
>>>> optional string mykey = 2;
>>>> optional string myvalue = 3;
>>>> }
>>>>
>>>> And the message is encoded and sent to the broker like this:
>>>>
>>>> // note TestRespGPBDef is created earlier
>>>> let payload = {
>>>> header: {
>>>> timestamp: 'monkeybutt'
>>>> },
>>>> mykey: 'hi',
>>>> myvalue: 'there'
>>>> };
>>>> let verifyErr = TestRespGPBDef.verify(payload);
>>>> if( verifyErr ) {
>>>> console.log(`GPB OBJECT/PAYLOAD ERROR`, {verifyErr});
>>>> return;
>>>> }
>>>> let gmsg = TestRespGPBDef.create(payload);
>>>> let buff = TestRespGPBDef.encode(gmsg).finish();
>>>> let str = buff.toString();
>>>> stompClient.publish('mytopic',str);
>>>>
>>>>
>>>> And the message is rd'x and decoded by the consumer like this:
>>>>
>>>> // note TestRespGPBDef is created earlier
>>>> client.subscribe('mytopic', function(body, headers) {
>>>> utils.printObj(`CONSUMER << RXD AMQ header and body`);
>>>> utils.printObj(`AMQ body\n [${body.length}] elements`, body);
>>>> let binArray = utils.stringToBinArray(body);
>>>> utils.printObj(`As binArray`,binArray);
>>>> try {
>>>> let gpbMsg = TestRespGPBDef.decode(binArray);
>>>> utils.printObj('',gpbMsg);
>>>> } catch (decodeExc) {
>>>> console.log(`Could not decode msg`,{decodeExc});
>>>> }
>>>> });
>>>>
>>>>
>>>> When the consumer receives the body from AMQ and tries to decode it,
>>>> this is the exception:
>>>>
>>>> CONSUMER << RXD AMQ header and body
>>>> AMQ body:
>>>> [26] elements (typeof [string]):
>>>> ['\u0012\u0002hi\u001a\u0005there�\u0006\f\n\nmonkeybutt']
>>>> As binArray (typeof [object]): [[
>>>> 18, 2, 104,
>>>> 105, 26, 5,
>>>> 116, 104, 101,
>>>> 114, 101, 65533,
>>>> 6, 12, 10,
>>>> 10, 109, 111,
>>>> 110, 107, 101,
>>>> 121, 98, 117,
>>>> 116, 116, [length]: 26
>>>> ]]
>>>> Could not decode msg {
>>>> decodeExc: Error: invalid wire type 7 at offset 18
>>>> at Reader.skipType
>>>> (/mytest/node_modules/protobufjs/src/reader.js:377:19)
>>>> at Type.TestResponse$decode [as decode] (eval at Codegen
>>>> (/mytest/node_modules/@protobufjs/codegen/index.js:50:33),
>>>> <anonymous>:20:5)
>>>> at
>>>> /Users/rcox/gitprojects/POSTAPOCALYPSE/speb-hmi/deviceModules/keplerAdapter/test/utils/amqprototest.js:37:39
>>>> at
>>>> /Users/rcox/gitprojects/POSTAPOCALYPSE/speb-hmi/deviceModules/keplerAdapter/node_modules/stomp-client/lib/client.js:206:9
>>>> at Array.map (<anonymous>)
>>>> at StompFrameEmitter.<anonymous>
>>>> (/mytest/node_modules/stomp-client/lib/client.js:205:28)
>>>> at StompFrameEmitter.emit (events.js:210:5)
>>>> at StompFrameEmitter.parseBody
>>>> (/mytest/node_modules/stomp-client/lib/parser.js:136:12)
>>>> at StompFrameEmitter.handleData
>>>> (/mytest/node_modules/stomp-client/lib/parser.js:42:12)
>>>> at Socket.<anonymous>
>>>> (/mytest/node_modules/stomp-client/lib/client.js:188:18)
>>>> }
>>>>
>>>> When the "header" index is set to 1, and the experiment is repeated,
>>>> the decoding does not throw the exception.
>>>> On Friday, January 8, 2021 at 12:52:16 AM UTC-7 [email protected]
>>>> wrote:
>>>>
>>>>> 100 isn't "non-standard" as such, and shouldn't cause anything to
>>>>> fail. What exactly are you seeing?
>>>>>
>>>>> The valid range is 1-536870911, omitting 19000-19999 (and any reserved
>>>>> areas in your specific messages) smaller numbers are cheaper (fewer
>>>>> bytes)
>>>>> to encode, so are usually preferred - but: that's it.
>>>>>
>>>>> I'm guessing you're actually using some non-compliant code that is
>>>>> assuming field-headers are single bytes? That is true for very low field
>>>>> numbers (4 bytes, so: 1-15), but field 100 will take two bytes for the
>>>>> header. Is that the problem here?
>>>>>
>>>>> On Thu, 7 Jan 2021, 22:00 peo stri tise safe, <[email protected]>
>>>>> wrote:
>>>>>
>>>>>> What is the purpose of setting starting index values for a GPB
>>>>>> message definition to something other than 1? I am seeing a value of
>>>>>> 100
>>>>>> in a particular application and it is causing the encoding of the
>>>>>> message
>>>>>> to fail. Just wondered why the use of values other than 1 for the first
>>>>>> item in the message.
>>>>>>
>>>>>> --
>>>>>> You received this message because you are subscribed to the Google
>>>>>> Groups "Protocol Buffers" group.
>>>>>> To unsubscribe from this group and stop receiving emails from it,
>>>>>> send an email to [email protected].
>>>>>> To view this discussion on the web visit
>>>>>> https://groups.google.com/d/msgid/protobuf/957aae3b-32f8-4d9d-8f5f-732e26e08cedn%40googlegroups.com
>>>>>>
>>>>>> <https://groups.google.com/d/msgid/protobuf/957aae3b-32f8-4d9d-8f5f-732e26e08cedn%40googlegroups.com?utm_medium=email&utm_source=footer>
>>>>>> .
>>>>>>
>>>>> --
>>> You received this message because you are subscribed to the Google
>>> Groups "Protocol Buffers" group.
>>> To unsubscribe from this group and stop receiving emails from it, send
>>> an email to [email protected].
>>>
>> To view this discussion on the web visit
>>> https://groups.google.com/d/msgid/protobuf/2a726cea-c829-4527-9878-232f746b32b8n%40googlegroups.com
>>>
>>> <https://groups.google.com/d/msgid/protobuf/2a726cea-c829-4527-9878-232f746b32b8n%40googlegroups.com?utm_medium=email&utm_source=footer>
>>> .
>>>
>>
>>
>> --
>> Regards,
>>
>> Marc
>>
>
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/protobuf/dfd8e4cd-cc7a-4952-83b4-f2bd59e860f5n%40googlegroups.com.
