Sorry, I am a bit swamped, thus the sluggish reply. Will be on the road most of tomorrow and Friday as well (with Monday being a public holiday over here). This sounds interesting. I just want to make sure you have seen the all-json property which generates a json format. I am not sure if that is useful for your case... Will have a better review asap.
Rainer > -----Original Message----- > From: [email protected] [mailto:rsyslog- > [email protected]] On Behalf Of Christian Brunner > Sent: Tuesday, June 07, 2011 9:43 PM > To: [email protected] > Subject: [rsyslog] [PATCH] a json strgen module > > I was playing with liblognorm and a document-oriented database that is > taking input in json format. Therefore I wanted to generate the json > messages directly in rsyslog. > > With this module you can generate json messages and send it through any of > the rsyslog output modules. > > The module isn't completely finished yet, but it is working and I would like to > get some feedback. > > Thanks > Christian > --- > Makefile.am | 5 + > configure.ac | 21 ++++ > doc/smjson.html | 64 +++++++++++ > plugins/smjson/Makefile.am | 6 + > plugins/smjson/smjson.c | 262 > ++++++++++++++++++++++++++++++++++++++++++++ > 5 files changed, 358 insertions(+), 0 deletions(-) create mode 100644 > doc/smjson.html create mode 100644 plugins/smjson/Makefile.am create > mode 100644 plugins/smjson/smjson.c > > diff --git a/Makefile.am b/Makefile.am > index d689b9e..3a3db59 100644 > --- a/Makefile.am > +++ b/Makefile.am > @@ -208,6 +208,10 @@ if ENABLE_ORACLE > SUBDIRS += plugins/omoracle > endif > > +if ENABLE_SMJSON > +SUBDIRS += plugins/smjson > +endif > + > if ENABLE_GUI > SUBDIRS += java > endif > @@ -253,5 +257,6 @@ DISTCHECK_CONFIGURE_FLAGS= --enable- > gssapi_krb5 \ > --enable-imtemplate \ > --enable-omtemplate \ > --enable-mmsnmptrapd \ > + --enable-smjson \ > --with- > systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) > ACLOCAL_AMFLAGS = -I m4 > diff --git a/configure.ac b/configure.ac index f6a09fa..d3307d0 100644 > --- a/configure.ac > +++ b/configure.ac > @@ -1220,6 +1220,25 @@ AM_CONDITIONAL(ENABLE_OMMONGODB, test > x$enable_ommongodb = xyes) # end of copy template - be sure to search > for omtemplate to find everything! > > > +# SMJSON SUPPORT > + > +AC_ARG_ENABLE(smjson, > + [AS_HELP_STRING([--enable-smjson],[Compiles smjson strgen module > @<:@default=no@:>@])], > + [case "${enableval}" in > + yes) enable_smjson="yes" ;; > + no) enable_smjson="no" ;; > + *) AC_MSG_ERROR(bad value ${enableval} for --enable-smjson) ;; > + esac], > + [enable_smjson=no] > +) > +# > +# you may want to do some library checks here - see snmp, mysql, pgsql > +modules # for samples # AM_CONDITIONAL(ENABLE_SMJSON, test > +x$enable_smjson = xyes) # end of copy template - be sure to search for > +omtemplate to find everything! > + > + > AC_CONFIG_FILES([Makefile \ > runtime/Makefile \ > tools/Makefile \ > @@ -1262,6 +1281,7 @@ AC_CONFIG_FILES([Makefile \ > plugins/omoracle/Makefile \ > plugins/omudpspoof/Makefile \ > plugins/mmnormalize/Makefile \ > + plugins/smjson/Makefile \ > plugins/sm_cust_bindcdr/Makefile \ > plugins/mmsnmptrapd/Makefile \ > plugins/cust1/Makefile \ > @@ -1317,6 +1337,7 @@ echo " mmsnmptrapd module will be compiled: > $enable_mmsnmptrapd" > echo > echo "---{ strgen modules }---" > echo " sm_cust_bindcdr module will be compiled: > $enable_sm_cust_bindcdr" > +echo " smjsonmodule will be compiled: $enable_smjson" > echo > echo "---{ database support }---" > echo " MySql support enabled: $enable_mysql" > diff --git a/doc/smjson.html b/doc/smjson.html new file mode 100644 index > 0000000..870163b > --- /dev/null > +++ b/doc/smjson.html > @@ -0,0 +1,64 @@ > +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> > +<html><head> <meta http-equiv="Content-Language" content="en"> > +<title>JSON Strgen Module</title></head> > + > +<body> > +<a href="rsyslog_conf_modules.html">back</a> > + > +<h1>JSON Strgen Module</h1> > +<p><b>Module Name: smjson</b></p> > +<p><b>Author: Christian Brunner <[email protected]></b></p> > +<p><b>Description</b>:</p> <p>Provides the ability to format syslog > +messages in JSON syntax. > +This module uses the <a target="_blank" > +href="http://libestr.adiscon.com/";> > +libestr</a> and the <a target="_blank" > +href="http://www.libee.org/";>libee</a> library. In order to compile > +this module, you will need to have the corresponding developer > +(headers) package installed. </p> <p> </p> <p><b>Configuration > +Directives</b>:</p> <ul> > + <li><strong>$smjsonadd key,value</strong><br> > + Add a key/value pair to the generated message.<br> > + transport types which are supported by NET-SNMP. <br> > + <strong>key</strong> is an arbitrary text string.<br> > + <strong>value</strong> can be a rsyslog template string (has to start > + and end with %) or a text string.<br> > + <br> > + Example: <strong>$smjsonadd date,%TIMESTAMP:::date- > rfc3339%<br> > + </strong></li> > +</ul> > +<p> </p> > +<p><b>Caveats/Known Bugs:</b></p><ul><li>Json output will be in > reverse > +order. </li></ul> <p><b>Sample:</b></p> <p>To generate a logfile with > +json messages like this:</p> > + > +<textarea rows="10" cols="60"> {"type": "SYSLOG", "cee":{"srcport": > +"54161", "srcip": "192.168.2.195", "user": "chb"}, "msg": "Accepted > +publickey for chb from 192.168.2.195 port 54161 ssh2", "hostname": > +"sir", "facility": "authpriv", "level": "info", "date": > +"2011-06-07T21:20:48+02:00"} </textarea> > + > +<p>You will need the following commands:</p> <textarea rows="10" > +cols="60">$ModLoad omsnmp > + > +$ModLoad smjson.so > +$template json,=SMJSON > +$smjsonadd date,%TIMESTAMP:::date-rfc3339% $smjsonadd > +level,%syslogpriority-text% $smjsonadd facility,%syslogfacility-text% > +$smjsonadd hostname,%hostname% $smjsonadd msg,%msg% $smjsonadd > +cee,%$!all-json% $smjsonadd type,SYSLOG > +*.* /var/tmp/json.log;json > + > +</textarea> > + > +<p>The example above is using mmnormalize to normalize the log message > +($!all-json).</p> > + > +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a > +href="manual.html">manual index</a>] [<a > +href="http://www.rsyslog.com/";>rsyslog site</a>]</p> <p><font > +size="2">This documentation is part of the <a > +href="http://www.rsyslog.com/";>rsyslog</a> project.<br> Copyright (c) > +2008 by <a href="http://www.gerhards.net/rainer";>Rainer Gerhards</a> > +and <a href="http://www.adiscon.com/";>Adiscon</a>. Released under > the > +GNU GPL version 3 or higher.</font></p> > + > +</body></html> > diff --git a/plugins/smjson/Makefile.am b/plugins/smjson/Makefile.am new > file mode 100644 index 0000000..c0e9327 > --- /dev/null > +++ b/plugins/smjson/Makefile.am > @@ -0,0 +1,6 @@ > +pkglib_LTLIBRARIES = smjson.la > + > +smjson_la_SOURCES = smjson.c > +smjson_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) > $(RSRT_CFLAGS) > +$(LIBEE_CFLAGS) smjson_la_LDFLAGS = -module -avoid-version > +$(LIBEE_LIBS) smjson_la_LIBADD = > diff --git a/plugins/smjson/smjson.c b/plugins/smjson/smjson.c new file > mode 100644 index 0000000..9d0b7d3 > --- /dev/null > +++ b/plugins/smjson/smjson.c > @@ -0,0 +1,262 @@ > +/* smjson.c > + * > + * A strgen module to transform log messages into the json format. > + * > + * File begun on 2011-06-02 by Christian Brunner <[email protected]> > + * > + * Rsyslog is free software: you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation, either version 3 of the License, or > + * (at your option) any later version. > + * > + * Rsyslog is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. > + * > + * A copy of the GPL can be found in the file "COPYING" in this distribution. > + */ > + > +#include "config.h" > +#include "rsyslog.h" > +#include <stdlib.h> > +#include <ctype.h> > +#include <string.h> > +#include <assert.h> > +#include <errno.h> > +#include "conf.h" > +#include "syslogd-types.h" > +#include "cfsysline.h" > +#include "template.h" > +#include "msg.h" > +#include "module-template.h" > +#include "unicode-helper.h" > +#include "errmsg.h" > + > +#include <libee/libee.h> > +#include <libee/field.h> > + > +MODULE_TYPE_STRGEN > +MODULE_TYPE_NOKEEP > +STRGEN_NAME("SMJSON") > + > +/* internal structures > + */ > +DEF_SMOD_STATIC_DATA > +DEFobjCurrIf(errmsg) > + > +typedef struct jsonentry_s { > + char *key; > + struct ee_field *constfield; > + struct templateEntry *pTpe; > + struct jsonentry_s *next; > +} jsonentry_t; > + > +static jsonentry_t *root; > +static ee_ctx ctx; > + > +static rsRetVal addJsonEntry(void __attribute__((unused)) *pVal, uchar > +*pNewVal) { > + jsonentry_t *pNew; > + DEFiRet; > + > + struct template *pTpl; > + struct templateEntry *pTpe; > + > + char *key, *value; > + es_str_t *estr; > + > + struct ee_value *eevalue; > + > + key = (char *) pNewVal; > + if (!(value = strchr((char *) pNewVal, ','))) { > + errmsg.LogError(0, NO_ERRCODE, "error: key/value separator > " > + "not found in '%s'", pNewVal); > + ABORT_FINALIZE(RS_RET_ERR); > + } > + *value = '\0'; > + value++; > + > + CHKmalloc(pNew = malloc(sizeof(jsonentry_t))); > + pNew->key = key; > + > + if (value[0] == '%') { > + unsigned char *templateString; > + CHKmalloc(templateString = malloc(strlen(value) + 3)); > + templateString[0] = '"'; > + memcpy(templateString+1, value, strlen(value)); > + memcpy(templateString+1+strlen(value), "\"", 2); > + > + pTpl = tplAddLine("smjson-intern", &templateString); > + pTpe = pTpl->pEntryRoot; > + > + pNew->constfield = NULL; > + pNew->pTpe = pTpe; > + > + free(templateString); > + } else { > + pNew->constfield = ee_newField(ctx); > + pNew->constfield->name = es_newStrFromBuf(key, > strlen(key)); > + estr = es_newStrFromBuf(value, strlen(value)); > + eevalue = ee_newValue(ctx); > + ee_setStrValue(eevalue, estr); > + ee_addValueToField(pNew->constfield, eevalue); > + > + pNew->pTpe = NULL; > + } > + > + pNew->next = root; > + root = pNew; > + > + DBGPRINTF("smjson: key/value '%s':'%s' added.\n", key, value); > + > +finalize_it: > + if(iRet != RS_RET_OK) { > + free(pNewVal); > + } > + > + RETiRet; > +} > + > + > +/* This strgen uses libee to generate the output string. > + */ > + > +#define JSON_END "\n" > +BEGINstrgen > + es_str_t *json; > + es_str_t *estr; > + struct ee_field *field; > + struct ee_value *eevalue; > + propid_t propID; > + size_t propLen, jsonLen; > + uchar *pszProp = NULL; > + unsigned short bMustBeFreed = 0; > + char *jsonStr; > + > + char *key; > + > + jsonentry_t *iter; > + struct templateEntry *pTpe; > + > + if((json = es_newStr(256)) == NULL) goto finalize_it; > + > + es_addChar(&json, '{'); > + > + for(iter = root; iter != NULL; iter = iter->next) > + { > + key = iter->key; > + > + if (iter->constfield) { > + ee_addField_JSON(iter->constfield, &json, 0); > + > + goto nextiter; > + } > + > + pTpe = iter->pTpe; > + propID = pTpe->data.field.propid; > + > + if(propID == PROP_CEE_ALL_JSON) { > + char *ceestr; > + > + es_addChar(&json, '"'); > + es_addBuf(&json, key, strlen(key)); > + es_addBuf(&json, "\":", 2); > + ee_fmtEventToJSON(pMsg->event, &estr); > + ceestr = es_str2cstr(estr, "#000"); > + es_deleteStr(estr); > + es_addBuf(&json, ceestr, strlen(ceestr)); > + free(ceestr); > + > + goto nextiter; > + } > + > + field = ee_newField(ctx); > + field->name = es_newStrFromBuf(key, strlen(key)); > + > + pszProp = MsgGetProp(pMsg, pTpe, propID, > + pTpe->data.field.propName, &propLen, > &bMustBeFreed); > + > + if(propID == PROP_MSG && pszProp[0] == ' ') { > + estr = es_newStrFromBuf((char *) pszProp+1, > + strlen((char *) pszProp)-1); > + } else { > + estr = es_newStrFromBuf((char *) pszProp, > + strlen((char *) pszProp)); > + } > + > + eevalue = ee_newValue(ctx); > + ee_setStrValue(eevalue, estr); > + ee_addValueToField(field, eevalue); > + > + ee_addField_JSON(field, &json, 0); > + ee_deleteField(field); > + > + if(bMustBeFreed) { > + free(pszProp); > + bMustBeFreed = 0; > + } > + > +nextiter: > + if (iter->next) > + es_addBuf(&json, ", ", ee_ctxIsEncUltraCompact(ctx) > ? 1 : 2); > + } > + > + es_addChar(&json, '}'); > + > + jsonStr = (char*) es_str2cstr(json, "#000"); > + jsonLen = strlen(jsonStr); > + > + es_deleteStr(json); > + > + if((jsonLen + sizeof(JSON_END)) >= *pLenBuf) > + CHKiRet(ExtendBuf(ppBuf, pLenBuf, jsonLen + > + sizeof(JSON_END))); > + > + memcpy(*ppBuf, jsonStr, jsonLen); > + memcpy(*ppBuf + jsonLen, JSON_END, sizeof(JSON_END)); > + > + free(jsonStr); > + ee_exitCtx(ctx); > + > +finalize_it: > + if(bMustBeFreed) > + free(pszProp); > +ENDstrgen > + > +BEGINmodExit > + jsonentry_t *iter, *jdel; > +CODESTARTmodExit > + for(iter = root; iter != NULL; ) { > + jdel = iter; > + iter = iter->next; > + if (jdel->constfield) { > + ee_deleteField(jdel->constfield); > + } > + free(jdel->key); > + } > + objRelease(errmsg, CORE_COMPONENT); > +ENDmodExit > + > + > +BEGINqueryEtryPt > +CODESTARTqueryEtryPt > +CODEqueryEtryPt_STD_SMOD_QUERIES > +ENDqueryEtryPt > + > +BEGINmodInit() > +CODESTARTmodInit > + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the > +current interface specification */ CODEmodInit_QueryRegCFSLineHdlr > + CHKiRet(objUse(errmsg, CORE_COMPONENT)); > + > + root = NULL; > + ctx = ee_initCtx(); > + > + CHKiRet(omsdRegCFSLineHdlr((uchar *)"smjsonadd", 0, > eCmdHdlrGetWord, > + addJsonEntry, NULL, STD_LOADABLE_MODULE_ID, > + eConfObjGlobal)); > + > + dbgprintf("rsyslog json strgen init called, compiled with > +version %s\n", VERSION); ENDmodInit > -- > 1.7.1 > > _______________________________________________ > rsyslog mailing list > http://lists.adiscon.net/mailman/listinfo/rsyslog > http://www.rsyslog.com _______________________________________________ rsyslog mailing list http://lists.adiscon.net/mailman/listinfo/rsyslog http://www.rsyslog.com
