Hi Chris,
Thanks for the patch and the feedback. Let me review so that we
see if we have something for the mainstream release.
Cheers,
Paolo
On Thu, Jun 05, 2014 at 12:35:53PM +0300, Chris Wilson wrote:
> Hi Paolo,
>
> On Thu, 5 Jun 2014, Paolo Lucente wrote:
>
> >>DEBUG ( default/amqp ): publishing [E=pmacct RK=acct DM=0]:
> >>{"timestamp_start": "2014-06-03 22:42:00.202820", "ip_dst":
> >>"196.223.145.xxx", "ip_proto": "tcp", "tos": 0, "ip_src":
> >>"86.30.131.xxx", "bytes": 142, "port_dst": 36363, "packets": 1,
> >>"port_src": 2201, "timestamp_end": "1970-01-01 03:00:00.0"}
> >>
> >>Is this a bug? Would it be easy to fix?
> >
> >This is not a bug. This is result of the fact a single packet has
> >a single timestamp (or two coinciding) hence only one of the two
> >values, timestamp_start, is populated.
>
> OK sorry, I found that out by rereading CONFIG-KEYS while trying to
> work out if there was any way to get the bucket start and end times
> into the JSON output.
>
> >Try to:
> >
> >* capture your own traffic with pmacctd attaching to it a
> > nfprobe plugin, the NetFlow/IPFIX probe plugin. Set the
> > export to localhost.
> >
> >* on localhost you bind nfacctd that listens for NetFlow/IPFIX
> > packets (generated by pmacctd/nfprobe) and writes wherever you
> > want to like with the aggregation you like (this time you will
> > see both timestamp_start and timestamp_end populated - as a
> > result of the flow-aware cache of nfprobe).
> >
> >This is the slightly more involved solution i was proposing, which
> >i don't know if you like or not (definitely good for a proof of
> >concept).
>
> What I have done so far is to modify pmacctd to send two extra
> timestamps: the start and end times of the history bucket. This is
> working well for me and it would be great to have something like
> this integrated into pmacct. Patch attached.
>
> I also had to modify sql_history to allow it to be set to 1 second
> intervals, which was previously blocked by a warning. I've included
> this part in the attached patch as well.
>
> Finally I changed the timestamps into GMT instead of local time.
>
> Cheers, Chris.
> --
> Aptivate | http://www.aptivate.org | Phone: +44 1223 967 838
> Citylife House, Sturton Street, Cambridge, CB1 2QF, UK
>
> Aptivate is a not-for-profit company registered in England and Wales
> with company number 04980791.
> Only in pmacct-1.5.0rc3-chris: config.cache
> Only in pmacct-1.5.0rc3-chris: config.log
> Only in pmacct-1.5.0rc3-chris: config.status
> Only in pmacct-1.5.0rc3-chris: Makefile
> Only in pmacct-1.5.0rc3-chris/src: acct.o
> Only in pmacct-1.5.0rc3-chris/src: addr.o
> diff -ru pmacct-1.5.0rc3/src/amqp_plugin.c
> pmacct-1.5.0rc3-chris/src/amqp_plugin.c
> --- pmacct-1.5.0rc3/src/amqp_plugin.c 2014-03-24 02:59:04.000000000 +0300
> +++ pmacct-1.5.0rc3-chris/src/amqp_plugin.c 2014-06-04 14:50:00.706672411
> +0300
> @@ -142,7 +142,7 @@
>
> if (config.sql_history) {
> while (now > (basetime.tv_sec + timeslot)) {
> - new_basetime.tv_sec = basetime.tv_sec;
> + new_basetime.tv_sec = basetime.tv_sec;
> basetime.tv_sec += timeslot;
> if (config.sql_history == COUNT_MONTHLY)
> timeslot = calc_monthly_timeslot(basetime.tv_sec,
> config.sql_history_howmany, ADD);
> @@ -341,7 +341,7 @@
> json_str = compose_json(config.what_to_count, config.what_to_count_2,
> queue[j]->flow_type,
> &queue[j]->primitives, pbgp, pnat, pmpls, pcust,
> queue[j]->bytes_counter,
> queue[j]->packet_counter, queue[j]->flow_counter,
> queue[j]->tcp_flags,
> - &queue[j]->basetime);
> + &queue[j]->basetime, &new_basetime);
>
> if (json_str) {
> if (is_routing_key_dyn)
> amqp_handle_routing_key_dyn_strings(config.sql_table, SRVBUFLEN,
> orig_amqp_routing_key,
> diff -ru pmacct-1.5.0rc3/src/cfg_handlers.c
> pmacct-1.5.0rc3-chris/src/cfg_handlers.c
> --- pmacct-1.5.0rc3/src/cfg_handlers.c 2014-03-19 01:27:42.000000000
> +0300
> +++ pmacct-1.5.0rc3-chris/src/cfg_handlers.c 2014-06-04 13:29:00.490676835
> +0300
> @@ -3564,6 +3564,7 @@
>
> k = atoi(value);
> if (k > 0) {
> + /*
> if (*mu == COUNT_SECONDLY) {
> if (k % 60) {
> Log(LOG_WARNING, "WARN ( %s ): Ignoring invalid time value: %d
> (residual secs afters conversion in mins)\n", filename, k);
> @@ -3574,6 +3575,7 @@
> *mu = COUNT_MINUTELY;
> }
> }
> + */
> *howmany = k;
> }
> else {
> diff -ru pmacct-1.5.0rc3/src/plugin_common.c
> pmacct-1.5.0rc3-chris/src/plugin_common.c
> --- pmacct-1.5.0rc3/src/plugin_common.c 2014-03-26 19:32:46.000000000
> +0300
> +++ pmacct-1.5.0rc3-chris/src/plugin_common.c 2014-06-04 14:21:16.874628272
> +0300
> @@ -612,7 +612,8 @@
> basetime.tv_sec = now;
> basetime.tv_usec = 0;
>
> - if (config.sql_history == COUNT_MINUTELY) timeslot =
> config.sql_history_howmany*60;
> + if (config.sql_history == COUNT_SECONDLY) timeslot =
> config.sql_history_howmany;
> + else if (config.sql_history == COUNT_MINUTELY) timeslot =
> config.sql_history_howmany*60;
> else if (config.sql_history == COUNT_HOURLY) timeslot =
> config.sql_history_howmany*3600;
> else if (config.sql_history == COUNT_DAILY) timeslot =
> config.sql_history_howmany*86400;
> else if (config.sql_history == COUNT_WEEKLY) timeslot =
> config.sql_history_howmany*86400*7;
> diff -ru pmacct-1.5.0rc3/src/util.c pmacct-1.5.0rc3-chris/src/util.c
> --- pmacct-1.5.0rc3/src/util.c 2014-03-16 19:20:44.000000000 +0300
> +++ pmacct-1.5.0rc3-chris/src/util.c 2014-06-04 19:29:59.318621575 +0300
> @@ -26,6 +26,7 @@
> #include "pmacct-data.h"
> #include "ip_flow.h"
> #include "classifier.h"
> +#include "plugin_common.h"
> #ifdef WITH_JANSSON
> #include <jansson.h>
> #endif
> @@ -1045,8 +1046,8 @@
> if (a->tv_sec == b->tv_sec) {
> if (a->tv_usec > b->tv_usec) return 1;
> if (a->tv_usec < b->tv_usec) return -1;
> - if (a->tv_usec == b->tv_usec) return 0;
> }
> + return 0;
> }
>
> /*
> @@ -1597,7 +1598,7 @@
> char *compose_json(u_int64_t wtc, u_int64_t wtc_2, u_int8_t flow_type,
> struct pkt_primitives *pbase,
> struct pkt_bgp_primitives *pbgp, struct pkt_nat_primitives
> *pnat, struct pkt_mpls_primitives *pmpls,
> char *pcust, pm_counter_t bytes_counter, pm_counter_t
> packet_counter, pm_counter_t flow_counter,
> - u_int32_t tcp_flags, struct timeval *basetime)
> + u_int32_t tcp_flags, struct timeval *timeslot_start, struct
> timeval *timeslot_end)
> {
> char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN],
> dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN];
> char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], *as_path, *bgp_comm,
> empty_string[] = "", *tmpbuf;
> @@ -2000,20 +2001,16 @@
> }
> }
>
> - if (basetime && config.sql_history) {
> - struct timeval tv;
> -
> - tv.tv_sec = basetime->tv_sec;
> - tv.tv_usec = 0;
> - compose_timestamp(tstamp_str, SRVBUFLEN, &tv, FALSE);
> - kv = json_pack("{ss}", "stamp_inserted", tstamp_str);
> + if (config.sql_history) {
> + compose_timestamp(tstamp_str, SRVBUFLEN, &basetime, FALSE);
> + kv = json_pack("{ss}", "timeslot_start", tstamp_str);
> json_object_update_missing(obj, kv);
> json_decref(kv);
>
> - tv.tv_sec = time(NULL);
> - tv.tv_usec = 0;
> + struct timeval tv = basetime;
> + tv.tv_sec += timeslot;
> compose_timestamp(tstamp_str, SRVBUFLEN, &tv, FALSE);
> - kv = json_pack("{ss}", "stamp_updated", tstamp_str);
> + kv = json_pack("{ss}", "timeslot_end", tstamp_str);
> json_object_update_missing(obj, kv);
> json_decref(kv);
> }
> @@ -2058,7 +2055,7 @@
> struct tm *time2;
>
> time1 = tv->tv_sec;
> - time2 = localtime(&time1);
> + time2 = gmtime(&time1);
> strftime(tmpbuf, SRVBUFLEN, "%Y-%m-%d %H:%M:%S", time2);
>
> if (usec) snprintf(buf, buflen, "%s.%u", tmpbuf, tv->tv_usec);
> Only in pmacct-1.5.0rc3-chris/src: .util.c.swp
> diff -ru pmacct-1.5.0rc3/src/util.h pmacct-1.5.0rc3-chris/src/util.h
> --- pmacct-1.5.0rc3/src/util.h 2013-11-08 05:43:01.000000000 +0300
> +++ pmacct-1.5.0rc3-chris/src/util.h 2014-06-04 14:46:31.726667060 +0300
> @@ -99,7 +99,8 @@
> EXT char *compose_json(u_int64_t, u_int64_t, u_int8_t, struct pkt_primitives
> *,
> struct pkt_bgp_primitives *, struct pkt_nat_primitives *,
> struct pkt_mpls_primitives *, char *, pm_counter_t,
> - pm_counter_t, pm_counter_t, u_int32_t, struct timeval *);
> + pm_counter_t, pm_counter_t, u_int32_t, struct timeval *,
> + struct timeval *);
> EXT void compose_timestamp(char *, int, struct timeval *, int);
>
> EXT struct packet_ptrs *copy_packet_ptrs(struct packet_ptrs *);
> Only in pmacct-1.5.0rc3-chris/src: .util.h.swp
> Only in pmacct-1.5.0rc3-chris/src: util.o
> Only in pmacct-1.5.0rc3-chris/src: xflow_status.o
_______________________________________________
pmacct-discussion mailing list
http://www.pmacct.net/#mailinglists
