Hi Paolo,
On Thu, 5 Jun 2014, Paolo Lucente wrote:
DEBUG ( default/amqp ): publishing [E=pmacct RK=acct DM=0]:
{"timestamp_start": "2014-06-03 22:42:00.202820", "ip_dst":
"196.223.145.xxx", "ip_proto": "tcp", "tos": 0, "ip_src":
"86.30.131.xxx", "bytes": 142, "port_dst": 36363, "packets": 1,
"port_src": 2201, "timestamp_end": "1970-01-01 03:00:00.0"}
Is this a bug? Would it be easy to fix?
This is not a bug. This is result of the fact a single packet has a
single timestamp (or two coinciding) hence only one of the two values,
timestamp_start, is populated.
OK sorry, I found that out by rereading CONFIG-KEYS while trying to work
out if there was any way to get the bucket start and end times into the
JSON output.
Try to:
* capture your own traffic with pmacctd attaching to it a
nfprobe plugin, the NetFlow/IPFIX probe plugin. Set the
export to localhost.
* on localhost you bind nfacctd that listens for NetFlow/IPFIX
packets (generated by pmacctd/nfprobe) and writes wherever you
want to like with the aggregation you like (this time you will
see both timestamp_start and timestamp_end populated - as a
result of the flow-aware cache of nfprobe).
This is the slightly more involved solution i was proposing, which i
don't know if you like or not (definitely good for a proof of concept).
What I have done so far is to modify pmacctd to send two extra timestamps:
the start and end times of the history bucket. This is working well for me
and it would be great to have something like this integrated into pmacct.
Patch attached.
I also had to modify sql_history to allow it to be set to 1 second
intervals, which was previously blocked by a warning. I've included this
part in the attached patch as well.
Finally I changed the timestamps into GMT instead of local time.
Cheers, Chris.
--
Aptivate | http://www.aptivate.org | Phone: +44 1223 967 838
Citylife House, Sturton Street, Cambridge, CB1 2QF, UK
Aptivate is a not-for-profit company registered in England and Wales
with company number 04980791.
Only in pmacct-1.5.0rc3-chris: config.cache
Only in pmacct-1.5.0rc3-chris: config.log
Only in pmacct-1.5.0rc3-chris: config.status
Only in pmacct-1.5.0rc3-chris: Makefile
Only in pmacct-1.5.0rc3-chris/src: acct.o
Only in pmacct-1.5.0rc3-chris/src: addr.o
diff -ru pmacct-1.5.0rc3/src/amqp_plugin.c pmacct-1.5.0rc3-chris/src/amqp_plugin.c
--- pmacct-1.5.0rc3/src/amqp_plugin.c 2014-03-24 02:59:04.000000000 +0300
+++ pmacct-1.5.0rc3-chris/src/amqp_plugin.c 2014-06-04 14:50:00.706672411 +0300
@@ -142,7 +142,7 @@
if (config.sql_history) {
while (now > (basetime.tv_sec + timeslot)) {
- new_basetime.tv_sec = basetime.tv_sec;
+ new_basetime.tv_sec = basetime.tv_sec;
basetime.tv_sec += timeslot;
if (config.sql_history == COUNT_MONTHLY)
timeslot = calc_monthly_timeslot(basetime.tv_sec, config.sql_history_howmany, ADD);
@@ -341,7 +341,7 @@
json_str = compose_json(config.what_to_count, config.what_to_count_2, queue[j]->flow_type,
&queue[j]->primitives, pbgp, pnat, pmpls, pcust, queue[j]->bytes_counter,
queue[j]->packet_counter, queue[j]->flow_counter, queue[j]->tcp_flags,
- &queue[j]->basetime);
+ &queue[j]->basetime, &new_basetime);
if (json_str) {
if (is_routing_key_dyn) amqp_handle_routing_key_dyn_strings(config.sql_table, SRVBUFLEN, orig_amqp_routing_key,
diff -ru pmacct-1.5.0rc3/src/cfg_handlers.c pmacct-1.5.0rc3-chris/src/cfg_handlers.c
--- pmacct-1.5.0rc3/src/cfg_handlers.c 2014-03-19 01:27:42.000000000 +0300
+++ pmacct-1.5.0rc3-chris/src/cfg_handlers.c 2014-06-04 13:29:00.490676835 +0300
@@ -3564,6 +3564,7 @@
k = atoi(value);
if (k > 0) {
+ /*
if (*mu == COUNT_SECONDLY) {
if (k % 60) {
Log(LOG_WARNING, "WARN ( %s ): Ignoring invalid time value: %d (residual secs afters conversion in mins)\n", filename, k);
@@ -3574,6 +3575,7 @@
*mu = COUNT_MINUTELY;
}
}
+ */
*howmany = k;
}
else {
diff -ru pmacct-1.5.0rc3/src/plugin_common.c pmacct-1.5.0rc3-chris/src/plugin_common.c
--- pmacct-1.5.0rc3/src/plugin_common.c 2014-03-26 19:32:46.000000000 +0300
+++ pmacct-1.5.0rc3-chris/src/plugin_common.c 2014-06-04 14:21:16.874628272 +0300
@@ -612,7 +612,8 @@
basetime.tv_sec = now;
basetime.tv_usec = 0;
- if (config.sql_history == COUNT_MINUTELY) timeslot = config.sql_history_howmany*60;
+ if (config.sql_history == COUNT_SECONDLY) timeslot = config.sql_history_howmany;
+ else if (config.sql_history == COUNT_MINUTELY) timeslot = config.sql_history_howmany*60;
else if (config.sql_history == COUNT_HOURLY) timeslot = config.sql_history_howmany*3600;
else if (config.sql_history == COUNT_DAILY) timeslot = config.sql_history_howmany*86400;
else if (config.sql_history == COUNT_WEEKLY) timeslot = config.sql_history_howmany*86400*7;
diff -ru pmacct-1.5.0rc3/src/util.c pmacct-1.5.0rc3-chris/src/util.c
--- pmacct-1.5.0rc3/src/util.c 2014-03-16 19:20:44.000000000 +0300
+++ pmacct-1.5.0rc3-chris/src/util.c 2014-06-04 19:29:59.318621575 +0300
@@ -26,6 +26,7 @@
#include "pmacct-data.h"
#include "ip_flow.h"
#include "classifier.h"
+#include "plugin_common.h"
#ifdef WITH_JANSSON
#include <jansson.h>
#endif
@@ -1045,8 +1046,8 @@
if (a->tv_sec == b->tv_sec) {
if (a->tv_usec > b->tv_usec) return 1;
if (a->tv_usec < b->tv_usec) return -1;
- if (a->tv_usec == b->tv_usec) return 0;
}
+ return 0;
}
/*
@@ -1597,7 +1598,7 @@
char *compose_json(u_int64_t wtc, u_int64_t wtc_2, u_int8_t flow_type, struct pkt_primitives *pbase,
struct pkt_bgp_primitives *pbgp, struct pkt_nat_primitives *pnat, struct pkt_mpls_primitives *pmpls,
char *pcust, pm_counter_t bytes_counter, pm_counter_t packet_counter, pm_counter_t flow_counter,
- u_int32_t tcp_flags, struct timeval *basetime)
+ u_int32_t tcp_flags, struct timeval *timeslot_start, struct timeval *timeslot_end)
{
char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN];
char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], *as_path, *bgp_comm, empty_string[] = "", *tmpbuf;
@@ -2000,20 +2001,16 @@
}
}
- if (basetime && config.sql_history) {
- struct timeval tv;
-
- tv.tv_sec = basetime->tv_sec;
- tv.tv_usec = 0;
- compose_timestamp(tstamp_str, SRVBUFLEN, &tv, FALSE);
- kv = json_pack("{ss}", "stamp_inserted", tstamp_str);
+ if (config.sql_history) {
+ compose_timestamp(tstamp_str, SRVBUFLEN, &basetime, FALSE);
+ kv = json_pack("{ss}", "timeslot_start", tstamp_str);
json_object_update_missing(obj, kv);
json_decref(kv);
- tv.tv_sec = time(NULL);
- tv.tv_usec = 0;
+ struct timeval tv = basetime;
+ tv.tv_sec += timeslot;
compose_timestamp(tstamp_str, SRVBUFLEN, &tv, FALSE);
- kv = json_pack("{ss}", "stamp_updated", tstamp_str);
+ kv = json_pack("{ss}", "timeslot_end", tstamp_str);
json_object_update_missing(obj, kv);
json_decref(kv);
}
@@ -2058,7 +2055,7 @@
struct tm *time2;
time1 = tv->tv_sec;
- time2 = localtime(&time1);
+ time2 = gmtime(&time1);
strftime(tmpbuf, SRVBUFLEN, "%Y-%m-%d %H:%M:%S", time2);
if (usec) snprintf(buf, buflen, "%s.%u", tmpbuf, tv->tv_usec);
Only in pmacct-1.5.0rc3-chris/src: .util.c.swp
diff -ru pmacct-1.5.0rc3/src/util.h pmacct-1.5.0rc3-chris/src/util.h
--- pmacct-1.5.0rc3/src/util.h 2013-11-08 05:43:01.000000000 +0300
+++ pmacct-1.5.0rc3-chris/src/util.h 2014-06-04 14:46:31.726667060 +0300
@@ -99,7 +99,8 @@
EXT char *compose_json(u_int64_t, u_int64_t, u_int8_t, struct pkt_primitives *,
struct pkt_bgp_primitives *, struct pkt_nat_primitives *,
struct pkt_mpls_primitives *, char *, pm_counter_t,
- pm_counter_t, pm_counter_t, u_int32_t, struct timeval *);
+ pm_counter_t, pm_counter_t, u_int32_t, struct timeval *,
+ struct timeval *);
EXT void compose_timestamp(char *, int, struct timeval *, int);
EXT struct packet_ptrs *copy_packet_ptrs(struct packet_ptrs *);
Only in pmacct-1.5.0rc3-chris/src: .util.h.swp
Only in pmacct-1.5.0rc3-chris/src: util.o
Only in pmacct-1.5.0rc3-chris/src: xflow_status.o
_______________________________________________
pmacct-discussion mailing list
http://www.pmacct.net/#mailinglists
