Hi,
We would like to propose a patch for the sipcapture module where we
retry a write to the homer database in case the first attempt fails.
This proves useful under heavy loads.
We add the insert_retries parameter to limit the maximum number of write
retries and also the insert_retry_timeout as a time limit for the retry.
Thank you,
Lucian Balaceanu
>From 4851ece2126454a7c1a1b3f92ac491a58505d5d7 Mon Sep 17 00:00:00 2001
From: lucian balanceanu <[email protected]>
Date: Wed, 20 Aug 2014 13:49:20 +0300
Subject: [PATCH] sipcapture: retry insert in DB on initial failure
- added insert_retries and insert_retry_timeout parameters, which
control how many times and for how long Kamailio should try to
rewrite to the Homer database in case the first attempt fails;
trying to rewrite proves useful especially in heavy load scenarios.
---
modules/sipcapture/README | 177 ++++++++++++++++++---------
modules/sipcapture/doc/sipcapture_admin.xml | 42 +++++++
modules/sipcapture/sipcapture.c | 58 +++++++--
3 files changed, 203 insertions(+), 74 deletions(-)
diff --git a/modules/sipcapture/README b/modules/sipcapture/README
index edf544d..172afb5 100644
--- a/modules/sipcapture/README
+++ b/modules/sipcapture/README
@@ -10,9 +10,9 @@ Alexandr Dubovikov
<[email protected]>
- Copyright © 2011 QSC AG
+ Copyright © 2011 QSC AG
- Copyright © 2011 http://www.qsc.de
+ Copyright © 2011 http://www.qsc.de
__________________________________________________________________
Table of Contents
@@ -33,15 +33,18 @@ Alexandr Dubovikov
3.4. hash_source (str)
3.5. db_insert_mode (integer)
3.6. capture_on (integer)
- 3.7. hep_capture_on (integer)
- 3.8. raw_ipip_capture_on (integer)
- 3.9. raw_moni_capture_on (integer)
- 3.10. raw_socket_listen (string)
- 3.11. raw_interface (string)
- 3.12. raw_sock_children (integer)
- 3.13. promiscuous_on (integer)
- 3.14. raw_moni_bpf_on (integer)
- 3.15. capture_node (str)
+ 3.7. capture_mode (integer)
+ 3.8. hep_capture_on (integer)
+ 3.9. raw_ipip_capture_on (integer)
+ 3.10. raw_moni_capture_on (integer)
+ 3.11. raw_socket_listen (string)
+ 3.12. raw_interface (string)
+ 3.13. raw_sock_children (integer)
+ 3.14. promiscuous_on (integer)
+ 3.15. raw_moni_bpf_on (integer)
+ 3.16. capture_node (str)
+ 3.17. insert_retries (integer)
+ 3.18. insert_retry_timeout (integer)
4. MI Commands
@@ -62,15 +65,18 @@ Alexandr Dubovikov
1.4. Set mt_mode parameter
1.5. db_insert_mode example
1.6. Set capture_on parameter
- 1.7. Set hep_capture_on parameter
- 1.8. Set raw_ipip_capture_on parameter
- 1.9. Set raw_moni_capture_on parameter
- 1.10. Set raw_socket_listen parameter
- 1.11. Set raw_interface parameter
- 1.12. Set raw_sock_children parameter
- 1.13. Set promiscous_on parameter
- 1.14. Set raw_moni_bpf_on parameter
- 1.15. Set capture_node parameter
+ 1.7. capture_mode example
+ 1.8. Set hep_capture_on parameter
+ 1.9. Set raw_ipip_capture_on parameter
+ 1.10. Set raw_moni_capture_on parameter
+ 1.11. Set raw_socket_listen parameter
+ 1.12. Set raw_interface parameter
+ 1.13. Set raw_sock_children parameter
+ 1.14. Set promiscous_on parameter
+ 1.15. Set raw_moni_bpf_on parameter
+ 1.16. Set capture_node parameter
+ 1.17. Set insert_retries parameter
+ 1.18. Set insert_retry_timeout parameter
Chapter 1. Admin Guide
@@ -90,15 +96,18 @@ Chapter 1. Admin Guide
3.4. hash_source (str)
3.5. db_insert_mode (integer)
3.6. capture_on (integer)
- 3.7. hep_capture_on (integer)
- 3.8. raw_ipip_capture_on (integer)
- 3.9. raw_moni_capture_on (integer)
- 3.10. raw_socket_listen (string)
- 3.11. raw_interface (string)
- 3.12. raw_sock_children (integer)
- 3.13. promiscuous_on (integer)
- 3.14. raw_moni_bpf_on (integer)
- 3.15. capture_node (str)
+ 3.7. capture_mode (integer)
+ 3.8. hep_capture_on (integer)
+ 3.9. raw_ipip_capture_on (integer)
+ 3.10. raw_moni_capture_on (integer)
+ 3.11. raw_socket_listen (string)
+ 3.12. raw_interface (string)
+ 3.13. raw_sock_children (integer)
+ 3.14. promiscuous_on (integer)
+ 3.15. raw_moni_bpf_on (integer)
+ 3.16. capture_node (str)
+ 3.17. insert_retries (integer)
+ 3.18. insert_retry_timeout (integer)
4. MI Commands
@@ -151,15 +160,18 @@ Chapter 1. Admin Guide
3.4. hash_source (str)
3.5. db_insert_mode (integer)
3.6. capture_on (integer)
- 3.7. hep_capture_on (integer)
- 3.8. raw_ipip_capture_on (integer)
- 3.9. raw_moni_capture_on (integer)
- 3.10. raw_socket_listen (string)
- 3.11. raw_interface (string)
- 3.12. raw_sock_children (integer)
- 3.13. promiscuous_on (integer)
- 3.14. raw_moni_bpf_on (integer)
- 3.15. capture_node (str)
+ 3.7. capture_mode (integer)
+ 3.8. hep_capture_on (integer)
+ 3.9. raw_ipip_capture_on (integer)
+ 3.10. raw_moni_capture_on (integer)
+ 3.11. raw_socket_listen (string)
+ 3.12. raw_interface (string)
+ 3.13. raw_sock_children (integer)
+ 3.14. promiscuous_on (integer)
+ 3.15. raw_moni_bpf_on (integer)
+ 3.16. capture_node (str)
+ 3.17. insert_retries (integer)
+ 3.18. insert_retry_timeout (integer)
3.1. db_url (str)
@@ -235,29 +247,46 @@ modparam("sipcapture", "db_insert_mode", 1)
modparam("sipcapture", "capture_on", 1)
...
-3.7. hep_capture_on (integer)
+3.7. capture_mode (integer)
+
+ This parameter can be used for defining a capture mode which can be
+ used in the sip_capture calls as a parameter. A capture mode has a name
+ and some parameters. It must be defined in the format:
+ name=>param1=val1;param2=val2;... The parameters are db_url,
+ table_name, mt_mode and hash_source (optional). Multiple capture modes
+ can be defined by using this parameter multiple times. After this, the
+ capture modes can be used like: sip_capture ("", "CAPTURE_MODE");
+
+ Example 1.7. capture_mode example
+modparam("sipcapture", "capture_mode", "mode1=>db_url=mysql://user:passwd@host/d
+bname1;table_name=homer_capture1|homer_capture2;mt_mode=hash;hash_source=call_id
+;")
+modparam("sipcapture", "capture_mode", "mode2=>db_url=mysql://user:passwd@host/d
+bname2;table_name=homer_capture3|homer_capture4;mt_mode=rand;")
+
+3.8. hep_capture_on (integer)
Parameter to enable/disable capture of HEP (on(1)/off(0))
Default value is "0".
- Example 1.7. Set hep_capture_on parameter
+ Example 1.8. Set hep_capture_on parameter
...
modparam("sipcapture", "hep_capture_on", 1)
...
-3.8. raw_ipip_capture_on (integer)
+3.9. raw_ipip_capture_on (integer)
Parameter to enable/disable IPIP capturing (on(1)/off(0))
Default value is "0".
- Example 1.8. Set raw_ipip_capture_on parameter
+ Example 1.9. Set raw_ipip_capture_on parameter
...
modparam("sipcapture", "raw_ipip_capture_on", 1)
...
-3.9. raw_moni_capture_on (integer)
+3.10. raw_moni_capture_on (integer)
Parameter to enable/disable monitoring/mirroring port capturing
(on(1)/off(0)) Only one mode on raw socket can be enabled! Monitoring
@@ -265,12 +294,12 @@ modparam("sipcapture", "raw_ipip_capture_on", 1)
Default value is "0".
- Example 1.9. Set raw_moni_capture_on parameter
+ Example 1.10. Set raw_moni_capture_on parameter
...
modparam("sipcapture", "raw_moni_capture_on", 1)
...
-3.10. raw_socket_listen (string)
+3.11. raw_socket_listen (string)
Parameter indicate an listen IP address of RAW socket for IPIP
capturing. You can also define a port/portrange for IPIP/Mirroring
@@ -288,49 +317,49 @@ modparam("sipcapture", "raw_moni_capture_on", 1)
Default value is "".
- Example 1.10. Set raw_socket_listen parameter
+ Example 1.11. Set raw_socket_listen parameter
...
modparam("sipcapture", "raw_socket_listen", "10.0.0.1:5060-5090")
...
modparam("sipcapture", "raw_socket_listen", "10.0.0.1:5060")
...
-3.11. raw_interface (string)
+3.12. raw_interface (string)
Name of the interface to bind on the raw socket.
Default value is "".
- Example 1.11. Set raw_interface parameter
+ Example 1.12. Set raw_interface parameter
...
modparam("sipcapture", "raw_interface", "eth0")
...
-3.12. raw_sock_children (integer)
+3.13. raw_sock_children (integer)
Parameter define how many children that must be created to listen the
raw socket.
Default value is "1".
- Example 1.12. Set raw_sock_children parameter
+ Example 1.13. Set raw_sock_children parameter
...
modparam("sipcapture", "raw_sock_children", 6)
...
-3.13. promiscuous_on (integer)
+3.14. promiscuous_on (integer)
Parameter to enable/disable promiscuous mode on the raw socket. Linux
only.
Default value is "0".
- Example 1.13. Set promiscous_on parameter
+ Example 1.14. Set promiscous_on parameter
...
modparam("sipcapture", "promiscuous_on", 1)
...
-3.14. raw_moni_bpf_on (integer)
+3.15. raw_moni_bpf_on (integer)
Activate Linux Socket Filter (LSF based on BPF) on the mirroring
interface. The structure is defined in linux/filter.h. The default LSF
@@ -339,27 +368,55 @@ modparam("sipcapture", "promiscuous_on", 1)
Default value is "0".
- Example 1.14. Set raw_moni_bpf_on parameter
+ Example 1.15. Set raw_moni_bpf_on parameter
...
modparam("sipcapture", "raw_moni_bpf_on", 1)
...
-3.15. capture_node (str)
+3.16. capture_node (str)
Name of the capture node.
Default value is "homer01".
- Example 1.15. Set capture_node parameter
+ Example 1.16. Set capture_node parameter
...
modparam("sipcapture", "capture_node", "homer03")
...
+3.17. insert_retries (integer)
+
+ The number of times Kamailio should retry to write to the Homer
+ database in case the first attempt failed. The retry is also limited
+ timewise by the insert_retry_timeout parameter. Values allowed range
+ from 0 to 500.
+
+ Default value is 0 (no retries).
+
+ Example 1.17. Set insert_retries parameter
+...
+modparam("sipcapture", "insert_retries", 5)
+...
+
+3.18. insert_retry_timeout (integer)
+
+ The time limit in seconds Kamailio retries to write to the Homer
+ database in case the first attempt failed. This parameter is only used
+ together with the insert_retries parameter. Values allowed range from 0
+ to 300.
+
+ Default value is 60 seconds.
+
+ Example 1.18. Set insert_retry_timeout parameter
+...
+modparam("sipcapture", "insert_retry_timeout", 10)
+...
+
4. MI Commands
4.1. sip_capture
-4.1. sip_capture
+4.1. sip_capture
Name: sip_capture
@@ -369,7 +426,7 @@ modparam("sipcapture", "capture_node", "homer03")
+ on
+ off
The parameter is optional - if missing, the command will return the
- status of the SIP message capturing (as string "on" or "off" )
+ status of the SIP message capturing (as string âonâ or âoffâ )
without changing anything.
MI FIFO Command Format:
@@ -381,7 +438,7 @@ modparam("sipcapture", "capture_node", "homer03")
5.1. sipcapture.status param
-5.1. sipcapture.status param
+5.1. sipcapture.status param
Name: sipcapture.status
@@ -389,7 +446,7 @@ modparam("sipcapture", "capture_node", "homer03")
* on or off: turns on/off SIP message capturing. Possible values are:
+ on
+ off
- * "check" does not change sipcapture status, just reports the current
+ * âcheckâ does not change sipcapture status, just reports the current
status.
6. Database setup
diff --git a/modules/sipcapture/doc/sipcapture_admin.xml b/modules/sipcapture/doc/sipcapture_admin.xml
index c172c5a..8923149 100644
--- a/modules/sipcapture/doc/sipcapture_admin.xml
+++ b/modules/sipcapture/doc/sipcapture_admin.xml
@@ -407,6 +407,48 @@ modparam("sipcapture", "capture_node", "homer03")
...
</programlisting>
</example>
+ </section>
+ <section id="sipcapture.p.insert_retries">
+ <title><varname>insert_retries</varname> (integer)</title>
+ <para>
+ The number of times Kamailio should retry to write to the Homer database in case
+ the first attempt failed. The retry is also limited timewise by the
+ insert_retry_timeout parameter. Values allowed range from 0 to 500.
+ </para>
+ <para>
+ <emphasis>
+ Default value is 0 (no retries).
+ </emphasis>
+ </para>
+ <example>
+ <title>Set <varname>insert_retries</varname> parameter</title>
+ <programlisting format="linespecific">
+...
+modparam("sipcapture", "insert_retries", 5)
+...
+ </programlisting>
+ </example>
+ </section>
+ <section id="sipcapture.p.insert_retry_timeout">
+ <title><varname>insert_retry_timeout</varname> (integer)</title>
+ <para>
+ The time limit in seconds Kamailio retries to write to the Homer database in case
+ the first attempt failed. This parameter is only used together with the insert_retries
+ parameter. Values allowed range from 0 to 300.
+ </para>
+ <para>
+ <emphasis>
+ Default value is 60 seconds.
+ </emphasis>
+ </para>
+ <example>
+ <title>Set <varname>insert_retry_timeout</varname> parameter</title>
+ <programlisting format="linespecific">
+...
+modparam("sipcapture", "insert_retry_timeout", 10)
+...
+ </programlisting>
+ </example>
</section>
</section>
<section>
diff --git a/modules/sipcapture/sipcapture.c b/modules/sipcapture/sipcapture.c
index 43f072c..5471f54 100644
--- a/modules/sipcapture/sipcapture.c
+++ b/modules/sipcapture/sipcapture.c
@@ -198,6 +198,8 @@ int db_insert_mode = 0;
int promisc_on = 0;
int bpf_on = 0;
int hep_capture_on = 0;
+int insert_retries = 0;
+int insert_retry_timeout = 60;
int hep_offset = 0;
str raw_socket_listen = { 0, 0 };
str raw_interface = { 0, 0 };
@@ -225,8 +227,6 @@ static struct sock_filter BPF_code[] = { { 0x28, 0, 0, 0x0000000c }, { 0x15, 0,
unsigned int no_tables = 0;
-
-
enum e_mt_mode mtmode = mode_random ;
enum hash_source source = hs_error;
@@ -234,7 +234,6 @@ enum hash_source source = hs_error;
struct hep_timehdr* heptime;
-
/*! \brief
* Exported functions
*/
@@ -306,6 +305,8 @@ static param_export_t params[] = {
{"raw_moni_bpf_on", INT_PARAM, &bpf_on },
{"callid_aleg_header", PARAM_STR, &callid_aleg_header},
{"capture_mode", PARAM_STRING|USE_FUNC_PARAM, (void *)capture_mode_param},
+ {"insert_retries", INT_PARAM, &insert_retries },
+ {"insert_retry_timeout",INT_PARAM, &insert_retry_timeout },
{0, 0, 0}
};
@@ -355,7 +356,6 @@ struct module_exports exports = {
};
-
/* returns number of tables if successful
* <0 if failed
*/
@@ -788,7 +788,19 @@ static int mod_init(void) {
return -1;
}
+ if ((insert_retries <0) || ( insert_retries > 500)) {
+ LM_ERR("insert_retries should be a value between 0 and 500");
+ return -1;
+ }
+ if (( 0 == insert_retries) && (insert_retry_timeout != 0)){
+ LM_ERR("insert_retry_timeout has no meaning when insert_retries is not set");
+ }
+
+ if ((insert_retry_timeout <0) || ( insert_retry_timeout > 300)) {
+ LM_ERR("insert_retry_timeout should be a value between 0 and 300");
+ return -1;
+ }
/* raw processes for IPIP encapsulation */
if (ipip_capture_on || moni_capture_on) {
@@ -1072,6 +1084,11 @@ static int sip_capture_store(struct _sipcapture_object *sco, str *dtable, _captu
str tmp;
int ii = 0;
+ int ret = 0;
+ int counter = 0;
+ db_insert_f insert;
+ time_t retry_failed_time = 0;
+
str *table = NULL;
_capture_mode_data_t *c = NULL;
@@ -1320,24 +1337,37 @@ static int sip_capture_store(struct _sipcapture_object *sco, str *dtable, _captu
LM_DBG("storing info...\n");
- if(db_insert_mode==1 && c->db_funcs.insert_delayed!=NULL) {
- if (c->db_funcs.insert_delayed(c->db_con, db_keys, db_vals, NR_KEYS) < 0) {
- LM_ERR("failed to insert delayed into database\n");
- goto error;
+ if (db_insert_mode == 1 && c->db_funcs.insert_delayed != NULL)
+ insert = c->db_funcs.insert_delayed;
+ else
+ insert = c->db_funcs.insert;
+ counter = 0;
+
+ ret = insert(c->db_con, db_keys, db_vals, NR_KEYS);
+ if (ret < 0)
+ LM_DBG("failed to insert into database(first attempt)\n");
+
+ while ((insert_retries != 0) && ((ret = insert(c->db_con, db_keys, db_vals,
+ NR_KEYS)) < 0)) {
+ counter++;
+ if (1 == counter) //first failed retry
+ retry_failed_time = time(NULL);
+
+ if ((counter > insert_retries) || (time(NULL) - retry_failed_time
+ > insert_retry_timeout)) {
+ LM_ERR("failed to insert into database(second attempt)\n");
+ break;
}
- } else if (c->db_funcs.insert(c->db_con, db_keys, db_vals, NR_KEYS) < 0) {
- LM_ERR("failed to insert into database\n");
- goto error;
}
-
+ if (ret < 0)
+ goto error;
#ifdef STATISTICS
update_stat(sco->stat, 1);
#endif
return 1;
-error:
- return -1;
+ error: return -1;
}
static int sip_capture(struct sip_msg *msg, str *_table, _capture_mode_data_t * cm_data)
--
1.7.4.1
_______________________________________________
sr-dev mailing list
[email protected]
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-dev
