Browse code

sipcapture: retry insert in DB on initial failure

- added insert_retries and insert_retry_timeout parameters, which
control how many times and for how long Kamailio should try to
rewrite to the Homer database in case the first attempt fails;
trying to rewrite proves useful especially in heavy load scenarios.

lucian balanceanu authored on 20/08/2014 15:08:03
Showing 3 changed files
... ...
@@ -10,9 +10,9 @@ Alexandr Dubovikov
10 10
 
11 11
    <alexandr.dubovikov@gmail.com>
12 12
 
13
-   Copyright � 2011 QSC AG
13
+   Copyright © 2011 QSC AG
14 14
 
15
-   Copyright � 2011 http://www.qsc.de
15
+   Copyright © 2011 http://www.qsc.de
16 16
      __________________________________________________________________
17 17
 
18 18
    Table of Contents
... ...
@@ -33,15 +33,18 @@ Alexandr Dubovikov
33 33
               3.4. hash_source (str)
34 34
               3.5. db_insert_mode (integer)
35 35
               3.6. capture_on (integer)
36
-              3.7. hep_capture_on (integer)
37
-              3.8. raw_ipip_capture_on (integer)
38
-              3.9. raw_moni_capture_on (integer)
39
-              3.10. raw_socket_listen (string)
40
-              3.11. raw_interface (string)
41
-              3.12. raw_sock_children (integer)
42
-              3.13. promiscuous_on (integer)
43
-              3.14. raw_moni_bpf_on (integer)
44
-              3.15. capture_node (str)
36
+              3.7. capture_mode (integer)
37
+              3.8. hep_capture_on (integer)
38
+              3.9. raw_ipip_capture_on (integer)
39
+              3.10. raw_moni_capture_on (integer)
40
+              3.11. raw_socket_listen (string)
41
+              3.12. raw_interface (string)
42
+              3.13. raw_sock_children (integer)
43
+              3.14. promiscuous_on (integer)
44
+              3.15. raw_moni_bpf_on (integer)
45
+              3.16. capture_node (str)
46
+              3.17. insert_retries (integer)
47
+              3.18. insert_retry_timeout (integer)
45 48
 
46 49
         4. MI Commands
47 50
 
... ...
@@ -62,15 +65,18 @@ Alexandr Dubovikov
62 65
    1.4. Set mt_mode parameter
63 66
    1.5. db_insert_mode example
64 67
    1.6. Set capture_on parameter
65
-   1.7. Set hep_capture_on parameter
66
-   1.8. Set raw_ipip_capture_on parameter
67
-   1.9. Set raw_moni_capture_on parameter
68
-   1.10. Set raw_socket_listen parameter
69
-   1.11. Set raw_interface parameter
70
-   1.12. Set raw_sock_children parameter
71
-   1.13. Set promiscous_on parameter
72
-   1.14. Set raw_moni_bpf_on parameter
73
-   1.15. Set capture_node parameter
68
+   1.7. capture_mode example
69
+   1.8. Set hep_capture_on parameter
70
+   1.9. Set raw_ipip_capture_on parameter
71
+   1.10. Set raw_moni_capture_on parameter
72
+   1.11. Set raw_socket_listen parameter
73
+   1.12. Set raw_interface parameter
74
+   1.13. Set raw_sock_children parameter
75
+   1.14. Set promiscous_on parameter
76
+   1.15. Set raw_moni_bpf_on parameter
77
+   1.16. Set capture_node parameter
78
+   1.17. Set insert_retries parameter
79
+   1.18. Set insert_retry_timeout parameter
74 80
 
75 81
 Chapter 1. Admin Guide
76 82
 
... ...
@@ -90,15 +96,18 @@ Chapter 1. Admin Guide
90 96
         3.4. hash_source (str)
91 97
         3.5. db_insert_mode (integer)
92 98
         3.6. capture_on (integer)
93
-        3.7. hep_capture_on (integer)
94
-        3.8. raw_ipip_capture_on (integer)
95
-        3.9. raw_moni_capture_on (integer)
96
-        3.10. raw_socket_listen (string)
97
-        3.11. raw_interface (string)
98
-        3.12. raw_sock_children (integer)
99
-        3.13. promiscuous_on (integer)
100
-        3.14. raw_moni_bpf_on (integer)
101
-        3.15. capture_node (str)
99
+        3.7. capture_mode (integer)
100
+        3.8. hep_capture_on (integer)
101
+        3.9. raw_ipip_capture_on (integer)
102
+        3.10. raw_moni_capture_on (integer)
103
+        3.11. raw_socket_listen (string)
104
+        3.12. raw_interface (string)
105
+        3.13. raw_sock_children (integer)
106
+        3.14. promiscuous_on (integer)
107
+        3.15. raw_moni_bpf_on (integer)
108
+        3.16. capture_node (str)
109
+        3.17. insert_retries (integer)
110
+        3.18. insert_retry_timeout (integer)
102 111
 
103 112
    4. MI Commands
104 113
 
... ...
@@ -151,15 +160,18 @@ Chapter 1. Admin Guide
151 160
    3.4. hash_source (str)
152 161
    3.5. db_insert_mode (integer)
153 162
    3.6. capture_on (integer)
154
-   3.7. hep_capture_on (integer)
155
-   3.8. raw_ipip_capture_on (integer)
156
-   3.9. raw_moni_capture_on (integer)
157
-   3.10. raw_socket_listen (string)
158
-   3.11. raw_interface (string)
159
-   3.12. raw_sock_children (integer)
160
-   3.13. promiscuous_on (integer)
161
-   3.14. raw_moni_bpf_on (integer)
162
-   3.15. capture_node (str)
163
+   3.7. capture_mode (integer)
164
+   3.8. hep_capture_on (integer)
165
+   3.9. raw_ipip_capture_on (integer)
166
+   3.10. raw_moni_capture_on (integer)
167
+   3.11. raw_socket_listen (string)
168
+   3.12. raw_interface (string)
169
+   3.13. raw_sock_children (integer)
170
+   3.14. promiscuous_on (integer)
171
+   3.15. raw_moni_bpf_on (integer)
172
+   3.16. capture_node (str)
173
+   3.17. insert_retries (integer)
174
+   3.18. insert_retry_timeout (integer)
163 175
 
164 176
 3.1. db_url (str)
165 177
 
... ...
@@ -235,29 +247,46 @@ modparam("sipcapture", "db_insert_mode", 1)
235 247
 modparam("sipcapture", "capture_on", 1)
236 248
 ...
237 249
 
238
-3.7. hep_capture_on (integer)
250
+3.7. capture_mode (integer)
251
+
252
+   This parameter can be used for defining a capture mode which can be
253
+   used in the sip_capture calls as a parameter. A capture mode has a name
254
+   and some parameters. It must be defined in the format:
255
+   name=>param1=val1;param2=val2;... The parameters are db_url,
256
+   table_name, mt_mode and hash_source (optional). Multiple capture modes
257
+   can be defined by using this parameter multiple times. After this, the
258
+   capture modes can be used like: sip_capture ("", "CAPTURE_MODE");
259
+
260
+   Example 1.7. capture_mode example
261
+modparam("sipcapture", "capture_mode", "mode1=>db_url=mysql://user:passwd@host/d
262
+bname1;table_name=homer_capture1|homer_capture2;mt_mode=hash;hash_source=call_id
263
+;")
264
+modparam("sipcapture", "capture_mode", "mode2=>db_url=mysql://user:passwd@host/d
265
+bname2;table_name=homer_capture3|homer_capture4;mt_mode=rand;")
266
+
267
+3.8. hep_capture_on (integer)
239 268
 
240 269
    Parameter to enable/disable capture of HEP (on(1)/off(0))
241 270
 
242 271
    Default value is "0".
243 272
 
244
-   Example 1.7. Set hep_capture_on parameter
273
+   Example 1.8. Set hep_capture_on parameter
245 274
 ...
246 275
 modparam("sipcapture", "hep_capture_on", 1)
247 276
 ...
248 277
 
249
-3.8. raw_ipip_capture_on (integer)
278
+3.9. raw_ipip_capture_on (integer)
250 279
 
251 280
    Parameter to enable/disable IPIP capturing (on(1)/off(0))
252 281
 
253 282
    Default value is "0".
254 283
 
255
-   Example 1.8. Set raw_ipip_capture_on parameter
284
+   Example 1.9. Set raw_ipip_capture_on parameter
256 285
 ...
257 286
 modparam("sipcapture", "raw_ipip_capture_on", 1)
258 287
 ...
259 288
 
260
-3.9. raw_moni_capture_on (integer)
289
+3.10. raw_moni_capture_on (integer)
261 290
 
262 291
    Parameter to enable/disable monitoring/mirroring port capturing
263 292
    (on(1)/off(0)) Only one mode on raw socket can be enabled! Monitoring
... ...
@@ -265,12 +294,12 @@ modparam("sipcapture", "raw_ipip_capture_on", 1)
265 294
 
266 295
    Default value is "0".
267 296
 
268
-   Example 1.9. Set raw_moni_capture_on parameter
297
+   Example 1.10. Set raw_moni_capture_on parameter
269 298
 ...
270 299
 modparam("sipcapture", "raw_moni_capture_on", 1)
271 300
 ...
272 301
 
273
-3.10. raw_socket_listen (string)
302
+3.11. raw_socket_listen (string)
274 303
 
275 304
    Parameter indicate an listen IP address of RAW socket for IPIP
276 305
    capturing. You can also define a port/portrange for IPIP/Mirroring
... ...
@@ -288,49 +317,49 @@ modparam("sipcapture", "raw_moni_capture_on", 1)
288 317
 
289 318
    Default value is "".
290 319
 
291
-   Example 1.10. Set raw_socket_listen parameter
320
+   Example 1.11. Set raw_socket_listen parameter
292 321
 ...
293 322
 modparam("sipcapture", "raw_socket_listen", "10.0.0.1:5060-5090")
294 323
 ...
295 324
 modparam("sipcapture", "raw_socket_listen", "10.0.0.1:5060")
296 325
 ...
297 326
 
298
-3.11. raw_interface (string)
327
+3.12. raw_interface (string)
299 328
 
300 329
    Name of the interface to bind on the raw socket.
301 330
 
302 331
    Default value is "".
303 332
 
304
-   Example 1.11. Set raw_interface parameter
333
+   Example 1.12. Set raw_interface parameter
305 334
 ...
306 335
 modparam("sipcapture", "raw_interface", "eth0")
307 336
 ...
308 337
 
309
-3.12. raw_sock_children (integer)
338
+3.13. raw_sock_children (integer)
310 339
 
311 340
    Parameter define how many children that must be created to listen the
312 341
    raw socket.
313 342
 
314 343
    Default value is "1".
315 344
 
316
-   Example 1.12. Set raw_sock_children parameter
345
+   Example 1.13. Set raw_sock_children parameter
317 346
 ...
318 347
 modparam("sipcapture", "raw_sock_children", 6)
319 348
 ...
320 349
 
321
-3.13. promiscuous_on (integer)
350
+3.14. promiscuous_on (integer)
322 351
 
323 352
    Parameter to enable/disable promiscuous mode on the raw socket. Linux
324 353
    only.
325 354
 
326 355
    Default value is "0".
327 356
 
328
-   Example 1.13. Set promiscous_on parameter
357
+   Example 1.14. Set promiscous_on parameter
329 358
 ...
330 359
 modparam("sipcapture", "promiscuous_on", 1)
331 360
 ...
332 361
 
333
-3.14. raw_moni_bpf_on (integer)
362
+3.15. raw_moni_bpf_on (integer)
334 363
 
335 364
    Activate Linux Socket Filter (LSF based on BPF) on the mirroring
336 365
    interface. The structure is defined in linux/filter.h. The default LSF
... ...
@@ -339,22 +368,50 @@ modparam("sipcapture", "promiscuous_on", 1)
339 368
 
340 369
    Default value is "0".
341 370
 
342
-   Example 1.14. Set raw_moni_bpf_on parameter
371
+   Example 1.15. Set raw_moni_bpf_on parameter
343 372
 ...
344 373
 modparam("sipcapture", "raw_moni_bpf_on", 1)
345 374
 ...
346 375
 
347
-3.15. capture_node (str)
376
+3.16. capture_node (str)
348 377
 
349 378
    Name of the capture node.
350 379
 
351 380
    Default value is "homer01".
352 381
 
353
-   Example 1.15. Set capture_node parameter
382
+   Example 1.16. Set capture_node parameter
354 383
 ...
355 384
 modparam("sipcapture", "capture_node", "homer03")
356 385
 ...
357 386
 
387
+3.17. insert_retries (integer)
388
+
389
+   The number of times Kamailio should retry to write to the Homer
390
+   database in case the first attempt failed. The retry is also limited
391
+   timewise by the insert_retry_timeout parameter. Values allowed range
392
+   from 0 to 500.
393
+
394
+   Default value is 0 (no retries).
395
+
396
+   Example 1.17. Set insert_retries parameter
397
+...
398
+modparam("sipcapture", "insert_retries", 5)
399
+...
400
+
401
+3.18. insert_retry_timeout (integer)
402
+
403
+   The time limit in seconds Kamailio retries to write to the Homer
404
+   database in case the first attempt failed. This parameter is only used
405
+   together with the insert_retries parameter. Values allowed range from 0
406
+   to 300.
407
+
408
+   Default value is 60 seconds.
409
+
410
+   Example 1.18. Set insert_retry_timeout parameter
411
+...
412
+modparam("sipcapture", "insert_retry_timeout", 10)
413
+...
414
+
358 415
 4. MI Commands
359 416
 
360 417
    4.1. sip_capture
... ...
@@ -369,7 +426,7 @@ modparam("sipcapture", "capture_node", "homer03")
369 426
           + on
370 427
           + off
371 428
        The parameter is optional - if missing, the command will return the
372
-       status of the SIP message capturing (as string "on" or "off" )
429
+       status of the SIP message capturing (as string “on” or “off” )
373 430
        without changing anything.
374 431
 
375 432
    MI FIFO Command Format:
... ...
@@ -389,7 +446,7 @@ modparam("sipcapture", "capture_node", "homer03")
389 446
      * on or off: turns on/off SIP message capturing. Possible values are:
390 447
           + on
391 448
           + off
392
-     * "check" does not change sipcapture status, just reports the current
449
+     * “check” does not change sipcapture status, just reports the current
393 450
        status.
394 451
 
395 452
 6. Database setup
... ...
@@ -408,6 +408,48 @@ modparam("sipcapture", "capture_node", "homer03")
408 408
 </programlisting>
409 409
 		</example>
410 410
 	</section>
411
+	<section id="sipcapture.p.insert_retries">
412
+		<title><varname>insert_retries</varname> (integer)</title>
413
+		<para>
414
+		The number of times Kamailio should retry to write to the Homer database in case
415
+		the first attempt failed. The retry is also limited timewise by the
416
+		insert_retry_timeout parameter. Values allowed range from 0 to 500.
417
+		</para>
418
+		<para>
419
+		<emphasis>
420
+			Default value is 0 (no retries).
421
+		</emphasis>
422
+		</para>
423
+		<example>
424
+			<title>Set <varname>insert_retries</varname> parameter</title>
425
+			<programlisting format="linespecific">
426
+...
427
+modparam("sipcapture", "insert_retries", 5)
428
+...
429
+			</programlisting>
430
+		</example>
431
+	</section>
432
+	<section id="sipcapture.p.insert_retry_timeout">
433
+		<title><varname>insert_retry_timeout</varname> (integer)</title>
434
+		<para>
435
+		The time limit in seconds Kamailio retries to write to the Homer database in case
436
+		the first attempt failed. This parameter is only used together with the insert_retries
437
+		parameter. Values allowed range from 0 to 300.
438
+		</para>
439
+		<para>
440
+		<emphasis>
441
+			Default value is 60 seconds.
442
+		</emphasis>
443
+		</para>
444
+		<example>
445
+			<title>Set <varname>insert_retry_timeout</varname> parameter</title>
446
+			<programlisting format="linespecific">
447
+...
448
+modparam("sipcapture", "insert_retry_timeout", 10)
449
+...
450
+			</programlisting>
451
+		</example>
452
+	</section>
411 453
 </section>	
412 454
     <section>
413 455
 	<title>MI Commands</title>
... ...
@@ -198,6 +198,8 @@ int db_insert_mode = 0;
198 198
 int promisc_on = 0;
199 199
 int bpf_on = 0;
200 200
 int hep_capture_on   = 0;
201
+int insert_retries = 0;
202
+int insert_retry_timeout = 60;
201 203
 int hep_offset = 0;
202 204
 str raw_socket_listen = { 0, 0 };
203 205
 str raw_interface = { 0, 0 };
... ...
@@ -225,8 +227,6 @@ static struct sock_filter BPF_code[] = { { 0x28, 0, 0, 0x0000000c }, { 0x15, 0,
225 227
 
226 228
 unsigned int no_tables = 0;
227 229
 
228
-
229
-
230 230
 enum e_mt_mode mtmode = mode_random ;
231 231
 enum hash_source source = hs_error;
232 232
 
... ...
@@ -234,7 +234,6 @@ enum hash_source source = hs_error;
234 234
 
235 235
 struct hep_timehdr* heptime;
236 236
 
237
-
238 237
 /*! \brief
239 238
  * Exported functions
240 239
  */
... ...
@@ -306,6 +305,8 @@ static param_export_t params[] = {
306 305
         {"raw_moni_bpf_on",  		INT_PARAM, &bpf_on   },		
307 306
         {"callid_aleg_header",          PARAM_STR, &callid_aleg_header},
308 307
         {"capture_mode",		PARAM_STRING|USE_FUNC_PARAM, (void *)capture_mode_param},
308
+    {"insert_retries",   	INT_PARAM, &insert_retries },
309
+    {"insert_retry_timeout",INT_PARAM, &insert_retry_timeout },
309 310
 		{0, 0, 0}
310 311
 };
311 312
 
... ...
@@ -355,7 +356,6 @@ struct module_exports exports = {
355 356
 };
356 357
 
357 358
 
358
-
359 359
 /* returns number of tables if successful
360 360
  * <0 if failed
361 361
  */
... ...
@@ -788,7 +788,19 @@ static int mod_init(void) {
788 788
 		return -1;		                		
789 789
 	}
790 790
 	
791
+	if ((insert_retries <0) || ( insert_retries > 500)) {
792
+		LM_ERR("insert_retries should be a value between 0 and 500");
793
+		return -1;
794
+	}
791 795
 
796
+	if  (( 0 == insert_retries) && (insert_retry_timeout != 0)){
797
+		LM_ERR("insert_retry_timeout has no meaning when insert_retries is not set");
798
+	}
799
+
800
+	if ((insert_retry_timeout <0) || ( insert_retry_timeout > 300)) {
801
+		LM_ERR("insert_retry_timeout should be a value between 0 and 300");
802
+		return -1;
803
+	}
792 804
 
793 805
 	/* raw processes for IPIP encapsulation */
794 806
 	if (ipip_capture_on || moni_capture_on) {
... ...
@@ -1072,6 +1084,11 @@ static int sip_capture_store(struct _sipcapture_object *sco, str *dtable, _captu
1072 1084
 
1073 1085
 	str tmp;
1074 1086
 	int ii = 0;
1087
+	int ret = 0;
1088
+	int counter = 0;
1089
+	db_insert_f insert;
1090
+	time_t retry_failed_time = 0;
1091
+
1075 1092
 	str *table = NULL;
1076 1093
 	_capture_mode_data_t *c = NULL;
1077 1094
 
... ...
@@ -1319,25 +1336,38 @@ static int sip_capture_store(struct _sipcapture_object *sco, str *dtable, _captu
1319 1336
 	c->db_funcs.use_table(c->db_con, table);
1320 1337
 
1321 1338
 	LM_DBG("storing info...\n");
1322
-	
1323
-	if(db_insert_mode==1 && c->db_funcs.insert_delayed!=NULL) {
1324
-                if (c->db_funcs.insert_delayed(c->db_con, db_keys, db_vals, NR_KEYS) < 0) {
1325
-                	LM_ERR("failed to insert delayed into database\n");
1326
-                        goto error;
1327
-                }
1328
-        } else if (c->db_funcs.insert(c->db_con, db_keys, db_vals, NR_KEYS) < 0) {
1329
-		LM_ERR("failed to insert into database\n");
1330
-                goto error;               
1339
+
1340
+	if (db_insert_mode == 1 && c->db_funcs.insert_delayed != NULL)
1341
+		insert = c->db_funcs.insert_delayed;
1342
+	else
1343
+		insert = c->db_funcs.insert;
1344
+	ret = insert(c->db_con, db_keys, db_vals, NR_KEYS);
1345
+
1346
+	if (ret < 0) {
1347
+		LM_DBG("failed to insert into database(first attempt)\n");
1348
+		if (insert_retries != 0) {
1349
+			counter = 0;
1350
+			while ((ret = insert(c->db_con, db_keys, db_vals, NR_KEYS)) < 0) {
1351
+				counter++;
1352
+				if (1 == counter) //first failed retry
1353
+					retry_failed_time = time(NULL);
1354
+
1355
+				if ((counter > insert_retries) || (time(NULL)
1356
+						- retry_failed_time > insert_retry_timeout)) {
1357
+					LM_ERR("failed to insert into database(second attempt)\n");
1358
+					break;
1359
+				}
1360
+			}
1361
+		}
1331 1362
 	}
1332
-	
1333
-	
1363
+	if (ret < 0)
1364
+		goto error;
1334 1365
 #ifdef STATISTICS
1335 1366
 	update_stat(sco->stat, 1);
1336 1367
 #endif	
1337 1368
 
1338 1369
 	return 1;
1339
-error:
1340
-	return -1;
1370
+	error: return -1;
1341 1371
 }
1342 1372
 
1343 1373
 static int sip_capture(struct sip_msg *msg, str *_table, _capture_mode_data_t * cm_data)