Browse code

db_text: implement fetch and memory constraints

when dealing with large db_text files, pkg_memory is not suitable for
operating the database.

implementing fetch allows modules like presence & registrar & usrloc
to query large tables without constraints on pkg_memory.

creates tmp tables in shared memory for query results

(cherry picked from commit 18c64d2c9ff1527655055f75aa22e7d68c307874)

Conflicts:
src/modules/db_text/db_text.c
src/modules/db_text/dbt_api.c
src/modules/db_text/dbt_base.c
src/modules/db_text/dbt_lib.c

lazedo authored on 18/01/2017 10:00:22 • Luis Azedo committed on 18/01/2017 10:30:32
Showing 1 changed files
... ...
@@ -137,7 +137,7 @@ dbt_table_p dbt_table_new(const str *_tbname, const str *_dbname, const char *pa
137 137
 {
138 138
 	struct stat s;
139 139
 	dbt_table_p dtp = NULL;
140
-	if(!_tbname || !_dbname || !path)
140
+	if(!_tbname || !_dbname)
141 141
 		return NULL;
142 142
 
143 143
 	dtp = (dbt_table_p)shm_malloc(sizeof(dbt_table_t));
... ...
@@ -175,7 +175,7 @@ dbt_table_p dbt_table_new(const str *_tbname, const str *_dbname, const char *pa
175 175
 	dtp->nrrows = dtp->nrcols = dtp->auto_val = 0;
176 176
 	dtp->auto_col = -1;
177 177
 	dtp->mt = 0;
178
-	if(stat(path, &s) == 0)
178
+	if(path && stat(path, &s) == 0)
179 179
 	{
180 180
 		dtp->mt = s.st_mtime;
181 181
 		LM_DBG("mtime is %d\n", (int)s.st_mtime);
Browse code

core, lib, modules: updated include paths for header files

Daniel-Constantin Mierla authored on 07/12/2016 11:07:22
Showing 1 changed files
... ...
@@ -28,10 +28,10 @@
28 28
 #include <sys/stat.h>
29 29
 #include <dirent.h>
30 30
 
31
-#include "../../mem/shm_mem.h"
32
-#include "../../mem/mem.h"
33
-#include "../../dprint.h"
34
-#include "../../locking.h"
31
+#include "../../core/mem/shm_mem.h"
32
+#include "../../core/mem/mem.h"
33
+#include "../../core/dprint.h"
34
+#include "../../core/locking.h"
35 35
 
36 36
 #include "dbt_util.h"
37 37
 #include "dbt_lib.h"
Browse code

core, lib, modules: restructured source code tree

- new folder src/ to hold the source code for main project applications
- main.c is in src/
- all core files are subfolder are in src/core/
- modules are in src/modules/
- libs are in src/lib/
- application Makefiles are in src/
- application binary is built in src/ (src/kamailio)

Daniel-Constantin Mierla authored on 07/12/2016 11:03:51
Showing 1 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,493 @@
1
+/*
2
+ * DBText library
3
+ *
4
+ * Copyright (C) 2001-2003 FhG Fokus
5
+ *
6
+ * This file is part of Kamailio, a free SIP server.
7
+ *
8
+ * Kamailio is free software; you can redistribute it and/or modify
9
+ * it under the terms of the GNU General Public License as published by
10
+ * the Free Software Foundation; either version 2 of the License, or
11
+ * (at your option) any later version
12
+ *
13
+ * Kamailio is distributed in the hope that it will be useful,
14
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16
+ * GNU General Public License for more details.
17
+ *
18
+ * You should have received a copy of the GNU General Public License
19
+ * along with this program; if not, write to the Free Software
20
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
21
+ *
22
+ */
23
+
24
+#include <stdio.h>
25
+#include <string.h>
26
+#include <time.h>
27
+#include <sys/types.h>
28
+#include <sys/stat.h>
29
+#include <dirent.h>
30
+
31
+#include "../../mem/shm_mem.h"
32
+#include "../../mem/mem.h"
33
+#include "../../dprint.h"
34
+#include "../../locking.h"
35
+
36
+#include "dbt_util.h"
37
+#include "dbt_lib.h"
38
+
39
+
40
+/**
41
+ *
42
+ */
43
+dbt_column_p dbt_column_new(char *_s, int _l)
44
+{
45
+	dbt_column_p dcp = NULL;
46
+	if(!_s || _l <=0)
47
+		return NULL;
48
+	dcp = (dbt_column_p)shm_malloc(sizeof(dbt_column_t));
49
+	if(!dcp)
50
+		return NULL;
51
+	dcp->name.s  = (char*)shm_malloc((_l+1)*sizeof(char));
52
+	if(!dcp->name.s)
53
+	{
54
+		shm_free(dcp);
55
+		return NULL;
56
+	}
57
+	dcp->name.len = _l;
58
+	strncpy(dcp->name.s, _s, _l);
59
+	dcp->name.s[_l] = '\0';
60
+	dcp->next = dcp->prev = NULL;
61
+	dcp->type = 0;
62
+	dcp->flag = DBT_FLAG_UNSET;
63
+
64
+	return dcp;
65
+}
66
+
67
+/**
68
+ *
69
+ */
70
+int dbt_column_free(dbt_column_p dcp)
71
+{
72
+
73
+	if(!dcp)
74
+		return -1;
75
+	if(dcp->name.s)
76
+		shm_free(dcp->name.s);
77
+	shm_free(dcp);
78
+
79
+	return 0;
80
+}
81
+
82
+/**
83
+ *
84
+ */
85
+dbt_row_p dbt_row_new(int _nf)
86
+{
87
+	int i;
88
+	dbt_row_p _drp = NULL;
89
+
90
+	_drp = (dbt_row_p)shm_malloc(sizeof(dbt_row_t));
91
+	if(!_drp)
92
+		return NULL;
93
+
94
+	_drp->fields = (dbt_val_p)shm_malloc(_nf*sizeof(dbt_val_t));
95
+	if(!_drp->fields)
96
+	{
97
+		shm_free(_drp);
98
+		return NULL;
99
+	}
100
+	memset(_drp->fields, 0, _nf*sizeof(dbt_val_t));
101
+	for(i=0; i<_nf; i++)
102
+		_drp->fields[i].nul = 1;
103
+
104
+	_drp->next = _drp->prev = NULL;
105
+
106
+	return _drp;
107
+}
108
+
109
+/**
110
+ *
111
+ */
112
+int dbt_row_free(dbt_table_p _dtp, dbt_row_p _drp)
113
+{
114
+	int i;
115
+
116
+	if(!_dtp || !_drp)
117
+		return -1;
118
+
119
+	if(_drp->fields)
120
+	{
121
+		for(i=0; i<_dtp->nrcols; i++)
122
+			if((_dtp->colv[i]->type==DB1_STR || _dtp->colv[i]->type==DB1_STRING
123
+						|| _dtp->colv[i]->type==DB1_BLOB)
124
+					&& _drp->fields[i].val.str_val.s)
125
+				shm_free(_drp->fields[i].val.str_val.s);
126
+		shm_free(_drp->fields);
127
+	}
128
+	shm_free(_drp);
129
+
130
+	return 0;
131
+}
132
+
133
+/**
134
+ *
135
+ */
136
+dbt_table_p dbt_table_new(const str *_tbname, const str *_dbname, const char *path)
137
+{
138
+	struct stat s;
139
+	dbt_table_p dtp = NULL;
140
+	if(!_tbname || !_dbname || !path)
141
+		return NULL;
142
+
143
+	dtp = (dbt_table_p)shm_malloc(sizeof(dbt_table_t));
144
+	if(!dtp)
145
+		goto done;
146
+	memset(dtp, 0, sizeof(dbt_table_t));
147
+	dtp->name.s = (char*)shm_malloc((_tbname->len+1)*sizeof(char));
148
+	if(!dtp->name.s)
149
+	{
150
+		shm_free(dtp);
151
+		dtp = NULL;
152
+		goto done;
153
+	}
154
+	memcpy(dtp->name.s, _tbname->s, _tbname->len);
155
+	dtp->name.s[_tbname->len] = '\0';
156
+	dtp->name.len = _tbname->len;
157
+
158
+	dtp->dbname.s = (char*)shm_malloc((_dbname->len+1)*sizeof(char));
159
+	if(!dtp->dbname.s)
160
+	{
161
+		shm_free(dtp->name.s);
162
+		shm_free(dtp);
163
+		dtp = NULL;
164
+		goto done;
165
+	}
166
+	memcpy(dtp->dbname.s, _dbname->s, _dbname->len);
167
+	dtp->dbname.s[_dbname->len] = '\0';
168
+	dtp->dbname.len = _dbname->len;
169
+
170
+	dtp->rows = NULL;
171
+	dtp->cols = NULL;
172
+	dtp->colv = NULL;
173
+	dtp->mark = (int)time(NULL);
174
+	dtp->flag = DBT_TBFL_ZERO;
175
+	dtp->nrrows = dtp->nrcols = dtp->auto_val = 0;
176
+	dtp->auto_col = -1;
177
+	dtp->mt = 0;
178
+	if(stat(path, &s) == 0)
179
+	{
180
+		dtp->mt = s.st_mtime;
181
+		LM_DBG("mtime is %d\n", (int)s.st_mtime);
182
+	}
183
+
184
+done:
185
+	return dtp;
186
+}
187
+
188
+/**
189
+ *
190
+ */
191
+int dbt_table_free_rows(dbt_table_p _dtp)
192
+{
193
+	dbt_row_p _rp=NULL, _rp0=NULL;
194
+
195
+	if(!_dtp || !_dtp->rows || !_dtp->colv)
196
+		return -1;
197
+	_rp = _dtp->rows;
198
+	while(_rp)
199
+	{
200
+		_rp0=_rp;
201
+		_rp=_rp->next;
202
+		dbt_row_free(_dtp, _rp0);
203
+	}
204
+
205
+	dbt_table_update_flags(_dtp, DBT_TBFL_MODI, DBT_FL_SET, 1);
206
+
207
+	_dtp->rows = NULL;
208
+	_dtp->nrrows = 0;
209
+
210
+	return 0;
211
+}
212
+
213
+/**
214
+ *
215
+ */
216
+int dbt_table_add_row(dbt_table_p _dtp, dbt_row_p _drp)
217
+{
218
+	if(!_dtp || !_drp)
219
+		return -1;
220
+
221
+	if(dbt_table_check_row(_dtp, _drp))
222
+		return -1;
223
+
224
+	dbt_table_update_flags(_dtp, DBT_TBFL_MODI, DBT_FL_SET, 1);
225
+
226
+	if(_dtp->rows)
227
+		(_dtp->rows)->prev = _drp;
228
+	_drp->next = _dtp->rows;
229
+	_dtp->rows = _drp;
230
+	_dtp->nrrows++;
231
+
232
+	return 0;
233
+}
234
+
235
+/**
236
+ *
237
+ */
238
+int dbt_table_free(dbt_table_p _dtp)
239
+{
240
+	dbt_column_p _cp=NULL, _cp0=NULL;
241
+
242
+	if(!_dtp)
243
+		return -1;
244
+
245
+	if(_dtp->name.s)
246
+		shm_free(_dtp->name.s);
247
+	if(_dtp->dbname.s)
248
+		shm_free(_dtp->dbname.s);
249
+
250
+	if(_dtp->rows && _dtp->nrrows>0)
251
+		dbt_table_free_rows(_dtp);
252
+
253
+	_cp = _dtp->cols;
254
+	while(_cp)
255
+	{
256
+		_cp0=_cp;
257
+		_cp=_cp->next;
258
+		dbt_column_free(_cp0);
259
+	}
260
+	if(_dtp->colv)
261
+		shm_free(_dtp->colv);
262
+
263
+	shm_free(_dtp);
264
+
265
+	return 0;
266
+}
267
+
268
+/**
269
+ *
270
+ */
271
+int dbt_row_set_val(dbt_row_p _drp, dbt_val_p _vp, int _t, int _idx)
272
+{
273
+	if(!_drp || !_vp || _idx<0)
274
+		return -1;
275
+
276
+	_drp->fields[_idx].nul = _vp->nul;
277
+	_drp->fields[_idx].type = _t;
278
+
279
+	if(!_vp->nul)
280
+	{
281
+		switch(_t)
282
+		{
283
+			case DB1_STR:
284
+			case DB1_BLOB:
285
+				_drp->fields[_idx].type = _t;
286
+				_drp->fields[_idx].val.str_val.s =
287
+					(char*)shm_malloc((_vp->val.str_val.len+1)*sizeof(char));
288
+				if(!_drp->fields[_idx].val.str_val.s)
289
+				{
290
+					_drp->fields[_idx].nul = 1;
291
+					return -1;
292
+				}
293
+				memcpy(_drp->fields[_idx].val.str_val.s, _vp->val.str_val.s,
294
+					_vp->val.str_val.len);
295
+				_drp->fields[_idx].val.str_val.s[_vp->val.str_val.len] = '\0';
296
+				_drp->fields[_idx].val.str_val.len = _vp->val.str_val.len;
297
+			break;
298
+
299
+			case DB1_STRING:
300
+				_drp->fields[_idx].type = _t;
301
+				_drp->fields[_idx].val.str_val.len=_vp->val.str_val.len;
302
+
303
+				_drp->fields[_idx].val.str_val.s =
304
+					(char*)shm_malloc((_drp->fields[_idx].val.str_val.len+1)
305
+								*sizeof(char));
306
+				if(!_drp->fields[_idx].val.str_val.s)
307
+				{
308
+					_drp->fields[_idx].nul = 1;
309
+					return -1;
310
+				}
311
+				memcpy(_drp->fields[_idx].val.str_val.s, _vp->val.string_val,
312
+					_drp->fields[_idx].val.str_val.len);
313
+				_drp->fields[_idx].val.str_val.s[_drp->fields[_idx].val.str_val.len] = '\0';
314
+			break;
315
+
316
+			case DB1_DOUBLE:
317
+				_drp->fields[_idx].type = DB1_DOUBLE;
318
+				_drp->fields[_idx].val.double_val = _vp->val.double_val;
319
+			break;
320
+
321
+			case DB1_INT:
322
+				_drp->fields[_idx].type = DB1_INT;
323
+				_drp->fields[_idx].val.int_val = _vp->val.int_val;
324
+			break;
325
+
326
+			case DB1_DATETIME:
327
+				_drp->fields[_idx].type = _t;
328
+				_drp->fields[_idx].val.int_val = (int)_vp->val.time_val;
329
+			break;
330
+
331
+			case DB1_BITMAP:
332
+				_drp->fields[_idx].type = DB1_INT;
333
+				_drp->fields[_idx].val.int_val = (int)_vp->val.bitmap_val;
334
+			break;
335
+
336
+			default:
337
+				_drp->fields[_idx].nul = 1;
338
+				return -1;
339
+		}
340
+	}
341
+
342
+	return 0;
343
+}
344
+
345
+/**
346
+ *
347
+ */
348
+int dbt_row_update_val(dbt_row_p _drp, dbt_val_p _vp, int _t, int _idx)
349
+{
350
+	if(!_drp || !_vp || _idx<0)
351
+		return -1;
352
+
353
+	_drp->fields[_idx].nul = _vp->nul;
354
+	_drp->fields[_idx].type = _t;
355
+
356
+	if(!_vp->nul)
357
+	{
358
+		switch(_t)
359
+		{
360
+			case DB1_BLOB:
361
+			case DB1_STR:
362
+				_drp->fields[_idx].type = _t;
363
+				// free if already exists
364
+				if(_drp->fields[_idx].val.str_val.s)
365
+					shm_free(_drp->fields[_idx].val.str_val.s);
366
+
367
+				_drp->fields[_idx].val.str_val.s =
368
+					(char*)shm_malloc((_vp->val.str_val.len+1)*sizeof(char));
369
+				if(!_drp->fields[_idx].val.str_val.s)
370
+				{
371
+					_drp->fields[_idx].nul = 1;
372
+					return -1;
373
+				}
374
+				memcpy(_drp->fields[_idx].val.str_val.s, _vp->val.str_val.s,
375
+					_vp->val.str_val.len);
376
+				_drp->fields[_idx].val.str_val.s[_vp->val.str_val.len] = '\0';
377
+				_drp->fields[_idx].val.str_val.len = _vp->val.str_val.len;
378
+			break;
379
+
380
+			case DB1_STRING:
381
+				/* free if already exists */
382
+				if(_drp->fields[_idx].val.str_val.s)
383
+					shm_free(_drp->fields[_idx].val.str_val.s);
384
+
385
+				_drp->fields[_idx].type = _t;
386
+				if(_vp->type==DB1_STR)
387
+					_drp->fields[_idx].val.str_val.len=_vp->val.str_val.len;
388
+				else
389
+					_drp->fields[_idx].val.str_val.len
390
+											=strlen(_vp->val.string_val);
391
+
392
+				_drp->fields[_idx].val.str_val.s =
393
+					(char*)shm_malloc((_drp->fields[_idx].val.str_val.len+1)
394
+									*sizeof(char));
395
+				if(!_drp->fields[_idx].val.str_val.s)
396
+				{
397
+					_drp->fields[_idx].nul = 1;
398
+					return -1;
399
+				}
400
+				memcpy(_drp->fields[_idx].val.str_val.s, _vp->val.string_val,
401
+					_drp->fields[_idx].val.str_val.len);
402
+				_drp->fields[_idx].val.str_val.s[_vp->val.str_val.len] = '\0';
403
+			break;
404
+
405
+			case DB1_DOUBLE:
406
+				_drp->fields[_idx].type = _t;
407
+				_drp->fields[_idx].val.double_val = _vp->val.double_val;
408
+			break;
409
+
410
+			case DB1_INT:
411
+				_drp->fields[_idx].type = _t;
412
+				_drp->fields[_idx].val.int_val = _vp->val.int_val;
413
+			break;
414
+
415
+			case DB1_DATETIME:
416
+				_drp->fields[_idx].type = _t;
417
+				_drp->fields[_idx].val.int_val = (int)_vp->val.time_val;
418
+			break;
419
+
420
+			case DB1_BITMAP:
421
+				_drp->fields[_idx].type = _t;
422
+				_drp->fields[_idx].val.int_val = (int)_vp->val.bitmap_val;
423
+			break;
424
+
425
+			default:
426
+				LM_ERR("unsupported type %d in update\n",_t);
427
+				_drp->fields[_idx].nul = 1;
428
+				return -1;
429
+		}
430
+	}
431
+
432
+	return 0;
433
+}
434
+
435
+/**
436
+ *
437
+ */
438
+int dbt_table_check_row(dbt_table_p _dtp, dbt_row_p _drp)
439
+{
440
+	int i;
441
+	if(!_dtp || _dtp->nrcols <= 0 || !_drp)
442
+		return -1;
443
+
444
+	for(i=0; i<_dtp->nrcols; i++)
445
+	{
446
+		if(!_drp->fields[i].nul
447
+				&& dbt_is_neq_type(_dtp->colv[i]->type, _drp->fields[i].type))
448
+		{
449
+			LM_ERR("incompatible types - field %d [%d/%d]\n",i,
450
+					_dtp->colv[i]->type, _drp->fields[i].type);
451
+			return -1;
452
+		}
453
+		if(_dtp->colv[i]->flag & DBT_FLAG_NULL)
454
+			continue;
455
+
456
+		if(!_drp->fields[i].nul)
457
+			continue;
458
+
459
+		if(_dtp->colv[i]->type==DB1_INT
460
+			&& (_dtp->colv[i]->flag & DBT_FLAG_AUTO)
461
+			&& i==_dtp->auto_col)
462
+		{
463
+			_drp->fields[i].nul = 0;
464
+			_drp->fields[i].val.int_val = ++_dtp->auto_val;
465
+			continue;
466
+		}
467
+
468
+		LM_ERR("null value not allowed - field %d\n",i);
469
+		return -1;
470
+	}
471
+
472
+	return 0;
473
+}
474
+
475
+/**
476
+ *
477
+ */
478
+int dbt_table_update_flags(dbt_table_p _dtp, int _f, int _o, int _m)
479
+{
480
+	if(!_dtp)
481
+		return -1;
482
+
483
+	if(_o == DBT_FL_SET)
484
+		_dtp->flag |= _f;
485
+	else if(_o == DBT_FL_UNSET)
486
+			_dtp->flag &= ~_f;
487
+
488
+	if(_m)
489
+		_dtp->mark = (int)time(NULL);
490
+
491
+	return 0;
492
+}
493
+