Browse code

db_text: implement fetch and memory constraints

when dealing with large db_text files, pkg_memory is not suitable for
operating the database.

implementing fetch allows modules like presence & registrar & usrloc
to query large tables without constraints on pkg_memory.

creates tmp tables in shared memory for query results

(cherry picked from commit 18c64d2c9ff1527655055f75aa22e7d68c307874)

Conflicts:
src/modules/db_text/db_text.c
src/modules/db_text/dbt_api.c
src/modules/db_text/dbt_base.c
src/modules/db_text/dbt_lib.c

lazedo authored on 18/01/2017 10:00:22 • Luis Azedo committed on 18/01/2017 10:30:32
Showing 1 changed files
... ...
@@ -39,7 +39,7 @@ int dbt_use_table(db1_con_t* _h, const str* _t)
39 39
 /*
40 40
  * Get and convert columns from a result
41 41
  */
42
-static int dbt_get_columns(db1_res_t* _r, dbt_result_p _dres)
42
+static int dbt_get_columns(db1_res_t* _r, dbt_table_p _dres)
43 43
 {
44 44
 	int col;
45 45
 
... ...
@@ -73,10 +73,10 @@ static int dbt_get_columns(db1_res_t* _r, dbt_result_p _dres)
73 73
 		LM_DBG("allocate %d bytes for RES_NAMES[%d] at %p\n",
74 74
 				(int)sizeof(str), col,
75 75
 				RES_NAMES(_r)[col]);
76
-		RES_NAMES(_r)[col]->s = _dres->colv[col].name.s;
77
-		RES_NAMES(_r)[col]->len = _dres->colv[col].name.len;
76
+		RES_NAMES(_r)[col]->s = _dres->colv[col]->name.s;
77
+		RES_NAMES(_r)[col]->len = _dres->colv[col]->name.len;
78 78
 
79
-		switch(_dres->colv[col].type)
79
+		switch(_dres->colv[col]->type)
80 80
 		{
81 81
 			case DB1_STR:
82 82
 			case DB1_STRING:
... ...
@@ -84,12 +84,12 @@ static int dbt_get_columns(db1_res_t* _r, dbt_result_p _dres)
84 84
 			case DB1_INT:
85 85
 			case DB1_DATETIME:
86 86
 			case DB1_DOUBLE:
87
-				RES_TYPES(_r)[col] = _dres->colv[col].type;
87
+				RES_TYPES(_r)[col] = _dres->colv[col]->type;
88 88
 			break;
89 89
 			default:
90 90
 				LM_WARN("unhandled data type column (%.*s) type id (%d), "
91 91
 						"use STR as default\n", RES_NAMES(_r)[col]->len,
92
-						RES_NAMES(_r)[col]->s, _dres->colv[col].type);
92
+						RES_NAMES(_r)[col]->s, _dres->colv[col]->type);
93 93
 				RES_TYPES(_r)[col] = DB1_STR;
94 94
 			break;
95 95
 		}
... ...
@@ -173,7 +173,7 @@ static int dbt_convert_row(db1_res_t* _res, db_row_t* _r, dbt_row_p _r1)
173 173
 			break;
174 174
 
175 175
 			default:
176
-				LM_ERR("val type [%d] not supported\n", RES_TYPES(_res)[i]);
176
+				LM_ERR("val type [%d] for column %i not supported\n", RES_TYPES(_res)[i], i);
177 177
 				return -1;
178 178
 		}
179 179
 	}
... ...
@@ -184,25 +184,31 @@ static int dbt_convert_row(db1_res_t* _res, db_row_t* _r, dbt_row_p _r1)
184 184
 /*
185 185
  * Convert rows from internal to db API representation
186 186
  */
187
-static int dbt_convert_rows(db1_res_t* _r, dbt_result_p _dres)
187
+static int dbt_convert_rows(db1_res_t* _r, dbt_table_p _dres, int offset, int nrows)
188 188
 {
189
-	int row;
189
+	int row = 0, c = 0;
190 190
 	dbt_row_p _rp = NULL;
191 191
 	if (!_r || !_dres) {
192 192
 		LM_ERR("invalid parameter\n");
193 193
 		return -1;
194 194
 	}
195
-	RES_ROW_N(_r) = _dres->nrrows;
196
-	if (!RES_ROW_N(_r)) {
195
+
196
+	if (nrows == 0) {
197 197
 		return 0;
198 198
 	}
199
+
199 200
 	if (db_allocate_rows(_r) < 0) {
200 201
 		LM_ERR("could not allocate rows\n");
201 202
 		return -2;
202 203
 	}
203
-	row = 0;
204
+
204 205
 	_rp = _dres->rows;
205
-	while(_rp) {
206
+	while(_rp && c < offset) {
207
+		c++;
208
+		_rp = _rp->next;
209
+	}
210
+
211
+	while(_rp && row < nrows) {
206 212
 		if (dbt_convert_row(_r, &(RES_ROWS(_r)[row]), _rp) < 0) {
207 213
 			LM_ERR("failed to convert row #%d\n", row);
208 214
 			RES_ROW_N(_r) = row;
... ...
@@ -212,36 +218,76 @@ static int dbt_convert_rows(db1_res_t* _r, dbt_result_p _dres)
212 218
 		row++;
213 219
 		_rp = _rp->next;
214 220
 	}
221
+	RES_ROW_N(_r) = row;
222
+	RES_LAST_ROW(_r) = c + row;
215 223
 	return 0;
216 224
 }
217 225
 
226
+static int dbt_convert_all_rows(db1_res_t* _r, dbt_table_p _dres)
227
+{
228
+	if (!_r || !_dres) {
229
+		LM_ERR("invalid parameter\n");
230
+		return -1;
231
+	}
232
+	RES_ROW_N(_r) = _dres->nrrows;
233
+	return dbt_convert_rows(_r, _dres, 0, _dres->nrrows);
234
+}
235
+
236
+
218 237
 
219 238
 /*
220 239
  * Fill the structure with data from database
221 240
  */
222
-static int dbt_convert_result(db1_res_t* _r, dbt_result_p _dres)
241
+//static int dbt_convert_result(db1_res_t* _r, dbt_table_p _dres)
242
+//{
243
+//	if (!_r || !_dres) {
244
+//		LM_ERR("invalid parameter\n");
245
+//		return -1;
246
+//	}
247
+//	if (dbt_get_columns(_r, _dres) < 0) {
248
+//		LM_ERR("failed to get column names\n");
249
+//		return -2;
250
+//	}
251
+//
252
+//	if (dbt_convert_all_rows(_r, _dres) < 0) {
253
+//		LM_ERR("failed to convert rows\n");
254
+//		db_free_columns(_r);
255
+//		return -3;
256
+//	}
257
+//	return 0;
258
+//}
259
+
260
+/*
261
+ * Retrieve result set
262
+ */
263
+int dbt_get_result(db1_res_t** _r, dbt_table_p _dres)
223 264
 {
224
-	if (!_r || !_dres) {
225
-		LM_ERR("invalid parameter\n");
226
-		return -1;
265
+	int res = dbt_init_result(_r, _dres);
266
+	if ( res != 0) {
267
+		return res;
227 268
 	}
228
-	if (dbt_get_columns(_r, _dres) < 0) {
229
-		LM_ERR("failed to get column names\n");
230
-		return -2;
269
+
270
+	if (dbt_convert_all_rows(*_r, _dres) < 0) {
271
+		LM_ERR("failed to convert rows\n");
272
+		db_free_columns(*_r);
273
+		return -3;
231 274
 	}
232 275
 
233
-	if (dbt_convert_rows(_r, _dres) < 0) {
276
+	return 0;
277
+}
278
+
279
+int dbt_get_next_result(db1_res_t** _r, int offset, int rows)
280
+{
281
+	dbt_table_p _dres = (dbt_table_p)(*_r)->ptr;
282
+	if (dbt_convert_rows(*_r, _dres, offset, rows) < 0) {
234 283
 		LM_ERR("failed to convert rows\n");
235
-		db_free_columns(_r);
284
+		db_free_columns(*_r);
236 285
 		return -3;
237 286
 	}
238 287
 	return 0;
239 288
 }
240 289
 
241
-/*
242
- * Retrieve result set
243
- */
244
-int dbt_get_result(db1_res_t** _r, dbt_result_p _dres)
290
+int dbt_init_result(db1_res_t** _r, dbt_table_p _dres)
245 291
 {
246 292
 	if ( !_r) {
247 293
 		LM_ERR("invalid parameter value\n");
... ...
@@ -262,13 +308,12 @@ int dbt_get_result(db1_res_t** _r, dbt_result_p _dres)
262 308
 		return -2;
263 309
 	}
264 310
 
265
-	if (dbt_convert_result(*_r, _dres) < 0)
266
-	{
267
-		LM_ERR("failed to convert result\n");
268
-		pkg_free(*_r);
269
-		return -4;
311
+	if (dbt_get_columns(*_r, _dres) < 0) {
312
+		LM_ERR("failed to get column names\n");
313
+		return -2;
270 314
 	}
271 315
 
316
+	RES_NUM_ROWS(*_r) = _dres->nrrows;
272 317
 	(*_r)->ptr = _dres;
273 318
 	return 0;
274 319
 }
Browse code

core, lib, modules: updated include paths for header files

Daniel-Constantin Mierla authored on 07/12/2016 11:07:22
Showing 1 changed files
... ...
@@ -25,7 +25,7 @@
25 25
 #include <string.h>
26 26
 
27 27
 #include "../../lib/srdb1/db.h"
28
-#include "../../mem/mem.h"
28
+#include "../../core/mem/mem.h"
29 29
 
30 30
 #include "dbt_res.h"
31 31
 #include "dbt_api.h"
Browse code

core, lib, modules: restructured source code tree

- new folder src/ to hold the source code for main project applications
- main.c is in src/
- all core files are subfolder are in src/core/
- modules are in src/modules/
- libs are in src/lib/
- application Makefiles are in src/
- application binary is built in src/ (src/kamailio)

Daniel-Constantin Mierla authored on 07/12/2016 11:03:51
Showing 1 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,274 @@
1
+/*
2
+ * DBText library
3
+ *
4
+ * Copyright (C) 2001-2003 FhG Fokus
5
+ *
6
+ * This file is part of Kamailio, a free SIP server.
7
+ *
8
+ * Kamailio is free software; you can redistribute it and/or modify
9
+ * it under the terms of the GNU General Public License as published by
10
+ * the Free Software Foundation; either version 2 of the License, or
11
+ * (at your option) any later version
12
+ *
13
+ * Kamailio is distributed in the hope that it will be useful,
14
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16
+ * GNU General Public License for more details.
17
+ *
18
+ * You should have received a copy of the GNU General Public License
19
+ * along with this program; if not, write to the Free Software
20
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
21
+ *
22
+ *
23
+ */
24
+
25
+#include <string.h>
26
+
27
+#include "../../lib/srdb1/db.h"
28
+#include "../../mem/mem.h"
29
+
30
+#include "dbt_res.h"
31
+#include "dbt_api.h"
32
+
33
+int dbt_use_table(db1_con_t* _h, const str* _t)
34
+{
35
+	return db_use_table(_h, _t);
36
+}
37
+
38
+
39
+/*
40
+ * Get and convert columns from a result
41
+ */
42
+static int dbt_get_columns(db1_res_t* _r, dbt_result_p _dres)
43
+{
44
+	int col;
45
+
46
+	if (!_r || !_dres) {
47
+		LM_ERR("invalid parameter\n");
48
+		return -1;
49
+	}
50
+
51
+	RES_COL_N(_r) = _dres->nrcols;
52
+	if (!RES_COL_N(_r)) {
53
+		LM_ERR("no columns\n");
54
+		return -2;
55
+	}
56
+	if (db_allocate_columns(_r, RES_COL_N(_r)) != 0) {
57
+		LM_ERR("could not allocate columns\n");
58
+		return -3;
59
+	}
60
+
61
+	for(col = 0; col < RES_COL_N(_r); col++) {
62
+		/*
63
+		 * Its would be not necessary to allocate here new memory, because of
64
+		 * the internal structure of the db_text module. But we do this anyway
65
+		 * to stay confirm to the other database modules.
66
+		 */
67
+		RES_NAMES(_r)[col] = (str*)pkg_malloc(sizeof(str));
68
+		if (! RES_NAMES(_r)[col]) {
69
+			LM_ERR("no private memory left\n");
70
+			db_free_columns(_r);
71
+			return -4;
72
+		}
73
+		LM_DBG("allocate %d bytes for RES_NAMES[%d] at %p\n",
74
+				(int)sizeof(str), col,
75
+				RES_NAMES(_r)[col]);
76
+		RES_NAMES(_r)[col]->s = _dres->colv[col].name.s;
77
+		RES_NAMES(_r)[col]->len = _dres->colv[col].name.len;
78
+
79
+		switch(_dres->colv[col].type)
80
+		{
81
+			case DB1_STR:
82
+			case DB1_STRING:
83
+			case DB1_BLOB:
84
+			case DB1_INT:
85
+			case DB1_DATETIME:
86
+			case DB1_DOUBLE:
87
+				RES_TYPES(_r)[col] = _dres->colv[col].type;
88
+			break;
89
+			default:
90
+				LM_WARN("unhandled data type column (%.*s) type id (%d), "
91
+						"use STR as default\n", RES_NAMES(_r)[col]->len,
92
+						RES_NAMES(_r)[col]->s, _dres->colv[col].type);
93
+				RES_TYPES(_r)[col] = DB1_STR;
94
+			break;
95
+		}
96
+	}
97
+	return 0;
98
+}
99
+
100
+/*
101
+ * Convert a row from result into db API representation
102
+ */
103
+static int dbt_convert_row(db1_res_t* _res, db_row_t* _r, dbt_row_p _r1)
104
+{
105
+	int i;
106
+	if (!_r || !_res || !_r1) {
107
+		LM_ERR("invalid parameter value\n");
108
+		return -1;
109
+	}
110
+
111
+	if (db_allocate_row(_res, _r) != 0) {
112
+		LM_ERR("could not allocate row\n");
113
+		return -2;
114
+	}
115
+
116
+	for(i = 0; i < RES_COL_N(_res); i++) {
117
+		(ROW_VALUES(_r)[i]).nul = _r1->fields[i].nul;
118
+		switch(RES_TYPES(_res)[i])
119
+		{
120
+			case DB1_INT:
121
+				VAL_INT(&(ROW_VALUES(_r)[i])) =
122
+						_r1->fields[i].val.int_val;
123
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_INT;
124
+			break;
125
+
126
+			case DB1_BIGINT:
127
+				LM_ERR("BIGINT not supported\n");
128
+				return -1;
129
+
130
+			case DB1_DOUBLE:
131
+				VAL_DOUBLE(&(ROW_VALUES(_r)[i])) =
132
+						_r1->fields[i].val.double_val;
133
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_DOUBLE;
134
+			break;
135
+
136
+			case DB1_STRING:
137
+				VAL_STR(&(ROW_VALUES(_r)[i])).s =
138
+						_r1->fields[i].val.str_val.s;
139
+				VAL_STR(&(ROW_VALUES(_r)[i])).len =
140
+						_r1->fields[i].val.str_val.len;
141
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_STRING;
142
+				VAL_FREE(&(ROW_VALUES(_r)[i])) = 0;
143
+			break;
144
+
145
+			case DB1_STR:
146
+				VAL_STR(&(ROW_VALUES(_r)[i])).s =
147
+						_r1->fields[i].val.str_val.s;
148
+				VAL_STR(&(ROW_VALUES(_r)[i])).len =
149
+						_r1->fields[i].val.str_val.len;
150
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_STR;
151
+				VAL_FREE(&(ROW_VALUES(_r)[i])) = 0;
152
+			break;
153
+
154
+			case DB1_DATETIME:
155
+				VAL_INT(&(ROW_VALUES(_r)[i])) =
156
+						_r1->fields[i].val.int_val;
157
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_DATETIME;
158
+			break;
159
+
160
+			case DB1_BLOB:
161
+				VAL_STR(&(ROW_VALUES(_r)[i])).s =
162
+						_r1->fields[i].val.str_val.s;
163
+				VAL_STR(&(ROW_VALUES(_r)[i])).len =
164
+						_r1->fields[i].val.str_val.len;
165
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_BLOB;
166
+				VAL_FREE(&(ROW_VALUES(_r)[i])) = 0;
167
+			break;
168
+
169
+			case DB1_BITMAP:
170
+				VAL_INT(&(ROW_VALUES(_r)[i])) =
171
+						_r1->fields[i].val.bitmap_val;
172
+				VAL_TYPE(&(ROW_VALUES(_r)[i])) = DB1_INT;
173
+			break;
174
+
175
+			default:
176
+				LM_ERR("val type [%d] not supported\n", RES_TYPES(_res)[i]);
177
+				return -1;
178
+		}
179
+	}
180
+	return 0;
181
+}
182
+
183
+
184
+/*
185
+ * Convert rows from internal to db API representation
186
+ */
187
+static int dbt_convert_rows(db1_res_t* _r, dbt_result_p _dres)
188
+{
189
+	int row;
190
+	dbt_row_p _rp = NULL;
191
+	if (!_r || !_dres) {
192
+		LM_ERR("invalid parameter\n");
193
+		return -1;
194
+	}
195
+	RES_ROW_N(_r) = _dres->nrrows;
196
+	if (!RES_ROW_N(_r)) {
197
+		return 0;
198
+	}
199
+	if (db_allocate_rows(_r) < 0) {
200
+		LM_ERR("could not allocate rows\n");
201
+		return -2;
202
+	}
203
+	row = 0;
204
+	_rp = _dres->rows;
205
+	while(_rp) {
206
+		if (dbt_convert_row(_r, &(RES_ROWS(_r)[row]), _rp) < 0) {
207
+			LM_ERR("failed to convert row #%d\n", row);
208
+			RES_ROW_N(_r) = row;
209
+			db_free_rows(_r);
210
+			return -4;
211
+		}
212
+		row++;
213
+		_rp = _rp->next;
214
+	}
215
+	return 0;
216
+}
217
+
218
+
219
+/*
220
+ * Fill the structure with data from database
221
+ */
222
+static int dbt_convert_result(db1_res_t* _r, dbt_result_p _dres)
223
+{
224
+	if (!_r || !_dres) {
225
+		LM_ERR("invalid parameter\n");
226
+		return -1;
227
+	}
228
+	if (dbt_get_columns(_r, _dres) < 0) {
229
+		LM_ERR("failed to get column names\n");
230
+		return -2;
231
+	}
232
+
233
+	if (dbt_convert_rows(_r, _dres) < 0) {
234
+		LM_ERR("failed to convert rows\n");
235
+		db_free_columns(_r);
236
+		return -3;
237
+	}
238
+	return 0;
239
+}
240
+
241
+/*
242
+ * Retrieve result set
243
+ */
244
+int dbt_get_result(db1_res_t** _r, dbt_result_p _dres)
245
+{
246
+	if ( !_r) {
247
+		LM_ERR("invalid parameter value\n");
248
+		return -1;
249
+	}
250
+
251
+	if (!_dres)
252
+	{
253
+		LM_ERR("failed to get result\n");
254
+		*_r = 0;
255
+		return -3;
256
+	}
257
+
258
+	*_r = db_new_result();
259
+	if (*_r == 0)
260
+	{
261
+		LM_ERR("no private memory left\n");
262
+		return -2;
263
+	}
264
+
265
+	if (dbt_convert_result(*_r, _dres) < 0)
266
+	{
267
+		LM_ERR("failed to convert result\n");
268
+		pkg_free(*_r);
269
+		return -4;
270
+	}
271
+
272
+	(*_r)->ptr = _dres;
273
+	return 0;
274
+}