Browse code

db_redis: performance improvements and fixes

- Support update of type key columns
- Support range scans on timestamp and int keys through a series of
wildcard matches when used with a < or > operator
- Support exponential increase and fallof for SCAN
- Pad bigint values to 10 digits for faster wildcard matching
- Use KEYS instead of SCAN by default for faster keys matching
- Support optional versioning of table names
- Simulate non-unique indexes through parent sets for O(1) counting of
entries

Richard Fuchs authored on 22/11/2019 13:35:49
Showing 6 changed files
... ...
@@ -23,6 +23,7 @@
23 23
 #include "db_redis_mod.h"
24 24
 #include "redis_connection.h"
25 25
 #include "redis_table.h"
26
+#include "redis_dbase.h"
26 27
 
27 28
 extern int db_redis_verbosity;
28 29
 
... ...
@@ -170,6 +171,31 @@ int db_redis_connect(km_redis_con_t *con) {
170 171
     freeReplyObject(reply); reply = NULL;
171 172
     LM_DBG("connection opened to %.*s\n", con->id->url.len, con->id->url.s);
172 173
 
174
+    reply = redisCommand(con->con, "SCRIPT LOAD %s", SREM_KEY_LUA);
175
+    if (!reply) {
176
+        LM_ERR("failed to load LUA script to server %.*s: %s\n",
177
+                con->id->url.len, con->id->url.s, con->con->errstr);
178
+        goto err;
179
+    }
180
+    if (reply->type == REDIS_REPLY_ERROR) {
181
+        LM_ERR("failed to load LUA script to server %.*s: %s\n",
182
+                con->id->url.len, con->id->url.s, reply->str);
183
+        goto err;
184
+    }
185
+    if (reply->type != REDIS_REPLY_STRING) {
186
+        LM_ERR("failed to load LUA script to server %.*s: %i\n",
187
+                con->id->url.len, con->id->url.s, reply->type);
188
+        goto err;
189
+    }
190
+    if (reply->len >= sizeof(con->srem_key_lua)) {
191
+        LM_ERR("failed to load LUA script to server %.*s: %i >= %i\n",
192
+                con->id->url.len, con->id->url.s, (int) reply->len, (int) sizeof(con->srem_key_lua));
193
+        goto err;
194
+    }
195
+    strcpy(con->srem_key_lua, reply->str);
196
+    freeReplyObject(reply); reply = NULL;
197
+    LM_DBG("connection opened to %.*s\n", con->id->url.len, con->id->url.s);
198
+
173 199
     return 0;
174 200
 
175 201
 err:
... ...
@@ -66,6 +66,7 @@ typedef struct km_redis_con {
66 66
     redis_command_t *command_queue;
67 67
     unsigned int append_counter;
68 68
     struct str_hash_table tables;
69
+    char srem_key_lua[41]; // sha-1 hex string
69 70
 } km_redis_con_t;
70 71
 
71 72
 
... ...
@@ -27,6 +27,8 @@
27 27
 #include "redis_dbase.h"
28 28
 #include "redis_table.h"
29 29
 
30
+#define TIMESTAMP_STR_LENGTH 19
31
+
30 32
 static void db_redis_dump_reply(redisReply *reply) {
31 33
     int i;
32 34
     if (reply->type == REDIS_REPLY_STRING) {
... ...
@@ -107,14 +109,14 @@ static int db_redis_val2str(const db_val_t *v, str *_str) {
107 109
             LM_DBG("converting bigint value %lld to str\n", VAL_BIGINT(v));
108 110
             _str->s = (char*)pkg_malloc(_str->len);
109 111
             if (!_str->s) goto memerr;
110
-            snprintf(_str->s, _str->len, "%lld", VAL_BIGINT(v));
112
+            snprintf(_str->s, _str->len, "%010lld", VAL_BIGINT(v));
111 113
             _str->len = strlen(_str->s);
112 114
             break;
113 115
         case DB1_UBIGINT:
114 116
             LM_DBG("converting ubigint value %llu to str\n", VAL_UBIGINT(v));
115 117
             _str->s = (char*)pkg_malloc(_str->len);
116 118
             if (!_str->s) goto memerr;
117
-            snprintf(_str->s, _str->len, "%llu", VAL_UBIGINT(v));
119
+            snprintf(_str->s, _str->len, "%010llu", VAL_UBIGINT(v));
118 120
             _str->len = strlen(_str->s);
119 121
             break;
120 122
         case DB1_STRING:
... ...
@@ -279,8 +281,9 @@ err:
279 281
 }
280 282
 
281 283
 static int db_redis_find_query_key(redis_key_t *key, const str *table_name,
282
-        str *type_name, const db_key_t *_k, const db_val_t *_v, const int _n,
283
-        str *key_name, int *key_found) {
284
+        redis_table_t *table,
285
+        str *type_name, const db_key_t *_k, const db_val_t *_v, const db_op_t *_op, const int _n,
286
+        str *key_name, int *key_found, uint64_t *ts_scan_start) {
284 287
 
285 288
     unsigned int len;
286 289
     str val = {NULL, 0};
... ...
@@ -297,11 +300,19 @@ static int db_redis_find_query_key(redis_key_t *key, const str *table_name,
297 300
         for (i = 0; i < _n; ++i) {
298 301
             const db_key_t k = _k[i];
299 302
             const db_val_t v = _v[i];
303
+            const db_op_t op = _op ? _op[i] : NULL;
300 304
 
301 305
             if (VAL_NULL(&v)) {
302 306
                 LM_DBG("Skipping null value for given key '%.*s'\n",
303 307
                         k->len, k->s);
304 308
                 break;
309
+            } else if (op && strcmp(op, OP_EQ)
310
+                    && !((VAL_TYPE(&v) == DB1_DATETIME || VAL_TYPE(&v) == DB1_BIGINT
311
+                            || VAL_TYPE(&v) == DB1_UBIGINT)
312
+                        && (!strcmp(op, OP_LT) || !strcmp(op, OP_GT)))) {
313
+                LM_DBG("Skipping non-EQ op (%s) for given key '%.*s'\n",
314
+                        op, k->len, k->s);
315
+                break;
305 316
             } else if (!str_strcmp(&key->key, (str*)k)) {
306 317
                 LM_DBG("found key in entry key\n");
307 318
                 if (db_redis_val2str(&v, &val) != 0) goto err;
... ...
@@ -311,14 +322,15 @@ static int db_redis_find_query_key(redis_key_t *key, const str *table_name,
311 322
                     break;
312 323
                 }
313 324
                 if (!key_name->len) {
314
-                    // <table_name>:<type>::<val>
315
-                    len = table_name->len + 1 + type_name->len + 2 + val.len + 1; //snprintf writes term 0 char
325
+                    // <version>:<table_name>:<type>::<val>
326
+                    len = table->version_code.len + table_name->len + 1 + type_name->len + 2 + val.len + 1; //snprintf writes term 0 char
316 327
                     key_name->s = (char*)pkg_malloc(len);
317 328
                     if (!key_name->s) {
318 329
                         LM_ERR("Failed to allocate key memory\n");
319 330
                         goto err;
320 331
                     }
321
-                    snprintf(key_name->s, len, "%.*s:%.*s::%.*s",
332
+                    snprintf(key_name->s, len, "%.*s%.*s:%.*s::%.*s",
333
+                            table->version_code.len, table->version_code.s,
322 334
                             table_name->len, table_name->s,
323 335
                             type_name->len, type_name->s,
324 336
                             val.len, val.s);
... ...
@@ -335,6 +347,39 @@ static int db_redis_find_query_key(redis_key_t *key, const str *table_name,
335 347
                             val.len, val.s);
336 348
                     key_name->len += (1 + val.len);
337 349
                 }
350
+                if (op && (VAL_TYPE(&v) == DB1_DATETIME || VAL_TYPE(&v) == DB1_BIGINT
351
+                            || VAL_TYPE(&v) == DB1_UBIGINT)
352
+                        && (!strcmp(op, OP_LT) || !strcmp(op, OP_GT))) {
353
+                    // Special case: we support matching < or > against timestamps and ints using a special
354
+                    // key scanning method. We do this only for a single timestamp/int occurance, and we
355
+                    // still do a table scan, just not a full table scan.
356
+                    if (!ts_scan_start) {
357
+                        LM_DBG("key '%.*s' for type '%.*s' found as timestamp or int, but table scans "
358
+                                "not supported, unable to use this type\n",
359
+                                key->key.len, key->key.s, type_name->len, type_name->s);
360
+                        break;
361
+                    }
362
+                    // ts_scan_start is: 31 bits of current full key length, 31 bits of this value length,
363
+                    // one bit of directionality, one bit of length variable indicator
364
+                    if (VAL_TYPE(&v) == DB1_DATETIME && *ts_scan_start == 0 && val.len == TIMESTAMP_STR_LENGTH) {
365
+                        *ts_scan_start = key_name->len | ((uint64_t) TIMESTAMP_STR_LENGTH << 31);
366
+                        if (!strcmp(op, OP_LT))
367
+                            *ts_scan_start |= 0x8000000000000000ULL;
368
+                        LM_DBG("preparing for timestamp range scan at key offset %llx\n",
369
+                                    (unsigned long long) *ts_scan_start);
370
+                        *key_found = 0; // this forces a table scan using the new match key
371
+                    }
372
+                    else if ((VAL_TYPE(&v) == DB1_BIGINT
373
+                                || VAL_TYPE(&v) == DB1_UBIGINT) && *ts_scan_start == 0) {
374
+                        *ts_scan_start = key_name->len | ((uint64_t) val.len << 31);
375
+                        *ts_scan_start |= 0x4000000000000000ULL; // length is variable
376
+                        if (!strcmp(op, OP_LT))
377
+                            *ts_scan_start |= 0x8000000000000000ULL;
378
+                        LM_DBG("preparing for int range scan at key offset %llx\n",
379
+                                    (unsigned long long) *ts_scan_start);
380
+                        *key_found = 0; // this forces a table scan using the new match key
381
+                    }
382
+                }
338 383
                 LM_DBG("entry key so far is '%.*s'\n", key_name->len, key_name->s);
339 384
                 subkey_found = 1;
340 385
                 pkg_free(val.s);
... ...
@@ -355,6 +400,22 @@ static int db_redis_find_query_key(redis_key_t *key, const str *table_name,
355 400
         }
356 401
     }
357 402
 
403
+    // for value-less master keys
404
+    if (!key_name->len) {
405
+        // <version>:<table_name>:<type>
406
+        len = table->version_code.len + table_name->len + 1 + type_name->len + 1;
407
+        key_name->s = (char*)pkg_malloc(len);
408
+        if (!key_name->s) {
409
+            LM_ERR("Failed to allocate key memory\n");
410
+            goto err;
411
+        }
412
+        snprintf(key_name->s, len, "%.*s%.*s:%.*s",
413
+                table->version_code.len, table->version_code.s,
414
+                table_name->len, table_name->s,
415
+                type_name->len, type_name->s);
416
+        key_name->len = len-1;
417
+    }
418
+
358 419
     return 0;
359 420
 
360 421
 err:
... ...
@@ -389,7 +450,7 @@ static int db_redis_build_entry_keys(km_redis_con_t *con, const str *table_name,
389 450
     }
390 451
     table = (redis_table_t*)table_e->u.p;
391 452
     key = table->entry_keys;
392
-    if (db_redis_find_query_key(key, table_name, &type_name, _k, _v, _n, &keyname, &key_found) != 0) {
453
+    if (db_redis_find_query_key(key, table_name, table, &type_name, _k, _v, NULL, _n, &keyname, &key_found, NULL) != 0) {
393 454
         goto err;
394 455
     }
395 456
     if (key_found) {
... ...
@@ -400,11 +461,12 @@ static int db_redis_build_entry_keys(km_redis_con_t *con, const str *table_name,
400 461
         LM_DBG("found suitable entry key '%.*s' for query\n",
401 462
                 (*keys)->key.len, (*keys)->key.s);
402 463
         *keys_count = 1;
403
-        pkg_free(keyname.s);
404 464
     } else {
405 465
         LM_ERR("Failed to create direct entry key, no matching key definition\n");
406 466
         goto err;
407 467
     }
468
+    if (keyname.s)
469
+        pkg_free(keyname.s);
408 470
 
409 471
     return 0;
410 472
 
... ...
@@ -453,7 +515,7 @@ err:
453 515
 
454 516
 static int db_redis_build_type_keys(km_redis_con_t *con, const str *table_name,
455 517
         const db_key_t *_k, const db_val_t *_v, const int _n,
456
-        redis_key_t **keys, int *keys_count) {
518
+        redis_key_t **keys, redis_key_t **set_keys, int *keys_count) {
457 519
 
458 520
     struct str_hash_entry *table_e;
459 521
     redis_table_t *table;
... ...
@@ -479,7 +541,7 @@ static int db_redis_build_type_keys(km_redis_con_t *con, const str *table_name,
479 541
         str keyname = {NULL, 0};
480 542
         key = type->keys;
481 543
 
482
-        if (db_redis_find_query_key(key, table_name, &type->type, _k, _v, _n, &keyname, &key_found) != 0) {
544
+        if (db_redis_find_query_key(key, table_name, table, &type->type, _k, _v, NULL, _n, &keyname, &key_found, NULL) != 0) {
483 545
             goto err;
484 546
         }
485 547
         if (key_found) {
... ...
@@ -491,8 +553,29 @@ static int db_redis_build_type_keys(km_redis_con_t *con, const str *table_name,
491 553
             LM_DBG("found key '%.*s' for type '%.*s'\n",
492 554
                     keyname.len, keyname.s,
493 555
                     type_name->len, type_name->s);
494
-            pkg_free(keyname.s);
556
+
557
+            if (set_keys) {
558
+                // add key for parent set
559
+                // <version>:<table>::index::<type>
560
+                pkg_free(keyname.s);
561
+                keyname.len = table->version_code.len + table_name->len + 9 + type->type.len;
562
+                keyname.s = pkg_malloc(keyname.len + 1);
563
+                if (!keyname.s) {
564
+                    LM_ERR("Failed to allocate memory for parent set key\n");
565
+                    goto err;
566
+                }
567
+                sprintf(keyname.s, "%.*s%.*s::index::%.*s",
568
+                        table->version_code.len, table->version_code.s,
569
+                        table_name->len, table_name->s,
570
+                        type->type.len, type->type.s);
571
+                if (db_redis_key_add_str(set_keys, &keyname) != 0) {
572
+                    LM_ERR("Failed to add query key to set key list\n");
573
+                    goto err;
574
+                }
575
+            }
495 576
         }
577
+        if (keyname.s)
578
+            pkg_free(keyname.s);
496 579
     }
497 580
 
498 581
     return 0;
... ...
@@ -506,7 +589,7 @@ err:
506 589
 static int db_redis_build_query_keys(km_redis_con_t *con, const str *table_name,
507 590
         const db_key_t *_k, const db_val_t *_v, const db_op_t *_op, const int _n,
508 591
         redis_key_t **query_keys, int *query_keys_count, int **manual_keys, int *manual_keys_count,
509
-        int *do_table_scan) {
592
+        int *do_table_scan, uint64_t *ts_scan_start, str *ts_scan_key) {
510 593
 
511 594
     struct str_hash_entry *table_e;
512 595
     redis_table_t *table;
... ...
@@ -536,7 +619,7 @@ static int db_redis_build_query_keys(km_redis_con_t *con, const str *table_name,
536 619
     keyname.len = 0;
537 620
     key = table->entry_keys;
538 621
 
539
-    if (db_redis_find_query_key(key, table_name, &typename, _k, _v, _n, &keyname, &key_found) != 0) {
622
+    if (db_redis_find_query_key(key, table_name, table, &typename, _k, _v, _op, _n, &keyname, &key_found, NULL) != 0) {
540 623
         goto err;
541 624
     }
542 625
     if (key_found) {
... ...
@@ -550,11 +633,15 @@ static int db_redis_build_query_keys(km_redis_con_t *con, const str *table_name,
550 633
         pkg_free(keyname.s);
551 634
         keyname.s = NULL;
552 635
     } else {
636
+        if (keyname.s)
637
+            pkg_free(keyname.s);
638
+        keyname.s = NULL;
553 639
         LM_DBG("no direct entry key found, checking type keys\n");
554 640
         for (type = table->types; type; type = type->next) {
555 641
             key = type->keys;
556 642
             LM_DBG("checking type '%.*s'\n", type->type.len, type->type.s);
557
-            if (db_redis_find_query_key(key, table_name, &type->type, _k, _v, _n, &keyname, &key_found) != 0) {
643
+            if (db_redis_find_query_key(key, table_name, table, &type->type, _k, _v, _op, _n, &keyname,
644
+                        &key_found, ts_scan_start) != 0) {
558 645
                 goto err;
559 646
             }
560 647
             if (key_found) {
... ...
@@ -592,7 +679,7 @@ static int db_redis_build_query_keys(km_redis_con_t *con, const str *table_name,
592 679
                             redisReply *subreply = reply->element[i];
593 680
                             if (subreply->type == REDIS_REPLY_STRING) {
594 681
                                 LM_DBG("adding resulting entry key '%s' from type query\n", subreply->str);
595
-                                if (db_redis_key_add_string(query_keys, subreply->str, strlen(subreply->str)) != 0) {
682
+                                if (db_redis_key_prepend_string(query_keys, subreply->str, strlen(subreply->str)) != 0) {
596 683
                                     LM_ERR("Failed to add query key\n");
597 684
                                     goto err;
598 685
                                 }
... ...
@@ -610,6 +697,16 @@ static int db_redis_build_query_keys(km_redis_con_t *con, const str *table_name,
610 697
                 db_redis_free_reply(&reply);
611 698
                 break;
612 699
             }
700
+            else if (keyname.s && *ts_scan_start) {
701
+                LM_DBG("will use key '%.*s' at offset %llx for timestamp/int range scan\n",
702
+                        keyname.len, keyname.s, (unsigned long long) *ts_scan_start);
703
+                *ts_scan_key = keyname;
704
+                keyname.s = NULL;
705
+            }
706
+            else if (keyname.s) {
707
+                pkg_free(keyname.s);
708
+                keyname.s = NULL;
709
+            }
613 710
         }
614 711
     }
615 712
 
... ...
@@ -639,36 +736,30 @@ err:
639 736
     return -1;
640 737
 }
641 738
 
642
-static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
643
-        const db_key_t *_k, const int _n,
739
+static int db_redis_scan_query_keys_pattern(km_redis_con_t *con, const str *match_pattern,
740
+        const int _n,
644 741
         redis_key_t **query_keys, int *query_keys_count,
645
-        int **manual_keys, int *manual_keys_count) {
742
+        int **manual_keys, int *manual_keys_count, unsigned int match_count_start_val) {
646 743
 
647 744
     size_t i = 0;
648 745
     redis_key_t *query_v = NULL;
649
-    char cursor_str[32] = "";
650 746
     redisReply *reply = NULL;
651
-    unsigned long cursor = 0;
652
-    char *match = NULL;
747
+    redisReply *keys_list = NULL;
653 748
     size_t j;
654 749
     int l;
655 750
 
656
-    str match_pattern = {":entry::*", strlen(":entry::*")};
657 751
 
658
-    *query_keys = NULL;
659
-    *query_keys_count = 0;
660
-    *manual_keys = NULL;
661
-    *manual_keys_count = 0;
752
+#undef USE_SCAN
753
+
754
+#ifdef USE_SCAN
755
+
756
+    char cursor_str[32] = "";
757
+    unsigned long cursor = 0;
758
+    unsigned int match_count = match_count_start_val;
759
+    char match_count_str[16];
662 760
 
663 761
     do {
664 762
         snprintf(cursor_str, sizeof(cursor_str), "%lu", cursor);
665
-        match = (char*)pkg_malloc(table_name->len + match_pattern.len + 1);
666
-        if (!match) {
667
-            LM_ERR("Failed to allocate memory for match pattern\n");
668
-            goto err;
669
-        }
670
-        snprintf(match, table_name->len + match_pattern.len + 1, "%s%s\n",
671
-                table_name->s, match_pattern.s);
672 763
 
673 764
         if (db_redis_key_add_string(&query_v, "SCAN", 4) != 0) {
674 765
             LM_ERR("Failed to add scan command to scan query\n");
... ...
@@ -682,7 +773,7 @@ static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
682 773
             LM_ERR("Failed to add match command to scan query\n");
683 774
             goto err;
684 775
         }
685
-        if (db_redis_key_add_string(&query_v, match, strlen(match)) != 0) {
776
+        if (db_redis_key_add_string(&query_v, match_pattern->s, match_pattern->len) != 0) {
686 777
             LM_ERR("Failed to add match pattern to scan query\n");
687 778
             goto err;
688 779
         }
... ...
@@ -690,23 +781,27 @@ static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
690 781
             LM_ERR("Failed to add count command to scan query\n");
691 782
             goto err;
692 783
         }
693
-        if (db_redis_key_add_string(&query_v, "1000", 5) != 0) {
784
+        l = snprintf(match_count_str, sizeof(match_count_str), "%u", match_count);
785
+        if (l <= 0) {
786
+            LM_ERR("Failed to print integer for scan query\n");
787
+            goto err;
788
+        }
789
+        if (db_redis_key_add_string(&query_v, match_count_str, l) != 0) {
694 790
             LM_ERR("Failed to add count value to scan query\n");
695 791
             goto err;
696 792
         }
697
-        pkg_free(match); match = NULL;
698 793
 
699 794
         reply = db_redis_command_argv(con, query_v);
700 795
         db_redis_key_free(&query_v);
701 796
         db_redis_check_reply(con, reply, err);
702 797
         if (reply->type != REDIS_REPLY_ARRAY) {
703 798
             LM_ERR("Invalid reply type for scan on table '%.*s', expected array\n",
704
-                    table_name->len, table_name->s);
799
+                    match_pattern->len, match_pattern->s);
705 800
             goto err;
706 801
         }
707 802
         if (reply->elements != 2) {
708 803
             LM_ERR("Invalid number of reply elements for scan on table '%.*s', expected 2, got %lu\n",
709
-                    table_name->len, table_name->s, reply->elements);
804
+                    match_pattern->len, match_pattern->s, reply->elements);
710 805
             goto err;
711 806
         }
712 807
 
... ...
@@ -717,34 +812,50 @@ static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
717 812
             cursor = reply->element[0]->integer;
718 813
         } else {
719 814
             LM_ERR("Invalid cursor type for scan on table '%.*s', expected string or integer\n",
720
-                    table_name->len, table_name->s);
815
+                    match_pattern->len, match_pattern->s);
721 816
             goto err;
722 817
         }
723 818
         LM_DBG("cursor is %lu\n", cursor);
724 819
 
725
-        if (reply->element[1]->type != REDIS_REPLY_ARRAY) {
820
+        keys_list = reply->element[1];
821
+
822
+#else // use KEYS
823
+
824
+    if (db_redis_key_add_string(&query_v, "KEYS", 4) != 0) {
825
+        LM_ERR("Failed to add scan command to scan query\n");
826
+        goto err;
827
+    }
828
+    if (db_redis_key_add_string(&query_v, match_pattern->s, match_pattern->len) != 0) {
829
+        LM_ERR("Failed to add match pattern to scan query\n");
830
+        goto err;
831
+    }
832
+
833
+    reply = db_redis_command_argv(con, query_v);
834
+    db_redis_key_free(&query_v);
835
+    db_redis_check_reply(con, reply, err);
836
+
837
+    keys_list = reply;
838
+
839
+#endif
840
+
841
+        if (keys_list->type != REDIS_REPLY_ARRAY) {
726 842
             LM_ERR("Invalid content type for scan on table '%.*s', expected array\n",
727
-                    table_name->len, table_name->s);
843
+                    match_pattern->len, match_pattern->s);
728 844
             goto err;
729 845
         }
730
-        if (reply->element[1]->elements == 0) {
731
-            LM_DBG("no matching entries found for scan on table '%.*s'\n",
732
-                    table_name->len, table_name->s);
733
-            return 0;
734
-        }
735 846
 
736
-        *query_keys_count += reply->element[1]->elements;
847
+        *query_keys_count += keys_list->elements;
737 848
 
738
-        for (j = 0; j < reply->element[1]->elements; ++i, ++j) {
739
-            redisReply *key = reply->element[1]->element[j];
849
+        for (j = 0; j < keys_list->elements; ++i, ++j) {
850
+            redisReply *key = keys_list->element[j];
740 851
             if (!key) {
741 852
                 LM_ERR("Invalid null key at cursor result index %lu while scanning table '%.*s'\n",
742
-                        j, table_name->len, table_name->s);
853
+                        j, match_pattern->len, match_pattern->s);
743 854
                 goto err;
744 855
             }
745 856
             if (key->type != REDIS_REPLY_STRING) {
746 857
                 LM_ERR("Invalid key type at cursor result index %lu while scanning table '%.*s', expected string\n",
747
-                        j, table_name->len, table_name->s);
858
+                        j, match_pattern->len, match_pattern->s);
748 859
                 goto err;
749 860
             }
750 861
             if (db_redis_key_prepend_string(query_keys, key->str, strlen(key->str)) != 0) {
... ...
@@ -752,19 +863,34 @@ static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
752 863
                 goto err;
753 864
             }
754 865
         }
866
+
867
+#ifdef USE_SCAN
868
+        // exponential increase and falloff, hovering around 1000 results
869
+        if (keys_list->elements > 1300 && match_count > 500)
870
+            match_count /= 2;
871
+        else if (keys_list->elements < 700 && match_count < 500000)
872
+            match_count *= 2;
873
+#endif
874
+
755 875
         db_redis_free_reply(&reply);
876
+
877
+#ifdef USE_SCAN
756 878
     } while (cursor > 0);
879
+#endif
757 880
 
758 881
     // for full table scans, we have to manually match all given keys
759
-    *manual_keys_count = _n;
760
-    *manual_keys = (int*)pkg_malloc(*manual_keys_count * sizeof(int));
761
-    if (! *manual_keys) {
762
-        LM_ERR("Failed to allocate memory for manual keys\n");
763
-        goto err;
764
-    }
765
-    memset(*manual_keys, 0, *manual_keys_count * sizeof(int));
766
-    for (l = 0; l < _n; ++l) {
767
-        (*manual_keys)[l] = l;
882
+    // but only do this once for repeated invocations
883
+    if (!*manual_keys) {
884
+        *manual_keys_count = _n;
885
+        *manual_keys = (int*)pkg_malloc(*manual_keys_count * sizeof(int));
886
+        if (! *manual_keys) {
887
+            LM_ERR("Failed to allocate memory for manual keys\n");
888
+            goto err;
889
+        }
890
+        memset(*manual_keys, 0, *manual_keys_count * sizeof(int));
891
+        for (l = 0; l < _n; ++l) {
892
+            (*manual_keys)[l] = l;
893
+        }
768 894
     }
769 895
 
770 896
     if (reply) {
... ...
@@ -775,8 +901,6 @@ static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
775 901
     return 0;
776 902
 
777 903
 err:
778
-    if (match)
779
-        pkg_free(match);
780 904
     if (reply)
781 905
         db_redis_free_reply(&reply);
782 906
     db_redis_key_free(&query_v);
... ...
@@ -789,6 +913,235 @@ err:
789 913
     return -1;
790 914
 }
791 915
 
916
+static int db_redis_scan_query_keys(km_redis_con_t *con, const str *table_name,
917
+        const int _n,
918
+        redis_key_t **query_keys, int *query_keys_count,
919
+        int **manual_keys, int *manual_keys_count, uint64_t ts_scan_start, const str *ts_scan_key) {
920
+
921
+    struct str_hash_entry *table_e;
922
+    redis_table_t *table;
923
+    char *match = NULL;
924
+    int ret;
925
+    redisReply *reply = NULL;
926
+
927
+    *query_keys = NULL;
928
+    *query_keys_count = 0;
929
+    *manual_keys = NULL;
930
+    *manual_keys_count = 0;
931
+    redis_key_t *set_keys = NULL;
932
+    int set_keys_count = 0;
933
+
934
+    table_e = str_hash_get(&con->tables, table_name->s, table_name->len);
935
+    if (!table_e) {
936
+        LM_ERR("query to undefined table '%.*s', define it in schema file!\n",
937
+                table_name->len, table_name->s);
938
+        return -1;
939
+    }
940
+    table = (redis_table_t*)table_e->u.p;
941
+
942
+    if (!ts_scan_start) {
943
+        // full table scan
944
+        match = (char*)pkg_malloc(table->version_code.len
945
+                + table_name->len + 10); // length of ':entry::*' plus \0
946
+        if (!match) {
947
+            LM_ERR("Failed to allocate memory for match pattern\n");
948
+            return -1;
949
+        }
950
+        int len = sprintf(match, "%.*s%.*s:entry::*",
951
+                table->version_code.len, table->version_code.s,
952
+                table_name->len, table_name->s);
953
+        str match_pattern = {match, len};
954
+        ret = db_redis_scan_query_keys_pattern(con, &match_pattern, _n, query_keys, query_keys_count,
955
+                manual_keys, manual_keys_count, 1000);
956
+        pkg_free(match);
957
+        return ret;
958
+    }
959
+
960
+    // timestamp range scan
961
+    // ex: 2019-07-17 17:33:16
962
+    // if >, we match: [3-9]???-??-?? ??:??:??, 2[1-9]??-??-?? ??:??:??, 20[2-9]?-??-?? ??:??:??, etc
963
+    // if <, we match: [0-1]???-??-?? ??:??:??, 200..., 201[0-8]..., etc
964
+    // the maximum match string length is ts_scan_key->len with one character replaced by 5 ('[a-b]')
965
+    //
966
+    // int range scan
967
+    // ex: 12345
968
+    // if >, we match: 2????, 1[3-9]???, ..., plus ?????*
969
+    // if <. we match: ?, ??, ???, ????, 1[0-1]???, 12[0-2]??, etc
970
+    //    ... however we expect a minimum length of 10 digits as per BIGINT printf format
971
+
972
+    match = pkg_malloc(ts_scan_key->len + 6);
973
+    if (!match) {
974
+        LM_ERR("Failed to allocate memory for match pattern\n");
975
+        return -1;
976
+    }
977
+
978
+    int scan_lt = (ts_scan_start & 0x8000000000000000ULL) ? 1 : 0;
979
+    int scan_len_variable = (ts_scan_start & 0x4000000000000000ULL) ? 1 : 0;
980
+    unsigned int scan_offset = ts_scan_start & 0x7fffffffULL;
981
+    unsigned int scan_length = (ts_scan_start >> 31) & 0x7fffffffULL;
982
+    scan_offset -= scan_length;
983
+    const char *suffix = ts_scan_key->s + scan_offset + scan_length;
984
+
985
+    LM_DBG("running timestamp/int range matching: lt %i, lv %i, off %u, len %u\n",
986
+            scan_lt, scan_len_variable, scan_offset, scan_length);
987
+
988
+    if (scan_lt && scan_len_variable) {
989
+        // match shorter strings
990
+
991
+        // copy unchanged prefix
992
+        memcpy(match, ts_scan_key->s, scan_offset);
993
+
994
+        // append a number of ?. minimum string length is 10 digits
995
+        for (int i = 0; i < scan_length - 1; i++) {
996
+            int len = scan_offset + i;
997
+            char match_char = ts_scan_key->s[len];
998
+            // skip non-numbers
999
+            if (match_char < '0' || match_char > '9') {
1000
+                match[len] = match_char;
1001
+                continue;
1002
+            }
1003
+            // append a single ?
1004
+            match[len] = '?';
1005
+            // append unchanged suffix
1006
+            strcpy(match + len + 1, suffix);
1007
+            len = strlen(match);
1008
+
1009
+            // minimum bigint printf string length
1010
+            if (i < 10)
1011
+                continue;
1012
+
1013
+            str match_pattern = {match, len};
1014
+            LM_DBG("running timestamp/int range matching using pattern '%.*s'\n", len, match);
1015
+
1016
+            ret = db_redis_scan_query_keys_pattern(con, &match_pattern, _n, &set_keys, &set_keys_count,
1017
+                    manual_keys, manual_keys_count, 5000);
1018
+            if (ret)
1019
+                goto out;
1020
+        }
1021
+    }
1022
+
1023
+    for (int i = 0; i < scan_length; i++) {
1024
+        int len = scan_offset + i;
1025
+        char match_char = ts_scan_key->s[len];
1026
+        // skip non-numbers
1027
+        if (match_char < '0' || match_char > '9')
1028
+            continue;
1029
+        // skip numbers that are at the edge of their match range
1030
+        if (match_char == '0' && scan_lt)
1031
+            continue;
1032
+        if (match_char == '1' && scan_lt && i == 0) // no leading 0
1033
+            continue;
1034
+        if (match_char == '9' && !scan_lt)
1035
+            continue;
1036
+
1037
+        // copy unchanged prefix
1038
+        memcpy(match, ts_scan_key->s, len);
1039
+        // append range matcher
1040
+        if (scan_lt)
1041
+            len += sprintf(match + len, "[0-%c]", match_char - 1);
1042
+        else
1043
+            len += sprintf(match + len, "[%c-9]", match_char + 1);
1044
+        // finish with trailing ?s
1045
+	for (int j = i + 1; j < scan_length; j++) {
1046
+            match_char = ts_scan_key->s[scan_offset + j];
1047
+            // skip non-numbers
1048
+            if (match_char < '0' || match_char > '9') {
1049
+                match[len++] = match_char;
1050
+                continue;
1051
+            }
1052
+            match[len++] = '?';
1053
+        }
1054
+        // append unchanged suffix
1055
+        strcpy(match + len, suffix);
1056
+        len = strlen(match);
1057
+
1058
+        str match_pattern = {match, len};
1059
+        LM_DBG("running timestamp/int range matching using pattern '%.*s'\n", len, match);
1060
+
1061
+        ret = db_redis_scan_query_keys_pattern(con, &match_pattern, _n, &set_keys, &set_keys_count,
1062
+                manual_keys, manual_keys_count, 5000);
1063
+        if (ret)
1064
+            goto out;
1065
+    }
1066
+
1067
+    if (!scan_lt && scan_len_variable) {
1068
+        // match longer strings
1069
+        int len = sprintf(match, "%.*s*%s", scan_offset + scan_length, ts_scan_key->s, suffix);
1070
+
1071
+        str match_pattern = {match, len};
1072
+        LM_DBG("running timestamp/int range matching using pattern '%.*s'\n", len, match);
1073
+
1074
+        ret = db_redis_scan_query_keys_pattern(con, &match_pattern, _n, &set_keys, &set_keys_count,
1075
+                manual_keys, manual_keys_count, 5000);
1076
+        if (ret)
1077
+            goto out;
1078
+    }
1079
+
1080
+    // we not have a list of matching type keys in set_keys. now we have to iterate through them
1081
+    // and retrieve the set members, and finally build our actual key list
1082
+
1083
+    ret = -1;
1084
+
1085
+    for (redis_key_t *set_key = set_keys; set_key; set_key = set_key->next) {
1086
+        LM_DBG("pulling set members from key '%.*s'\n", set_key->key.len, set_key->key.s);
1087
+
1088
+        redis_key_t *query_v = NULL;
1089
+        if (db_redis_key_add_string(&query_v, "SMEMBERS", 8) != 0) {
1090
+            LM_ERR("Failed to add smembers command to query\n");
1091
+            db_redis_key_free(&query_v);
1092
+            goto out;
1093
+        }
1094
+        if (db_redis_key_add_str(&query_v, &set_key->key) != 0) {
1095
+            LM_ERR("Failed to add key name to smembers query\n");
1096
+            db_redis_key_free(&query_v);
1097
+            goto out;
1098
+        }
1099
+
1100
+        reply = db_redis_command_argv(con, query_v);
1101
+        db_redis_key_free(&query_v);
1102
+        db_redis_check_reply(con, reply, out);
1103
+
1104
+        if (reply->type != REDIS_REPLY_ARRAY) {
1105
+            LM_ERR("Unexpected reply for type query, expecting an array\n");
1106
+            goto out;
1107
+        }
1108
+
1109
+        LM_DBG("adding %i keys returned from set", (int) reply->elements);
1110
+
1111
+        for (int i = 0; i < reply->elements; i++) {
1112
+            if (reply->element[i]->type != REDIS_REPLY_STRING) {
1113
+                LM_ERR("Unexpected entry key type in type query, expecting a string\n");
1114
+                goto out;
1115
+            }
1116
+            if (db_redis_key_prepend_string(query_keys, reply->element[i]->str, strlen(reply->element[i]->str))
1117
+                    != 0) {
1118
+                LM_ERR("Failed to prepend redis key\n");
1119
+                goto out;
1120
+            }
1121
+            LM_DBG("adding key '%s'\n", reply->element[i]->str);
1122
+        }
1123
+        *query_keys_count += reply->elements;
1124
+
1125
+        db_redis_free_reply(&reply);
1126
+    }
1127
+
1128
+    ret = 0;
1129
+
1130
+out:
1131
+    pkg_free(match);
1132
+    db_redis_key_free(&set_keys);
1133
+    db_redis_free_reply(&reply);
1134
+    if (ret) {
1135
+        db_redis_key_free(query_keys);
1136
+        *query_keys_count = 0;
1137
+        if (*manual_keys) {
1138
+            pkg_free(*manual_keys);
1139
+            *manual_keys = NULL;
1140
+        }
1141
+    }
1142
+    return ret;
1143
+}
1144
+
792 1145
 static int db_redis_compare_column(db_key_t k, db_val_t *v, db_op_t op, redisReply *reply) {
793 1146
     int i_value;
794 1147
     long long ll_value;
... ...
@@ -1073,7 +1426,8 @@ static int db_redis_perform_query(const db1_con_t* _h, km_redis_con_t *con, cons
1073 1426
         const db_val_t* _v, const db_op_t *_op, const db_key_t* _c,
1074 1427
         const int _n, const int _nc, db1_res_t** _r,
1075 1428
         redis_key_t **keys, int *keys_count,
1076
-        int **manual_keys, int *manual_keys_count, int do_table_scan) {
1429
+        int **manual_keys, int *manual_keys_count, int do_table_scan, uint64_t ts_scan_start,
1430
+        const str *ts_scan_key) {
1077 1431
 
1078 1432
     redisReply *reply = NULL;
1079 1433
     redis_key_t *query_v = NULL;
... ...
@@ -1101,9 +1455,9 @@ static int db_redis_perform_query(const db1_con_t* _h, km_redis_con_t *con, cons
1101 1455
             LM_WARN("  scan key %d is '%.*s'\n",
1102 1456
                     i, _k[i]->len, _k[i]->s);
1103 1457
         }
1104
-        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _k, _n,
1458
+        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _n,
1105 1459
                     keys, keys_count,
1106
-                    manual_keys, manual_keys_count) != 0) {
1460
+                    manual_keys, manual_keys_count, ts_scan_start, ts_scan_key) != 0) {
1107 1461
             LM_ERR("failed to scan query keys\n");
1108 1462
             goto error;
1109 1463
         }
... ...
@@ -1253,7 +1607,8 @@ error:
1253 1607
 static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, const db_key_t* _k,
1254 1608
         const db_val_t* _v, const db_op_t *_op, const int _n,
1255 1609
         redis_key_t **keys, int *keys_count,
1256
-        int **manual_keys, int *manual_keys_count, int do_table_scan) {
1610
+        int **manual_keys, int *manual_keys_count, int do_table_scan, uint64_t ts_scan_start,
1611
+        const str *ts_scan_key) {
1257 1612
 
1258 1613
     int i = 0, j = 0;
1259 1614
     redis_key_t *k = NULL;
... ...
@@ -1264,21 +1619,29 @@ static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, con
1264 1619
     redisReply *reply = NULL;
1265 1620
     redis_key_t *query_v = NULL;
1266 1621
     redis_key_t *type_keys = NULL;
1622
+    redis_key_t *set_keys = NULL;
1267 1623
     redis_key_t *all_type_keys = NULL;
1268 1624
     db_val_t *db_vals = NULL;
1269 1625
     db_key_t *db_keys = NULL;
1270 1626
     redis_key_t *type_key;
1627
+    redis_key_t *set_key;
1271 1628
 
1272 1629
     if (!*keys_count && do_table_scan) {
1273
-        LM_WARN("performing full table scan on table '%.*s' while performing delete\n",
1274
-                CON_TABLE(_h)->len, CON_TABLE(_h)->s);
1630
+        if (!ts_scan_start)
1631
+            LM_WARN("performing full table scan on table '%.*s' while performing delete\n",
1632
+                    CON_TABLE(_h)->len, CON_TABLE(_h)->s);
1633
+        else
1634
+            LM_WARN("performing table scan on table '%.*s' while performing delete using match key "
1635
+                    "'%.*s' at offset %llx\n",
1636
+                    CON_TABLE(_h)->len, CON_TABLE(_h)->s,
1637
+                    ts_scan_key->len, ts_scan_key->s, (unsigned long long) ts_scan_start);
1275 1638
         for(i = 0; i < _n; ++i) {
1276 1639
             LM_WARN("  scan key %d is '%.*s'\n",
1277 1640
                     i, _k[i]->len, _k[i]->s);
1278 1641
         }
1279
-        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _k, _n,
1642
+        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _n,
1280 1643
                     keys, keys_count,
1281
-                    manual_keys, manual_keys_count) != 0) {
1644
+                    manual_keys, manual_keys_count, ts_scan_start, ts_scan_key) != 0) {
1282 1645
             LM_ERR("failed to scan query keys\n");
1283 1646
             goto error;
1284 1647
         }
... ...
@@ -1412,7 +1775,7 @@ static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, con
1412 1775
             }
1413 1776
         }
1414 1777
         if (db_redis_build_type_keys(con, CON_TABLE(_h), db_keys, db_vals, all_type_keys_count,
1415
-                    &type_keys, &type_keys_count) != 0) {
1778
+                    &type_keys, &set_keys, &type_keys_count) != 0) {
1416 1779
             LM_ERR("failed to build type keys\n");
1417 1780
             goto error;
1418 1781
         }
... ...
@@ -1437,8 +1800,18 @@ static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, con
1437 1800
         db_redis_check_reply(con, reply, error);
1438 1801
         db_redis_free_reply(&reply);
1439 1802
 
1440
-        for (type_key = type_keys; type_key; type_key = type_key->next) {
1441
-            if (db_redis_key_add_string(&query_v, "SREM", 4) != 0) {
1803
+        for (type_key = type_keys, set_key = set_keys; type_key;
1804
+                type_key = type_key->next, set_key = set_key->next) {
1805
+
1806
+            if (db_redis_key_add_string(&query_v, "EVALSHA", 7) != 0) {
1807
+                LM_ERR("Failed to add srem command to post-delete query\n");
1808
+                goto error;
1809
+            }
1810
+            if (db_redis_key_add_string(&query_v, con->srem_key_lua, strlen(con->srem_key_lua)) != 0) {
1811
+                LM_ERR("Failed to add srem command to post-delete query\n");
1812
+                goto error;
1813
+            }
1814
+            if (db_redis_key_add_string(&query_v, "3", 1) != 0) {
1442 1815
                 LM_ERR("Failed to add srem command to post-delete query\n");
1443 1816
                 goto error;
1444 1817
             }
... ...
@@ -1446,6 +1819,10 @@ static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, con
1446 1819
                 LM_ERR("Failed to add key to delete query\n");
1447 1820
                 goto error;
1448 1821
             }
1822
+            if (db_redis_key_add_str(&query_v, &set_key->key) != 0) {
1823
+                LM_ERR("Failed to add key to delete query\n");
1824
+                goto error;
1825
+            }
1449 1826
             if (db_redis_key_add_str(&query_v, key) != 0) {
1450 1827
                 LM_ERR("Failed to add key to delete query\n");
1451 1828
                 goto error;
... ...
@@ -1457,6 +1834,7 @@ static int db_redis_perform_delete(const db1_con_t* _h, km_redis_con_t *con, con
1457 1834
         }
1458 1835
         LM_DBG("done with loop '%.*s'\n", k->key.len, k->key.s);
1459 1836
         db_redis_key_free(&type_keys);
1837
+        db_redis_key_free(&set_keys);
1460 1838
     }
1461 1839
     db_redis_key_free(&all_type_keys);
1462 1840
     db_redis_key_free(&query_v);
... ...
@@ -1473,6 +1851,7 @@ error:
1473 1851
         pkg_free(db_vals);
1474 1852
     db_redis_key_free(&query_v);
1475 1853
     db_redis_key_free(&type_keys);
1854
+    db_redis_key_free(&set_keys);
1476 1855
     db_redis_key_free(&all_type_keys);
1477 1856
     return -1;
1478 1857
 }
... ...
@@ -1481,7 +1860,8 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1481 1860
         const db_val_t* _v, const db_op_t *_op, const db_key_t* _uk, const db_val_t *_uv,
1482 1861
         const int _n, const int _nu,
1483 1862
         redis_key_t **keys, int *keys_count,
1484
-        int **manual_keys, int *manual_keys_count, int do_table_scan) {
1863
+        int **manual_keys, int *manual_keys_count, int do_table_scan, uint64_t ts_scan_start,
1864
+        const str *ts_scan_key) {
1485 1865
 
1486 1866
     redisReply *reply = NULL;
1487 1867
     redis_key_t *query_v = NULL;
... ...
@@ -1490,6 +1870,16 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1490 1870
     int i;
1491 1871
     int j;
1492 1872
     size_t col;
1873
+    redis_key_t *all_type_keys = NULL;
1874
+    int all_type_keys_count = 0;
1875
+    db_val_t *db_vals = NULL;
1876
+    db_key_t *db_keys = NULL;
1877
+    redis_key_t *type_keys = NULL;
1878
+    redis_key_t *set_keys = NULL;
1879
+    int type_keys_count = 0;
1880
+    redis_key_t *new_type_keys = NULL;
1881
+    int new_type_keys_count = 0;
1882
+    redis_key_t *all_type_key;
1493 1883
 
1494 1884
     if (!(*keys_count) && do_table_scan) {
1495 1885
         LM_WARN("performing full table scan on table '%.*s' while performing update\n",
... ...
@@ -1498,18 +1888,35 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1498 1888
             LM_WARN("  scan key %d is '%.*s'\n",
1499 1889
                     i, _k[i]->len, _k[i]->s);
1500 1890
         }
1501
-        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _k, _n,
1891
+        if (db_redis_scan_query_keys(con, CON_TABLE(_h), _n,
1502 1892
                     keys, keys_count,
1503
-                    manual_keys, manual_keys_count) != 0) {
1893
+                    manual_keys, manual_keys_count, ts_scan_start, ts_scan_key) != 0) {
1504 1894
             LM_ERR("failed to scan query keys\n");
1505 1895
             goto error;
1506 1896
         }
1507 1897
     }
1508 1898
 
1899
+    // TODO: this should be moved to redis_connection structure
1900
+    // and be parsed at startup:
1901
+    //
1902
+    // fetch list of keys in all types
1903
+    if (db_redis_get_keys_for_all_types(con, CON_TABLE(_h),
1904
+                &all_type_keys, &all_type_keys_count) != 0) {
1905
+            LM_ERR("failed to get full list of type keys\n");
1906
+            goto error;
1907
+    }
1908
+
1909
+    if (db_redis_build_type_keys(con, CON_TABLE(_h), _uk, _uv, _nu,
1910
+                &new_type_keys, NULL, &new_type_keys_count) != 0) {
1911
+        LM_ERR("failed to build type keys\n");
1912
+        goto error;
1913
+    }
1914
+    LM_DBG("%i new type keys\n", new_type_keys_count);
1915
+
1509 1916
     for (key = *keys; key; key = key->next) {
1510 1917
         str *keyname = &key->key;
1511 1918
 
1512
-        LM_DBG("fetching row for '%.*s' from redis\n", keyname->len, keyname->s);
1919
+        LM_DBG("fetching row for '%.*s' from redis for update\n", keyname->len, keyname->s);
1513 1920
 
1514 1921
 
1515 1922
         if (db_redis_key_add_string(&query_v, "EXISTS", 6) != 0) {
... ...
@@ -1527,19 +1934,9 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1527 1934
         db_redis_key_free(&query_v);
1528 1935
 
1529 1936
         // construct HMGET query
1530
-        if ((*manual_keys_count) == 0) {
1531
-            if (db_redis_key_add_string(&query_v, "HGETALL", 7) != 0) {
1532
-                LM_ERR("Failed to set hgetall command to pre-update hget query\n");
1533
-                goto error;
1534
-            }
1535
-            // TODO: actually we wouldn't have to fetch it at all, but then we'd
1536
-            // have to mark this key telling to not fetch reply of HMGET after
1537
-            // EXISTS returns false!
1538
-        } else {
1539
-            if (db_redis_key_add_string(&query_v, "HMGET", 5) != 0) {
1540
-                LM_ERR("Failed to set hgetall command to pre-update hget query\n");
1541
-                goto error;
1542
-            }
1937
+        if (db_redis_key_add_string(&query_v, "HMGET", 5) != 0) {
1938
+            LM_ERR("Failed to set hgetall command to pre-update hget query\n");
1939
+            goto error;
1543 1940
         }
1544 1941
         if (db_redis_key_add_str(&query_v, keyname) != 0) {
1545 1942
             LM_ERR("Failed to add key name to pre-update exists query\n");
... ...
@@ -1554,6 +1951,13 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1554 1951
                 goto error;
1555 1952
             }
1556 1953
         }
1954
+        // add all type keys to query
1955
+        for (all_type_key = all_type_keys; all_type_key; all_type_key = all_type_key->next) {
1956
+            if (db_redis_key_add_str(&query_v, &all_type_key->key) != 0) {
1957
+                LM_ERR("Failed to add type key to pre-update query\n");
1958
+                goto error;
1959
+            }
1960
+        }
1557 1961
 
1558 1962
         if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) {
1559 1963
             LM_ERR("Failed to append redis command\n");
... ...
@@ -1580,9 +1984,13 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1580 1984
 
1581 1985
 
1582 1986
     for (key = *keys; key; key = key->next) {
1987
+        redis_key_t *tmp = NULL;
1988
+        redis_key_t *type_key;
1989
+        redis_key_t *set_key;
1990
+        redis_key_t *new_type_key;
1583 1991
         int row_match;
1584 1992
 
1585
-        LM_DBG("fetching replies for '%.*s' from redis\n", key->key.len, key->key.s);
1993
+        LM_DBG("fetching replies for '%.*s' from redis for update\n", key->key.len, key->key.s);
1586 1994
 
1587 1995
         // get reply for EXISTS query
1588 1996
         if (db_redis_get_reply(con, (void**)&reply) != REDIS_OK) {
... ...
@@ -1639,13 +2047,50 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1639 2047
                 }
1640 2048
             }
1641 2049
         }
1642
-        db_redis_free_reply(&reply);
1643 2050
         if (!row_match) {
1644 2051
             continue;
1645 2052
         } else {
1646 2053
             LM_DBG("row matches manual filtering, proceed with update\n");
1647 2054
         }
1648 2055
 
2056
+        db_keys = (db_key_t*) pkg_malloc(all_type_keys_count * sizeof(db_key_t));
2057
+        if (!db_keys) {
2058
+            LM_ERR("Failed to allocate memory for db type keys\n");
2059
+            goto error;
2060
+        }
2061
+        for (j = 0, tmp = all_type_keys; tmp; ++j, tmp = tmp->next) {
2062
+            db_keys[j] = &tmp->key;
2063
+        }
2064
+
2065
+        db_vals = (db_val_t*) pkg_malloc(all_type_keys_count * sizeof(db_val_t));
2066
+        if (!db_vals) {
2067
+            LM_ERR("Failed to allocate memory for manual db vals\n");
2068
+            goto error;
2069
+        }
2070
+
2071
+        for (j = 0, all_type_key = all_type_keys; all_type_key; ++j, all_type_key = all_type_key->next) {
2072
+            db_val_t *v = &(db_vals[j]);
2073
+            str *key = &all_type_key->key;
2074
+            char *value = reply->element[*manual_keys_count + j]->str;
2075
+            int coltype = db_redis_schema_get_column_type(con, CON_TABLE(_h), key);
2076
+            if (value == NULL) {
2077
+                VAL_NULL(v) = 1;
2078
+            } else if (db_str2val(coltype, v, value, strlen(value), 0) != 0) {
2079
+                LM_ERR("Failed to convert redis reply column to db value\n");
2080
+                goto error;
2081
+            }
2082
+        }
2083
+        if (db_redis_build_type_keys(con, CON_TABLE(_h), db_keys, db_vals, all_type_keys_count,
2084
+                    &type_keys, &set_keys, &type_keys_count) != 0) {
2085
+            LM_ERR("failed to build type keys\n");
2086
+            goto error;
2087
+        }
2088
+        pkg_free(db_keys);
2089
+        db_keys = NULL;
2090
+        pkg_free(db_vals);
2091
+        db_vals = NULL;
2092
+        db_redis_free_reply(&reply);
2093
+
1649 2094
         if (db_redis_key_add_string(&query_v, "HMSET", 5) != 0) {
1650 2095
             LM_ERR("Failed to add hmset command to update query\n");
1651 2096
             goto error;
... ...
@@ -1681,6 +2126,108 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1681 2126
         }
1682 2127
 
1683 2128
         db_redis_key_free(&query_v);
2129
+
2130
+        for (type_key = type_keys, set_key = set_keys; type_key;
2131
+                type_key = type_key->next, set_key = set_key->next) {
2132
+
2133
+            LM_DBG("checking for update of type key '%.*s'\n",
2134
+                    type_key->key.len, type_key->key.s);
2135
+            char *prefix = ser_memmem(type_key->key.s, "::", type_key->key.len, 2);
2136
+            if (!prefix || prefix == type_key->key.s) {
2137
+                LM_DBG("Invalid key without :: '%.*s'\n",
2138
+                        type_key->key.len, type_key->key.s);
2139
+                continue;
2140
+            }
2141
+            for (new_type_key = new_type_keys; new_type_key; new_type_key = new_type_key->next) {
2142
+                // compare prefix to see if this is the same key
2143
+                if (memcmp(new_type_key->key.s, type_key->key.s, prefix - type_key->key.s))
2144
+                    continue;
2145
+                LM_DBG("checking for update of type key against '%.*s'\n",
2146
+                        new_type_key->key.len, new_type_key->key.s);
2147
+                if (!str_strcmp(&new_type_key->key, &type_key->key))
2148
+                    continue;
2149
+
2150
+                // add to new set key and delete from old
2151
+
2152
+                if (db_redis_key_add_string(&query_v, "SADD", 4) != 0) {
2153
+                    LM_ERR("Failed to set sadd command to post-update query\n");
2154
+                    goto error;
2155
+                }
2156
+                if (db_redis_key_add_str(&query_v, &new_type_key->key) != 0) {
2157
+                    LM_ERR("Failed to add map key to post-update query\n");
2158
+                    goto error;
2159
+                }
2160
+                if (db_redis_key_add_str(&query_v, &key->key) != 0) {
2161
+                    LM_ERR("Failed to set entry key to post-update query\n");
2162
+                    goto error;
2163
+                }
2164
+
2165
+                update_queries++;
2166
+                if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) {
2167
+                    LM_ERR("Failed to append redis command\n");
2168
+                    goto error;
2169
+                }
2170
+
2171
+                db_redis_key_free(&query_v);
2172
+
2173
+                if (db_redis_key_add_string(&query_v, "SADD", 4) != 0) {
2174
+                    LM_ERR("Failed to set sadd command to post-update query\n");
2175
+                    goto error;
2176
+                }
2177
+                if (db_redis_key_add_str(&query_v, &set_key->key) != 0) {
2178
+                    LM_ERR("Failed to add map key to post-update query\n");
2179
+                    goto error;
2180
+                }
2181
+                if (db_redis_key_add_str(&query_v, &new_type_key->key) != 0) {
2182
+                    LM_ERR("Failed to set entry key to post-update query\n");
2183
+                    goto error;
2184
+                }
2185
+
2186
+                update_queries++;
2187
+                if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) {
2188
+                    LM_ERR("Failed to append redis command\n");
2189
+                    goto error;
2190
+                }
2191
+
2192
+                db_redis_key_free(&query_v);
2193
+
2194
+                if (db_redis_key_add_string(&query_v, "EVAL", 4) != 0) {
2195
+                    LM_ERR("Failed to add srem command to post-delete query\n");
2196
+                    goto error;
2197
+                }
2198
+                if (db_redis_key_add_string(&query_v, SREM_KEY_LUA, strlen(SREM_KEY_LUA)) != 0) {
2199
+                    LM_ERR("Failed to add srem command to post-delete query\n");
2200
+                    goto error;
2201
+                }
2202
+                if (db_redis_key_add_string(&query_v, "3", 1) != 0) {
2203
+                    LM_ERR("Failed to add srem command to post-delete query\n");
2204
+                    goto error;
2205
+                }
2206
+                if (db_redis_key_add_str(&query_v, &type_key->key) != 0) {
2207
+                    LM_ERR("Failed to add key to delete query\n");
2208
+                    goto error;
2209
+                }
2210
+                if (db_redis_key_add_str(&query_v, &set_key->key) != 0) {
2211
+                    LM_ERR("Failed to add key to delete query\n");
2212
+                    goto error;
2213
+                }
2214
+                if (db_redis_key_add_str(&query_v, &key->key) != 0) {
2215
+                    LM_ERR("Failed to add key to delete query\n");
2216
+                    goto error;
2217
+                }
2218
+
2219
+                update_queries++;
2220
+                if (db_redis_append_command_argv(con, query_v, 1) != REDIS_OK) {
2221
+                    LM_ERR("Failed to append redis command\n");
2222
+                    goto error;
2223
+                }
2224
+
2225
+                db_redis_key_free(&query_v);
2226
+            }
2227
+        }
2228
+
2229
+        db_redis_key_free(&type_keys);
2230
+        db_redis_key_free(&set_keys);
1684 2231
     }
1685 2232
 
1686 2233
     LM_DBG("getting replies for %d queries\n", update_queries);
... ...
@@ -1697,6 +2244,8 @@ static int db_redis_perform_update(const db1_con_t* _h, km_redis_con_t *con, con
1697 2244
 
1698 2245
     LM_DBG("done performing update\n");
1699 2246
 
2247
+    db_redis_key_free(&all_type_keys);
2248
+    db_redis_key_free(&new_type_keys);
1700 2249
     return 0;
1701 2250
 
1702 2251
 error:
... ...
@@ -1704,6 +2253,10 @@ error:
1704 2253
     if (reply)
1705 2254
         db_redis_free_reply(&reply);
1706 2255
     db_redis_key_free(&query_v);
2256
+    db_redis_key_free(&all_type_keys);
2257
+    db_redis_key_free(&type_keys);
2258
+    db_redis_key_free(&set_keys);
2259
+    db_redis_key_free(&new_type_keys);
1707 2260
     return -1;
1708 2261
 }
1709 2262
 
... ...
@@ -1726,6 +2279,8 @@ int db_redis_query(const db1_con_t* _h, const db_key_t* _k, const db_op_t* _op,
1726 2279
     km_redis_con_t *con = NULL;
1727 2280
     int free_op = 0;
1728 2281
     int do_table_scan = 0;
2282
+    uint64_t ts_scan_start = 0;
2283
+    str ts_scan_key = {0,};
1729 2284
 
1730 2285
     redis_key_t *keys = NULL;
1731 2286
     int keys_count = 0;
... ...
@@ -1796,7 +2351,8 @@ int db_redis_query(const db1_con_t* _h, const db_key_t* _k, const db_op_t* _op,
1796 2351
 
1797 2352
     if (_n > 0) {
1798 2353
         if (db_redis_build_query_keys(con, CON_TABLE(_h), _k, _v, query_ops, _n,
1799
-                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan) != 0) {
2354
+                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan, &ts_scan_start,
2355
+                    &ts_scan_key) != 0) {
1800 2356
             LM_ERR("failed to build query keys\n");
1801 2357
             goto error;
1802 2358
         }
... ...
@@ -1814,7 +2370,7 @@ int db_redis_query(const db1_con_t* _h, const db_key_t* _k, const db_op_t* _op,
1814 2370
     }
1815 2371
 
1816 2372
     if (db_redis_perform_query(_h, con, _k, _v, query_ops, _c, _n, _nc, _r,
1817
-        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan) != 0) {
2373
+        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan, ts_scan_start, &ts_scan_key) != 0) {
1818 2374
         goto error;
1819 2375
     }
1820 2376
 
... ...
@@ -1828,6 +2384,8 @@ int db_redis_query(const db1_con_t* _h, const db_key_t* _k, const db_op_t* _op,
1828 2384
     if (manual_keys) {
1829 2385
         pkg_free(manual_keys);
1830 2386
     }
2387
+    if (ts_scan_key.s)
2388
+        pkg_free(ts_scan_key.s);
1831 2389
 
1832 2390
     db_redis_consume_replies(con);
1833 2391
     return 0;
... ...
@@ -1841,6 +2399,8 @@ error:
1841 2399
     if (manual_keys) {
1842 2400
         pkg_free(manual_keys);
1843 2401
     }
2402
+    if (ts_scan_key.s)
2403
+        pkg_free(ts_scan_key.s);
1844 2404
     db_redis_consume_replies(con);
1845 2405
 
1846 2406
 
... ...
@@ -1869,11 +2429,13 @@ int db_redis_insert(const db1_con_t* _h, const db_key_t* _k, const db_val_t* _v,
1869 2429
     redis_key_t *key = NULL;
1870 2430
     int keys_count = 0;
1871 2431
     redis_key_t *type_keys = NULL;
2432
+    redis_key_t *set_keys = NULL;
1872 2433
     int type_keys_count = 0;
1873 2434
     redis_key_t *query_v = NULL;
1874 2435
     redisReply *reply = NULL;
1875 2436
     int i;
1876 2437
     redis_key_t *k;
2438
+    redis_key_t *set_key;
1877 2439
 
1878 2440
     con = REDIS_CON(_h);
1879 2441
     if (con && con->con == NULL) {
... ...
@@ -1900,7 +2462,7 @@ int db_redis_insert(const db1_con_t* _h, const db_key_t* _k, const db_val_t* _v,
1900 2462
         goto error;
1901 2463
     }
1902 2464
     if (db_redis_build_type_keys(con, CON_TABLE(_h), _k, _v, _n,
1903
-                &type_keys, &type_keys_count) != 0) {
2465
+                &type_keys, &set_keys, &type_keys_count) != 0) {
1904 2466
         LM_ERR("failed to build type keys\n");
1905 2467
         goto error;
1906 2468
     }
... ...
@@ -1939,7 +2501,7 @@ int db_redis_insert(const db1_con_t* _h, const db_key_t* _k, const db_val_t* _v,
1939 2501
     db_redis_check_reply(con, reply, error);
1940 2502
     db_redis_free_reply(&reply);
1941 2503
 
1942
-    for (k = type_keys; k; k = k->next) {
2504
+    for (k = type_keys, set_key = set_keys; k; k = k->next, set_key = set_key->next) {
1943 2505
         str *type_key = &k->key;
1944 2506
 
1945 2507
         LM_DBG("inserting entry key '%.*s' to type map '%.*s'\n",
... ...
@@ -1962,10 +2524,29 @@ int db_redis_insert(const db1_con_t* _h, const db_key_t* _k, const db_val_t* _v,
1962 2524
         db_redis_key_free(&query_v);
1963 2525
         db_redis_check_reply(con, reply, error);
1964 2526
         db_redis_free_reply(&reply);
2527
+
2528
+        if (db_redis_key_add_string(&query_v, "SADD", 4) != 0) {
2529
+            LM_ERR("Failed to set sadd command to post-insert query\n");
2530
+            goto error;
2531
+        }
2532
+        if (db_redis_key_add_str(&query_v, &set_key->key) != 0) {
2533
+            LM_ERR("Failed to add map key to post-insert query\n");
2534
+            goto error;
2535
+        }
2536
+        if (db_redis_key_add_str(&query_v, type_key) != 0) {
2537
+            LM_ERR("Failed to set entry key to post-insert query\n");
2538
+            goto error;
2539
+        }
2540
+
2541
+        reply = db_redis_command_argv(con, query_v);
2542
+        db_redis_key_free(&query_v);
2543
+        db_redis_check_reply(con, reply, error);
2544
+        db_redis_free_reply(&reply);
1965 2545
     }
1966 2546
 
1967 2547
     db_redis_key_free(&key);
1968 2548
     db_redis_key_free(&type_keys);
2549
+    db_redis_key_free(&set_keys);
1969 2550
     db_redis_consume_replies(con);
1970 2551
 
1971 2552
     return 0;
... ...
@@ -1973,6 +2554,7 @@ int db_redis_insert(const db1_con_t* _h, const db_key_t* _k, const db_val_t* _v,
1973 2554
 error:
1974 2555
     db_redis_key_free(&key);
1975 2556
     db_redis_key_free(&type_keys);
2557
+    db_redis_key_free(&set_keys);
1976 2558
     db_redis_key_free(&query_v);
1977 2559
 
1978 2560
     if (reply)
... ...
@@ -2002,6 +2584,8 @@ int db_redis_delete(const db1_con_t* _h, const db_key_t* _k,
2002 2584
     int manual_keys_count = 0;
2003 2585
     int free_op = 0;
2004 2586
     int do_table_scan = 0;
2587
+    uint64_t ts_scan_start = 0;
2588
+    str ts_scan_key = {0,};
2005 2589
     db_op_t *query_ops = NULL;
2006 2590
     int i;
2007 2591
 
... ...
@@ -2046,7 +2630,8 @@ int db_redis_delete(const db1_con_t* _h, const db_key_t* _k,
2046 2630
 
2047 2631
     if (_n > 0) {
2048 2632
         if (db_redis_build_query_keys(con, CON_TABLE(_h), _k, _v, query_ops, _n,
2049
-                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan) != 0) {
2633
+                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan, &ts_scan_start,
2634
+                    &ts_scan_key) != 0) {
2050 2635
             LM_ERR("failed to build query keys\n");
2051 2636
             goto error;
2052 2637
         }
... ...
@@ -2063,7 +2648,7 @@ int db_redis_delete(const db1_con_t* _h, const db_key_t* _k,
2063 2648
     }
2064 2649
 
2065 2650
     if (db_redis_perform_delete(_h, con, _k, _v, query_ops, _n,
2066
-        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan) != 0) {
2651
+        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan, ts_scan_start, &ts_scan_key) != 0) {
2067 2652
         goto error;
2068 2653
     }
2069 2654
 
... ...
@@ -2075,6 +2660,8 @@ int db_redis_delete(const db1_con_t* _h, const db_key_t* _k,
2075 2660
     db_redis_key_free(&keys);
2076 2661
     if (manual_keys)
2077 2662
         pkg_free(manual_keys);
2663
+    if (ts_scan_key.s)
2664
+        pkg_free(ts_scan_key.s);
2078 2665
     db_redis_consume_replies(con);
2079 2666
 
2080 2667
     return 0;
... ...
@@ -2087,6 +2674,8 @@ error:
2087 2674
     db_redis_key_free(&keys);
2088 2675
     if (manual_keys)
2089 2676
         pkg_free(manual_keys);
2677
+    if (ts_scan_key.s)
2678
+        pkg_free(ts_scan_key.s);
2090 2679
     db_redis_consume_replies(con);
2091 2680
     return -1;
2092 2681
 }
... ...
@@ -2109,6 +2698,8 @@ int db_redis_update(const db1_con_t* _h, const db_key_t* _k,
2109 2698
     km_redis_con_t *con = NULL;
2110 2699
     int free_op = 0;
2111 2700
     int do_table_scan = 0;
2701
+    uint64_t ts_scan_start = 0;
2702
+    str ts_scan_key = {0,};
2112 2703
 
2113 2704
     redis_key_t *keys = NULL;
2114 2705
     int keys_count = 0;
... ...
@@ -2158,7 +2749,8 @@ int db_redis_update(const db1_con_t* _h, const db_key_t* _k,
2158 2749
 
2159 2750
     if (_n > 0) {
2160 2751
         if (db_redis_build_query_keys(con, CON_TABLE(_h), _k, _v, query_ops, _n,
2161
-                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan) != 0) {
2752
+                    &keys, &keys_count, &manual_keys, &manual_keys_count, &do_table_scan, &ts_scan_start,
2753
+                    &ts_scan_key) != 0) {
2162 2754
             LM_ERR("failed to build query keys\n");
2163 2755
             goto error;
2164 2756
         }
... ...
@@ -2175,7 +2767,7 @@ int db_redis_update(const db1_con_t* _h, const db_key_t* _k,
2175 2767
     }
2176 2768
 
2177 2769
     if (db_redis_perform_update(_h, con, _k, _v, query_ops, _uk, _uv, _n, _nu,
2178
-        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan) != 0) {
2770
+        &keys, &keys_count, &manual_keys, &manual_keys_count, do_table_scan, ts_scan_start, &ts_scan_key) != 0) {
2179 2771
         goto error;
2180 2772
     }
2181 2773
 
... ...
@@ -2189,6 +2781,8 @@ int db_redis_update(const db1_con_t* _h, const db_key_t* _k,
2189 2781
     if (manual_keys) {
2190 2782
         pkg_free(manual_keys);
2191 2783
     }
2784
+    if (ts_scan_key.s)
2785
+        pkg_free(ts_scan_key.s);
2192 2786
     db_redis_consume_replies(con);
2193 2787
     return 0;
2194 2788
 
... ...
@@ -2201,6 +2795,8 @@ error:
2201 2795
     if (manual_keys) {
2202 2796
         pkg_free(manual_keys);
2203 2797
     }
2798
+    if (ts_scan_key.s)
2799
+        pkg_free(ts_scan_key.s);
2204 2800
     db_redis_consume_replies(con);
2205 2801
     return -1;
2206 2802
 }
... ...
@@ -25,6 +25,9 @@
25 25
 
26 26
 #include "db_redis_mod.h"
27 27
 
28
+#define SREM_KEY_LUA "redis.call('SREM', KEYS[1], KEYS[3]); if redis.call('SCARD', KEYS[1]) == 0 then redis.call('SREM', KEYS[2], KEYS[1]) end"
29
+
30
+
28 31
 /*
29 32
  * Initialize database connection
30 33
  */
... ...
@@ -85,4 +88,4 @@ int db_redis_replace(const db1_con_t* handle, const db_key_t* keys, const db_val
85 88
  */
86 89
 int db_redis_use_table(db1_con_t* _h, const str* _t);
87 90