Browse code

- high connection number deadlock fix (tested with 50k open tcp connections) - experimental queue send fd ops code WARNING: lightly tested

Andrei Pelinescu-Onciul authored on 30/01/2006 19:24:30
Showing 1 changed files
... ...
@@ -59,7 +59,10 @@
59 59
  *  2005-07-08  tcp_max_connections, tcp_connection_lifetime, don't accept
60 60
  *               more connections if tcp_max_connections is exceeded (andrei)
61 61
  *  2005-10-21  cleanup all the open connections on exit
62
- *              decrement the no. of open connections on timeout too    (andrei)
62
+ *              decrement the no. of open connections on timeout too    (andrei) *  2006-01-30  queue send_fd request and execute them at the end of the
63
+ *              poll loop  (#ifdef) (andrei)
64
+ *              process all children requests, before attempting to send
65
+ *              them new stuff (fixes some deadlocks) (andrei)
63 66
  */
64 67
 
65 68
 
... ...
@@ -119,6 +122,13 @@
119 122
 
120 123
 #define MAX_TCP_CHILDREN 100
121 124
 
125
+/* #define SEND_FD_QUEUE */ /* queue send fd request, instead of sending 
126
+							   them immediately */
127
+#ifdef SEND_FD_QUEUE
128
+#define MAX_SEND_FD_QUEUE_SIZE	tcp_max_fd_no
129
+#define SEND_FD_QUEUE_SIZE		128  /* initial size */
130
+#define MAX_SEND_FD_RETRIES		3	 /* FIXME: increase */
131
+#endif
122 132
 
123 133
 
124 134
 enum fd_types { F_NONE, F_SOCKINFO /* a tcp_listen fd */,
... ...
@@ -911,7 +921,7 @@ int tcp_init(struct socket_info* sock_info)
911 921
 				strerror(errno));
912 922
 		goto error;
913 923
 	}
914
-	if (listen(sock_info->socket, 10)==-1){
924
+	if (listen(sock_info->socket, 1024)==-1){
915 925
 		LOG(L_ERR, "ERROR: tcp_init: listen(%x, %p, %d) on %s: %s\n",
916 926
 				sock_info->socket, &addr->s, 
917 927
 				(unsigned)sockaddru_len(*addr),
... ...
@@ -931,109 +941,134 @@ error:
931 941
 
932 942
 
933 943
 
934
-static int send2child(struct tcp_connection* tcpconn)
944
+#ifdef SEND_FD_QUEUE
945
+struct send_fd_info{
946
+	struct tcp_connection* tcp_conn;
947
+	int unix_sock;
948
+	int retries;
949
+};
950
+
951
+struct tcp_send_fd_q{
952
+	struct send_fd_info* data; /* buffer */
953
+	struct send_fd_info* crt;  /* pointer inside the buffer */
954
+	struct send_fd_info* end;  /* points after the last valid position */
955
+};
956
+
957
+
958
+static struct tcp_send_fd_q send2child_q;
959
+
960
+
961
+
962
+static int send_fd_queue_init(struct tcp_send_fd_q *q, unsigned int size)
935 963
 {
936
-	int i;
937
-	int min_busy;
938
-	int idx;
939
-	
940
-	min_busy=tcp_children[0].busy;
941
-	idx=0;
942
-	for (i=0; i<tcp_children_no; i++){
943
-		if (!tcp_children[i].busy){
944
-			idx=i;
945
-			min_busy=0;
946
-			break;
947
-		}else if (min_busy>tcp_children[i].busy){
948
-			min_busy=tcp_children[i].busy;
949
-			idx=i;
950
-		}
951
-	}
952
-	
953
-	tcp_children[idx].busy++;
954
-	tcp_children[idx].n_reqs++;
955
-	if (min_busy){
956
-		DBG("WARNING: send2child: no free tcp receiver, "
957
-				" connection passed to the least busy one (%d)\n",
958
-				min_busy);
959
-	}
960
-	DBG("send2child: to tcp child %d %d(%d), %p\n", idx, 
961
-					tcp_children[idx].proc_no,
962
-					tcp_children[idx].pid, tcpconn);
963
-	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
964
-			tcpconn->s)<=0){
965
-		LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
964
+	q->data=pkg_malloc(size*sizeof(struct send_fd_info));
965
+	if (q->data==0){
966
+		LOG(L_ERR, "ERROR: send_fd_queue_init: out of memory\n");
966 967
 		return -1;
967 968
 	}
968
-	
969
+	q->crt=&q->data[0];
970
+	q->end=&q->data[size];
969 971
 	return 0;
970 972
 }
971 973
 
974
+static void send_fd_queue_destroy(struct tcp_send_fd_q *q)
975
+{
976
+	if (q->data){
977
+		pkg_free(q->data);
978
+		q->data=0;
979
+		q->crt=q->end=0;
980
+	}
981
+}
972 982
 
973
-/* handles a new connection, called internally by tcp_main_loop/handle_io.
974
- * params: si - pointer to one of the tcp socket_info structures on which
975
- *              an io event was detected (connection attempt)
976
- * returns:  handle_* return convention: -1 on error, 0 on EAGAIN (no more
977
- *           io events queued), >0 on success. success/error refer only to
978
- *           the accept.
979
- */
980
-static inline int handle_new_connect(struct socket_info* si)
983
+
984
+
985
+static int init_send_fd_queues()
981 986
 {
982
-	union sockaddr_union su;
983
-	struct tcp_connection* tcpconn;
984
-	socklen_t su_len;
985
-	int new_sock;
987
+	if (send_fd_queue_init(&get_fd_q, SEND_FD_QUEUE_SIZE)!=0)
988
+		goto error;
989
+	if (send_fd_queue_init(&send2child_q, SEND_FD_QUEUE_SIZE)!=0)
990
+		goto error;
991
+	return 0;
992
+error:
993
+	LOG(L_ERR, "ERROR: init_send_fd_queues: init failed\n");
994
+	return -1;
995
+}
996
+
997
+
998
+
999
+static void destroy_send_fd_queues()
1000
+{
1001
+	send_fd_queue_destroy(&get_fd_q);
1002
+	send_fd_queue_destroy(&send2child_q);
1003
+}
1004
+
1005
+
1006
+
1007
+
1008
+inline static int send_fd_queue_add(	struct tcp_send_fd_q* q, 
1009
+										int unix_sock,
1010
+										struct tcp_connection *t)
1011
+{
1012
+	struct send_fd_info* tmp;
1013
+	unsigned long new_size;
986 1014
 	
987
-	/* got a connection on r */
988
-	su_len=sizeof(su);
989
-	new_sock=accept(si->socket, &(su.s), &su_len);
990
-	if (new_sock==-1){
991
-		if ((errno==EAGAIN)||(errno==EWOULDBLOCK))
992
-			return 0;
993
-		LOG(L_ERR,  "WARNING: handle_new_connect: error while accepting"
994
-				" connection(%d): %s\n", errno, strerror(errno));
995
-		return -1;
996
-	}
997
-	if (tcp_connections_no>=tcp_max_connections){
998
-		LOG(L_ERR, "ERROR: maximum number of connections exceeded: %d/%d\n",
999
-					tcp_connections_no, tcp_max_connections);
1000
-		close(new_sock);
1001
-		return 1; /* success, because the accept was succesfull */
1002
-	}
1003
-	if (init_sock_opt(new_sock)<0){
1004
-		LOG(L_ERR, "ERROR: handle_new_connect: init_sock_opt failed\n");
1005
-		close(new_sock);
1006
-		return 1; /* success, because the accept was succesfull */
1007
-	}
1015
+	if (q->crt>=q->end){
1016
+		new_size=q->end-&q->data[0];
1017
+		if (new_size< MAX_SEND_FD_QUEUE_SIZE/2){
1018
+			new_size*=2;
1019
+		}else new_size=MAX_SEND_FD_QUEUE_SIZE;
1020
+		if (q->crt>=&q->data[new_size]){
1021
+			LOG(L_ERR, "ERROR: send_fd_queue_add: queue full: %ld/%ld\n",
1022
+					q->crt-&q->data[0]-1, new_size);
1023
+			goto error;
1024
+		}
1025
+		LOG(L_CRIT, "INFO: send_fd_queue: queue full: %ld, extending to %ld\n",
1026
+				q->end-&q->data[0], new_size);
1027
+		tmp=pkg_realloc(q->data, new_size*sizeof(struct send_fd_info));
1028
+		if (tmp==0){
1029
+			LOG(L_ERR, "ERROR: send_fd_queue_add: out of memory\n");
1030
+			goto error;
1031
+		}
1032
+		q->crt=(q->crt-&q->data[0])+tmp;
1033
+		q->data=tmp;
1034
+		q->end=&q->data[new_size];
1035
+	}
1036
+	q->crt->tcp_conn=t;
1037
+	q->crt->unix_sock=unix_sock;
1038
+	q->crt->retries=0;
1039
+	q->crt++;
1040
+	return 0;
1041
+error:
1042
+	return -1;
1043
+}
1044
+
1045
+
1046
+
1047
+inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
1048
+{
1049
+	struct send_fd_info* p;
1050
+	struct send_fd_info* t;
1008 1051
 	
1009
-	/* add socket to list */
1010
-	tcpconn=tcpconn_new(new_sock, &su, si, si->proto, S_CONN_ACCEPT);
1011
-	if (tcpconn){
1012
-		tcpconn->refcnt++; /* safe, not yet available to the
1013
-							  outside world */
1014
-		tcpconn_add(tcpconn);
1015
-		DBG("handle_new_connect: new connection: %p %d flags: %04x\n",
1016
-			tcpconn, tcpconn->s, tcpconn->flags);
1017
-		/* pass it to a child */
1018
-		if(send2child(tcpconn)<0){
1019
-			LOG(L_ERR,"ERROR: handle_new_connect: no children "
1020
-					"available\n");
1021
-			TCPCONN_LOCK;
1022
-			tcpconn->refcnt--;
1023
-			if (tcpconn->refcnt==0){
1024
-				close(tcpconn->s);
1025
-				_tcpconn_rm(tcpconn);
1026
-			}else tcpconn->timeout=0; /* force expire */
1027
-			TCPCONN_UNLOCK;
1052
+	for (p=t=&q->data[0]; p<q->crt; p++){
1053
+		if (send_fd(p->unix_sock, &(p->tcp_conn),
1054
+					sizeof(struct tcp_connection*), p->tcp_conn->s)<=0){
1055
+			if (/*FIXME: E_WOULD_BLOCK && */(p->retries<MAX_SEND_FD_RETRIES)){
1056
+				/* leave in queue for a future try */
1057
+				*t=*p;
1058
+				t->retries++;
1059
+				t++;
1060
+			}else{
1061
+				LOG(L_ERR, "ERROR: rund_send_fd_queue: send_fd failed"
1062
+						   "on %d socket, %ld queue entry, retries %d \n",
1063
+						   p->unix_sock, p-&q->data[0], p->retries);
1064
+			}
1028 1065
 		}
1029
-	}else{ /*tcpconn==0 */
1030
-		LOG(L_ERR, "ERROR: handle_new_connect: tcpconn_new failed, "
1031
-				"closing socket\n");
1032
-		close(new_sock);
1033
-		
1034 1066
 	}
1035
-	return 1; /* accept() was succesfull */
1067
+	q->crt=t;
1036 1068
 }
1069
+#else
1070
+#define send_fd_queue_run(q)
1071
+#endif
1037 1072
 
1038 1073
 
1039 1074
 
... ...
@@ -1069,61 +1104,6 @@ static void tcpconn_destroy(struct tcp_connection* tcpconn)
1069 1104
 
1070 1105
 
1071 1106
 
1072
-/* handles an io event on one of the watched tcp connections
1073
- * 
1074
- * params: tcpconn - pointer to the tcp_connection for which we have an io ev.
1075
- *         fd_i    - index in the fd_array table (needed for delete)
1076
- * returns:  handle_* return convention, but on success it always returns 0
1077
- *           (because it's one-shot, after a succesfull execution the fd is
1078
- *            removed from tcp_main's watch fd list and passed to a child =>
1079
- *            tcp_main is not interested in further io events that might be
1080
- *            queued for this fd)
1081
- */
1082
-inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, int fd_i)
1083
-{
1084
-	int fd;
1085
-	
1086
-	/*  is refcnt!=0 really necessary? 
1087
-	 *  No, in fact it's a bug: I can have the following situation: a send only
1088
-	 *   tcp connection used by n processes simultaneously => refcnt = n. In 
1089
-	 *   the same time I can have a read event and this situation is perfectly
1090
-	 *   valid. -- andrei
1091
-	 */
1092
-#if 0
1093
-	if ((tcpconn->refcnt!=0)){
1094
-		/* FIXME: might be valid for sigio_rt iff fd flags are not cleared
1095
-		 *        (there is a short window in which it could generate a sig
1096
-		 *         that would be catched by tcp_main) */
1097
-		LOG(L_CRIT, "BUG: handle_tcpconn_ev: io event on referenced"
1098
-					" tcpconn (%p), refcnt=%d, fd=%d\n",
1099
-					tcpconn, tcpconn->refcnt, tcpconn->s);
1100
-		return -1;
1101
-	}
1102
-#endif
1103
-	/* pass it to child, so remove it from the io watch list */
1104
-	DBG("handle_tcpconn_ev: data available on %p %d\n", tcpconn, tcpconn->s);
1105
-	if (io_watch_del(&io_h, tcpconn->s, fd_i, 0)==-1) goto error;
1106
-	tcpconn->flags|=F_CONN_REMOVED;
1107
-	tcpconn_ref(tcpconn); /* refcnt ++ */
1108
-	if (send2child(tcpconn)<0){
1109
-		LOG(L_ERR,"ERROR: handle_tcpconn_ev: no children available\n");
1110
-		TCPCONN_LOCK;
1111
-		tcpconn->refcnt--;
1112
-		if (tcpconn->refcnt==0){
1113
-			fd=tcpconn->s;
1114
-			_tcpconn_rm(tcpconn);
1115
-			close(fd);
1116
-		}else tcpconn->timeout=0; /* force expire*/
1117
-		TCPCONN_UNLOCK;
1118
-	}
1119
-	return 0; /* we are not interested in possibly queued io events, 
1120
-				 the fd was either passed to a child, or closed */
1121
-error:
1122
-	return -1;
1123
-}
1124
-
1125
-
1126
-
1127 1107
 /* handles io from a tcp child process
1128 1108
  * params: tcp_c - pointer in the tcp_children array, to the entry for
1129 1109
  *                 which an io event was detected 
... ...
@@ -1357,6 +1337,186 @@ error:
1357 1337
 
1358 1338
 
1359 1339
 
1340
+/* sends a tcpconn + fd to a choosen child */
1341
+inline static int send2child(struct tcp_connection* tcpconn)
1342
+{
1343
+	int i;
1344
+	int min_busy;
1345
+	int idx;
1346
+	
1347
+	min_busy=tcp_children[0].busy;
1348
+	idx=0;
1349
+	for (i=0; i<tcp_children_no; i++){
1350
+		if (!tcp_children[i].busy){
1351
+			idx=i;
1352
+			min_busy=0;
1353
+			break;
1354
+		}else if (min_busy>tcp_children[i].busy){
1355
+			min_busy=tcp_children[i].busy;
1356
+			idx=i;
1357
+		}
1358
+	}
1359
+	
1360
+	tcp_children[idx].busy++;
1361
+	tcp_children[idx].n_reqs++;
1362
+	if (min_busy){
1363
+		DBG("WARNING: send2child: no free tcp receiver, "
1364
+				" connection passed to the least busy one (%d)\n",
1365
+				min_busy);
1366
+	}
1367
+	DBG("send2child: to tcp child %d %d(%d), %p\n", idx, 
1368
+					tcp_children[idx].proc_no,
1369
+					tcp_children[idx].pid, tcpconn);
1370
+	/* first make sure this child doesn't have pending request for
1371
+	 * tcp_main (to avoid a possible deadlock: e.g. child wants to
1372
+	 * send a release command, but the master fills its socket buffer
1373
+	 * with new connection commands => deadlock) */
1374
+	/* answer tcp_send requests first */
1375
+	while(handle_ser_child(&pt[tcp_children[idx].proc_no], -1)>0);
1376
+	/* process tcp readers requests */
1377
+	while(handle_tcp_child(&tcp_children[idx], -1)>0);
1378
+		
1379
+#ifdef SEND_FD_QUEUE
1380
+	if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
1381
+						tcpconn)!=0){
1382
+		LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
1383
+		return -1;
1384
+	}
1385
+#else
1386
+	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
1387
+			tcpconn->s)<=0){
1388
+		LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
1389
+		return -1;
1390
+	}
1391
+#endif
1392
+	
1393
+	return 0;
1394
+}
1395
+
1396
+
1397
+
1398
+/* handles a new connection, called internally by tcp_main_loop/handle_io.
1399
+ * params: si - pointer to one of the tcp socket_info structures on which
1400
+ *              an io event was detected (connection attempt)
1401
+ * returns:  handle_* return convention: -1 on error, 0 on EAGAIN (no more
1402
+ *           io events queued), >0 on success. success/error refer only to
1403
+ *           the accept.
1404
+ */
1405
+static inline int handle_new_connect(struct socket_info* si)
1406
+{
1407
+	union sockaddr_union su;
1408
+	struct tcp_connection* tcpconn;
1409
+	socklen_t su_len;
1410
+	int new_sock;
1411
+	
1412
+	/* got a connection on r */
1413
+	su_len=sizeof(su);
1414
+	new_sock=accept(si->socket, &(su.s), &su_len);
1415
+	if (new_sock==-1){
1416
+		if ((errno==EAGAIN)||(errno==EWOULDBLOCK))
1417
+			return 0;
1418
+		LOG(L_ERR,  "WARNING: handle_new_connect: error while accepting"
1419
+				" connection(%d): %s\n", errno, strerror(errno));
1420
+		return -1;
1421
+	}
1422
+	if (tcp_connections_no>=tcp_max_connections){
1423
+		LOG(L_ERR, "ERROR: maximum number of connections exceeded: %d/%d\n",
1424
+					tcp_connections_no, tcp_max_connections);
1425
+		close(new_sock);
1426
+		return 1; /* success, because the accept was succesfull */
1427
+	}
1428
+	if (init_sock_opt(new_sock)<0){
1429
+		LOG(L_ERR, "ERROR: handle_new_connect: init_sock_opt failed\n");
1430
+		close(new_sock);
1431
+		return 1; /* success, because the accept was succesfull */
1432
+	}
1433
+	
1434
+	/* add socket to list */
1435
+	tcpconn=tcpconn_new(new_sock, &su, si, si->proto, S_CONN_ACCEPT);
1436
+	if (tcpconn){
1437
+		tcpconn->refcnt++; /* safe, not yet available to the
1438
+							  outside world */
1439
+		tcpconn_add(tcpconn);
1440
+		DBG("handle_new_connect: new connection: %p %d flags: %04x\n",
1441
+			tcpconn, tcpconn->s, tcpconn->flags);
1442
+		/* pass it to a child */
1443
+		if(send2child(tcpconn)<0){
1444
+			LOG(L_ERR,"ERROR: handle_new_connect: no children "
1445
+					"available\n");
1446
+			TCPCONN_LOCK;
1447
+			tcpconn->refcnt--;
1448
+			if (tcpconn->refcnt==0){
1449
+				close(tcpconn->s);
1450
+				_tcpconn_rm(tcpconn);
1451
+			}else tcpconn->timeout=0; /* force expire */
1452
+			TCPCONN_UNLOCK;
1453
+		}
1454
+	}else{ /*tcpconn==0 */
1455
+		LOG(L_ERR, "ERROR: handle_new_connect: tcpconn_new failed, "
1456
+				"closing socket\n");
1457
+		close(new_sock);
1458
+		
1459
+	}
1460
+	return 1; /* accept() was succesfull */
1461
+}
1462
+
1463
+
1464
+
1465
+/* handles an io event on one of the watched tcp connections
1466
+ * 
1467
+ * params: tcpconn - pointer to the tcp_connection for which we have an io ev.
1468
+ *         fd_i    - index in the fd_array table (needed for delete)
1469
+ * returns:  handle_* return convention, but on success it always returns 0
1470
+ *           (because it's one-shot, after a succesfull execution the fd is
1471
+ *            removed from tcp_main's watch fd list and passed to a child =>
1472
+ *            tcp_main is not interested in further io events that might be
1473
+ *            queued for this fd)
1474
+ */
1475
+inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, int fd_i)
1476
+{
1477
+	int fd;
1478
+	
1479
+	/*  is refcnt!=0 really necessary? 
1480
+	 *  No, in fact it's a bug: I can have the following situation: a send only
1481
+	 *   tcp connection used by n processes simultaneously => refcnt = n. In 
1482
+	 *   the same time I can have a read event and this situation is perfectly
1483
+	 *   valid. -- andrei
1484
+	 */
1485
+#if 0
1486
+	if ((tcpconn->refcnt!=0)){
1487
+		/* FIXME: might be valid for sigio_rt iff fd flags are not cleared
1488
+		 *        (there is a short window in which it could generate a sig
1489
+		 *         that would be catched by tcp_main) */
1490
+		LOG(L_CRIT, "BUG: handle_tcpconn_ev: io event on referenced"
1491
+					" tcpconn (%p), refcnt=%d, fd=%d\n",
1492
+					tcpconn, tcpconn->refcnt, tcpconn->s);
1493
+		return -1;
1494
+	}
1495
+#endif
1496
+	/* pass it to child, so remove it from the io watch list */
1497
+	DBG("handle_tcpconn_ev: data available on %p %d\n", tcpconn, tcpconn->s);
1498
+	if (io_watch_del(&io_h, tcpconn->s, fd_i, 0)==-1) goto error;
1499
+	tcpconn->flags|=F_CONN_REMOVED;
1500
+	tcpconn_ref(tcpconn); /* refcnt ++ */
1501
+	if (send2child(tcpconn)<0){
1502
+		LOG(L_ERR,"ERROR: handle_tcpconn_ev: no children available\n");
1503
+		TCPCONN_LOCK;
1504
+		tcpconn->refcnt--;
1505
+		if (tcpconn->refcnt==0){
1506
+			fd=tcpconn->s;
1507
+			_tcpconn_rm(tcpconn);
1508
+			close(fd);
1509
+		}else tcpconn->timeout=0; /* force expire*/
1510
+		TCPCONN_UNLOCK;
1511
+	}
1512
+	return 0; /* we are not interested in possibly queued io events, 
1513
+				 the fd was either passed to a child, or closed */
1514
+error:
1515
+	return -1;
1516
+}
1517
+
1518
+
1519
+
1360 1520
 /* generic handle io routine, it will call the appropiate
1361 1521
  *  handle_xxx() based on the fd_map type
1362 1522
  *
... ...
@@ -1515,6 +1675,7 @@ void tcp_main_loop()
1515 1675
 			while(1){
1516 1676
 				/* wait and process IO */
1517 1677
 				io_wait_loop_poll(&io_h, TCP_MAIN_SELECT_TIMEOUT, 0); 
1678
+				send_fd_queue_run(&send2child_q); /* then new io */
1518 1679
 				/* remove old connections */
1519 1680
 				tcpconn_timeout(0);
1520 1681
 			}
... ...
@@ -1523,6 +1684,7 @@ void tcp_main_loop()
1523 1684
 		case POLL_SELECT:
1524 1685
 			while(1){
1525 1686
 				io_wait_loop_select(&io_h, TCP_MAIN_SELECT_TIMEOUT, 0);
1687
+				send_fd_queue_run(&send2child_q); /* then new io */
1526 1688
 				tcpconn_timeout(0);
1527 1689
 			}
1528 1690
 			break;
... ...
@@ -1531,6 +1693,7 @@ void tcp_main_loop()
1531 1693
 		case POLL_SIGIO_RT:
1532 1694
 			while(1){
1533 1695
 				io_wait_loop_sigio_rt(&io_h, TCP_MAIN_SELECT_TIMEOUT);
1696
+				send_fd_queue_run(&send2child_q); /* then new io */
1534 1697
 				tcpconn_timeout(0);
1535 1698
 			}
1536 1699
 			break;
... ...
@@ -1539,12 +1702,14 @@ void tcp_main_loop()
1539 1702
 		case POLL_EPOLL_LT:
1540 1703
 			while(1){
1541 1704
 				io_wait_loop_epoll(&io_h, TCP_MAIN_SELECT_TIMEOUT, 0);
1705
+				send_fd_queue_run(&send2child_q); /* then new io */
1542 1706
 				tcpconn_timeout(0);
1543 1707
 			}
1544 1708
 			break;
1545 1709
 		case POLL_EPOLL_ET:
1546 1710
 			while(1){
1547 1711
 				io_wait_loop_epoll(&io_h, TCP_MAIN_SELECT_TIMEOUT, 1);
1712
+				send_fd_queue_run(&send2child_q); /* then new io */
1548 1713
 				tcpconn_timeout(0);
1549 1714
 			}
1550 1715
 			break;
... ...
@@ -1553,6 +1718,7 @@ void tcp_main_loop()
1553 1718
 		case POLL_KQUEUE:
1554 1719
 			while(1){
1555 1720
 				io_wait_loop_kqueue(&io_h, TCP_MAIN_SELECT_TIMEOUT, 0);
1721
+				send_fd_queue_run(&send2child_q); /* then new io */
1556 1722
 				tcpconn_timeout(0);
1557 1723
 			}
1558 1724
 			break;
... ...
@@ -1561,6 +1727,7 @@ void tcp_main_loop()
1561 1727
 		case POLL_DEVPOLL:
1562 1728
 			while(1){
1563 1729
 				io_wait_loop_devpoll(&io_h, TCP_MAIN_SELECT_TIMEOUT, 0);
1730
+				send_fd_queue_run(&send2child_q); /* then new io */
1564 1731
 				tcpconn_timeout(0);
1565 1732
 			}
1566 1733
 			break;
... ...
@@ -1600,6 +1767,9 @@ void destroy_tcp()
1600 1767
 			lock_dealloc((void*)tcpconn_lock);
1601 1768
 			tcpconn_lock=0;
1602 1769
 		}
1770
+#ifdef SEND_FD_QUEUE
1771
+		destroy_send_fd_queues();
1772
+#endif
1603 1773
 }
1604 1774
 
1605 1775
 
... ...
@@ -1645,7 +1815,14 @@ int init_tcp()
1645 1815
 			TCP_ALIAS_HASH_SIZE * sizeof(struct tcp_conn_alias*));
1646 1816
 	memset((void*)tcpconn_id_hash, 0, 
1647 1817
 			TCP_ID_HASH_SIZE * sizeof(struct tcp_connection*));
1818
+	/* init send fd queues */
1648 1819
 	
1820
+#ifdef SEND_FD_QUEUE
1821
+	if (init_send_fd_queues()<0){
1822
+		LOG(L_CRIT, "ERROR: init_tcp: could not init send fd queues\n");
1823
+		goto error;
1824
+	}
1825
+#endif
1649 1826
 	/* fix config variables */
1650 1827
 	/* they can have only positive values due the config parser so we can
1651 1828
 	 * ignore most of them */