Browse code

- tcp children control fds are now non-blocking - send fd queue is used when a childs queue is full

Andrei Pelinescu-Onciul authored on 03/02/2006 16:15:18
Showing 4 changed files
... ...
@@ -109,7 +109,8 @@ again:
109 109
 	if (n<0){
110 110
 			/* error */
111 111
 		if (errno==EINTR) goto again; /* signal, try again */
112
-		LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
112
+		if ((errno!=EAGAIN) &&(errno!=EWOULDBLOCK))
113
+			LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
113 114
 					socket, strerror(errno));
114 115
 	}
115 116
 	return n;
... ...
@@ -158,8 +159,9 @@ again:
158 158
 	ret=sendmsg(unix_socket, &msg, 0);
159 159
 	if (ret<0){
160 160
 		if (errno==EINTR) goto again;
161
-		LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
162
-				unix_socket, strerror(errno));
161
+		if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK))
162
+			LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
163
+					unix_socket, strerror(errno));
163 164
 	}
164 165
 	
165 166
 	return ret;
... ...
@@ -43,7 +43,7 @@
43 43
 
44 44
 #define TCP_CON_MAX_ALIASES 4 /* maximum number of port aliases */
45 45
 
46
-#define TCP_BUF_SIZE 65535
46
+#define TCP_BUF_SIZE	4096 
47 47
 #define DEFAULT_TCP_CONNECTION_LIFETIME 120 /* in  seconds */
48 48
 #define DEFAULT_TCP_SEND_TIMEOUT 10 /* if a send can't write for more then 10s,
49 49
 									   timeout */
... ...
@@ -64,6 +64,8 @@
64 64
  *              process all children requests, before attempting to send
65 65
  *              them new stuff (fixes some deadlocks) (andrei)
66 66
  *  2006-02-03  timers are run only once per s (andrei)
67
+ *              tcp children fds can be non-blocking; send fds are queues on
68
+ *              EAGAIN (andrei)
67 69
  */
68 70
 
69 71
 
... ...
@@ -123,10 +125,14 @@
123 123
 
124 124
 #define MAX_TCP_CHILDREN 100
125 125
 #define TCP_LISTEN_BACKLOG 1024
126
-/* #define SEND_FD_QUEUE */ /* queue send fd request, instead of sending 
127
-							   them immediately */
126
+#define SEND_FD_QUEUE /* queue send fd requests on EAGAIN, instead of sending 
127
+							them immediately */
128
+#define TCP_CHILD_NON_BLOCKING 
128 129
 #ifdef SEND_FD_QUEUE
129
-#define MAX_SEND_FD_QUEUE_SIZE	tcp_max_fd_no
130
+#ifndef TCP_CHILD_NON_BLOCKING
131
+#define TCP_CHILD_NON_BLOCKING
132
+#endif
133
+#define MAX_SEND_FD_QUEUE_SIZE	1024  /* alternative: tcp_max_fd_no */
130 134
 #define SEND_FD_QUEUE_SIZE		128  /* initial size */
131 135
 #define MAX_SEND_FD_RETRIES		3	 /* FIXME: increase */
132 136
 #endif
... ...
@@ -985,8 +991,6 @@ static void send_fd_queue_destroy(struct tcp_send_fd_q *q)
985 985
 
986 986
 static int init_send_fd_queues()
987 987
 {
988
-	if (send_fd_queue_init(&get_fd_q, SEND_FD_QUEUE_SIZE)!=0)
989
-		goto error;
990 988
 	if (send_fd_queue_init(&send2child_q, SEND_FD_QUEUE_SIZE)!=0)
991 989
 		goto error;
992 990
 	return 0;
... ...
@@ -999,7 +1003,6 @@ error:
999 999
 
1000 1000
 static void destroy_send_fd_queues()
1001 1001
 {
1002
-	send_fd_queue_destroy(&get_fd_q);
1003 1002
 	send_fd_queue_destroy(&send2child_q);
1004 1003
 }
1005 1004
 
... ...
@@ -1053,13 +1056,14 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
1053 1053
 	for (p=t=&q->data[0]; p<q->crt; p++){
1054 1054
 		if (send_fd(p->unix_sock, &(p->tcp_conn),
1055 1055
 					sizeof(struct tcp_connection*), p->tcp_conn->s)<=0){
1056
-			if (/*FIXME: E_WOULD_BLOCK && */(p->retries<MAX_SEND_FD_RETRIES)){
1056
+			if ( ((errno==EAGAIN)||(errno==EWOULDBLOCK)) && 
1057
+							(p->retries<MAX_SEND_FD_RETRIES)){
1057 1058
 				/* leave in queue for a future try */
1058 1059
 				*t=*p;
1059 1060
 				t->retries++;
1060 1061
 				t++;
1061 1062
 			}else{
1062
-				LOG(L_ERR, "ERROR: rund_send_fd_queue: send_fd failed"
1063
+				LOG(L_ERR, "ERROR: run_send_fd_queue: send_fd failed"
1063 1064
 						   "on %d socket, %ld queue entry, retries %d \n",
1064 1065
 						   p->unix_sock, p-&q->data[0], p->retries);
1065 1066
 			}
... ...
@@ -1378,10 +1382,21 @@ inline static int send2child(struct tcp_connection* tcpconn)
1378 1378
 	while(handle_tcp_child(&tcp_children[idx], -1)>0);
1379 1379
 		
1380 1380
 #ifdef SEND_FD_QUEUE
1381
-	if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
1381
+	/* if queue full, try to queue the io */
1382
+	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
1383
+			tcpconn->s)<=0){
1384
+		if ((errno==EAGAIN)||(errno==EWOULDBLOCK)){
1385
+			/* FIXME: remove after debugging */
1386
+			LOG(L_WARN, "WARNING: tcp child %d, socket %d: queue full\n",
1387
+					idx, tcp_children[idx].unix_sock);
1388
+			if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
1382 1389
 						tcpconn)!=0){
1383
-		LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
1384
-		return -1;
1390
+				LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
1391
+				return -1;
1392
+			}
1393
+		}else{
1394
+			LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
1395
+		}
1385 1396
 	}
1386 1397
 #else
1387 1398
 	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
... ...
@@ -1856,6 +1871,29 @@ error:
1856 1856
 
1857 1857
 
1858 1858
 
1859
+/* returns -1 on error */
1860
+static int set_non_blocking(int s)
1861
+{
1862
+	int flags;
1863
+	/* non-blocking */
1864
+	flags=fcntl(s, F_GETFL);
1865
+	if (flags==-1){
1866
+		LOG(L_ERR, "ERROR: set_non_blocking: fnctl failed: (%d) %s\n",
1867
+				errno, strerror(errno));
1868
+		goto error;
1869
+	}
1870
+	if (fcntl(s, F_SETFL, flags|O_NONBLOCK)==-1){
1871
+		LOG(L_ERR, "ERROR: set_non_blocking: fcntl: set non-blocking failed:"
1872
+				" (%d) %s\n", errno, strerror(errno));
1873
+		goto error;
1874
+	}
1875
+	return 0;
1876
+error:
1877
+	return -1;
1878
+}
1879
+
1880
+
1881
+
1859 1882
 /* starts the tcp processes */
1860 1883
 int tcp_init_children()
1861 1884
 {
... ...
@@ -1893,6 +1931,15 @@ int tcp_init_children()
1893 1893
 					strerror(errno));
1894 1894
 			goto error;
1895 1895
 		}
1896
+#ifdef TCP_CHILD_NON_BLOCKING
1897
+		if ((set_non_blocking(reader_fd[0])<0) || 
1898
+			(set_non_blocking(reader_fd[1])<0)){
1899
+			LOG(L_ERR, "ERROR: tcp_main: failed to set non blocking"
1900
+						"on child sockets\n");
1901
+			/* continue, it's not critical (it will go slower under
1902
+			 * very high connection rates) */
1903
+		}
1904
+#endif
1896 1905
 		
1897 1906
 		process_no++;
1898 1907
 		child_rank++;
... ...
@@ -6,6 +6,7 @@ egcs-2.91.66
6 6
 3.0.4
7 7
 gcc (GCC) 3.2
8 8
 gcc-3.2 (GCC) 3.2.3 (Debian)
9
+gcc (GCC) 3.3 20030304 (Apple Computer, Inc. build 1809)
9 10
 gcc (GCC) 3.3.3 (NetBSD nb3 20040520)
10 11
 gcc (GCC) 3.3.5 (propolice)
11 12
 gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-6)
... ...
@@ -13,6 +14,7 @@ gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-13)
13 13
 gcc (GCC) 3.4.2 [FreeBSD] 20040728
14 14
 gcc (GCC) 3.4.3 (csl-sol210-3_4-branch+sol_rpath)
15 15
 gcc-3.4 (GCC) 3.4.4 20041218 (prerelease) (Debian 3.4.3-7)
16
+powerpc-apple-darwin8-gcc-3.5.0 (GCC) 3.5.0 20041026 (Apple Computer, Inc. build 4059)
16 17
 powerpc-apple-darwin8-gcc-4.0.0 (GCC) 4.0.0 20041026 (Apple Computer, Inc. build 4061)
17 18
 gcc-4.0 (GCC) 4.0.2 20050821 (prerelease) (Debian 4.0.1-6)
18 19
 gcc-4.0.1 (GCC) 3.3.6 (Gentoo 3.3.6, ssp-3.3.6-1.0, pie-8.7.8)