Browse code

- tcp children control fds are now non-blocking - send fd queue is used when a childs queue is full

Andrei Pelinescu-Onciul authored on 03/02/2006 16:15:18
Showing 4 changed files
... ...
@@ -109,7 +109,8 @@ again:
109 109
 	if (n<0){
110 110
 			/* error */
111 111
 		if (errno==EINTR) goto again; /* signal, try again */
112
-		LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
112
+		if ((errno!=EAGAIN) &&(errno!=EWOULDBLOCK))
113
+			LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
113 114
 					socket, strerror(errno));
114 115
 	}
115 116
 	return n;
... ...
@@ -158,8 +159,9 @@ again:
158 159
 	ret=sendmsg(unix_socket, &msg, 0);
159 160
 	if (ret<0){
160 161
 		if (errno==EINTR) goto again;
161
-		LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
162
-				unix_socket, strerror(errno));
162
+		if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK))
163
+			LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
164
+					unix_socket, strerror(errno));
163 165
 	}
164 166
 	
165 167
 	return ret;
... ...
@@ -43,7 +43,7 @@
43 43
 
44 44
 #define TCP_CON_MAX_ALIASES 4 /* maximum number of port aliases */
45 45
 
46
-#define TCP_BUF_SIZE 65535
46
+#define TCP_BUF_SIZE	4096 
47 47
 #define DEFAULT_TCP_CONNECTION_LIFETIME 120 /* in  seconds */
48 48
 #define DEFAULT_TCP_SEND_TIMEOUT 10 /* if a send can't write for more then 10s,
49 49
 									   timeout */
... ...
@@ -64,6 +64,8 @@
64 64
  *              process all children requests, before attempting to send
65 65
  *              them new stuff (fixes some deadlocks) (andrei)
66 66
  *  2006-02-03  timers are run only once per s (andrei)
67
+ *              tcp children fds can be non-blocking; send fds are queues on
68
+ *              EAGAIN (andrei)
67 69
  */
68 70
 
69 71
 
... ...
@@ -123,10 +125,14 @@
123 125
 
124 126
 #define MAX_TCP_CHILDREN 100
125 127
 #define TCP_LISTEN_BACKLOG 1024
126
-/* #define SEND_FD_QUEUE */ /* queue send fd request, instead of sending 
127
-							   them immediately */
128
+#define SEND_FD_QUEUE /* queue send fd requests on EAGAIN, instead of sending 
129
+							them immediately */
130
+#define TCP_CHILD_NON_BLOCKING 
128 131
 #ifdef SEND_FD_QUEUE
129
-#define MAX_SEND_FD_QUEUE_SIZE	tcp_max_fd_no
132
+#ifndef TCP_CHILD_NON_BLOCKING
133
+#define TCP_CHILD_NON_BLOCKING
134
+#endif
135
+#define MAX_SEND_FD_QUEUE_SIZE	1024  /* alternative: tcp_max_fd_no */
130 136
 #define SEND_FD_QUEUE_SIZE		128  /* initial size */
131 137
 #define MAX_SEND_FD_RETRIES		3	 /* FIXME: increase */
132 138
 #endif
... ...
@@ -985,8 +991,6 @@ static void send_fd_queue_destroy(struct tcp_send_fd_q *q)
985 991
 
986 992
 static int init_send_fd_queues()
987 993
 {
988
-	if (send_fd_queue_init(&get_fd_q, SEND_FD_QUEUE_SIZE)!=0)
989
-		goto error;
990 994
 	if (send_fd_queue_init(&send2child_q, SEND_FD_QUEUE_SIZE)!=0)
991 995
 		goto error;
992 996
 	return 0;
... ...
@@ -999,7 +1003,6 @@ error:
999 1003
 
1000 1004
 static void destroy_send_fd_queues()
1001 1005
 {
1002
-	send_fd_queue_destroy(&get_fd_q);
1003 1006
 	send_fd_queue_destroy(&send2child_q);
1004 1007
 }
1005 1008
 
... ...
@@ -1053,13 +1056,14 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
1053 1056
 	for (p=t=&q->data[0]; p<q->crt; p++){
1054 1057
 		if (send_fd(p->unix_sock, &(p->tcp_conn),
1055 1058
 					sizeof(struct tcp_connection*), p->tcp_conn->s)<=0){
1056
-			if (/*FIXME: E_WOULD_BLOCK && */(p->retries<MAX_SEND_FD_RETRIES)){
1059
+			if ( ((errno==EAGAIN)||(errno==EWOULDBLOCK)) && 
1060
+							(p->retries<MAX_SEND_FD_RETRIES)){
1057 1061
 				/* leave in queue for a future try */
1058 1062
 				*t=*p;
1059 1063
 				t->retries++;
1060 1064
 				t++;
1061 1065
 			}else{
1062
-				LOG(L_ERR, "ERROR: rund_send_fd_queue: send_fd failed"
1066
+				LOG(L_ERR, "ERROR: run_send_fd_queue: send_fd failed"
1063 1067
 						   "on %d socket, %ld queue entry, retries %d \n",
1064 1068
 						   p->unix_sock, p-&q->data[0], p->retries);
1065 1069
 			}
... ...
@@ -1378,10 +1382,21 @@ inline static int send2child(struct tcp_connection* tcpconn)
1378 1382
 	while(handle_tcp_child(&tcp_children[idx], -1)>0);
1379 1383
 		
1380 1384
 #ifdef SEND_FD_QUEUE
1381
-	if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
1385
+	/* if queue full, try to queue the io */
1386
+	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
1387
+			tcpconn->s)<=0){
1388
+		if ((errno==EAGAIN)||(errno==EWOULDBLOCK)){
1389
+			/* FIXME: remove after debugging */
1390
+			LOG(L_WARN, "WARNING: tcp child %d, socket %d: queue full\n",
1391
+					idx, tcp_children[idx].unix_sock);
1392
+			if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
1382 1393
 						tcpconn)!=0){
1383
-		LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
1384
-		return -1;
1394
+				LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
1395
+				return -1;
1396
+			}
1397
+		}else{
1398
+			LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
1399
+		}
1385 1400
 	}
1386 1401
 #else
1387 1402
 	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
... ...
@@ -1856,6 +1871,29 @@ error:
1856 1871
 
1857 1872
 
1858 1873
 
1874
+/* returns -1 on error */
1875
+static int set_non_blocking(int s)
1876
+{
1877
+	int flags;
1878
+	/* non-blocking */
1879
+	flags=fcntl(s, F_GETFL);
1880
+	if (flags==-1){
1881
+		LOG(L_ERR, "ERROR: set_non_blocking: fnctl failed: (%d) %s\n",
1882
+				errno, strerror(errno));
1883
+		goto error;
1884
+	}
1885
+	if (fcntl(s, F_SETFL, flags|O_NONBLOCK)==-1){
1886
+		LOG(L_ERR, "ERROR: set_non_blocking: fcntl: set non-blocking failed:"
1887
+				" (%d) %s\n", errno, strerror(errno));
1888
+		goto error;
1889
+	}
1890
+	return 0;
1891
+error:
1892
+	return -1;
1893
+}
1894
+
1895
+
1896
+
1859 1897
 /* starts the tcp processes */
1860 1898
 int tcp_init_children()
1861 1899
 {
... ...
@@ -1893,6 +1931,15 @@ int tcp_init_children()
1893 1931
 					strerror(errno));
1894 1932
 			goto error;
1895 1933
 		}
1934
+#ifdef TCP_CHILD_NON_BLOCKING
1935
+		if ((set_non_blocking(reader_fd[0])<0) || 
1936
+			(set_non_blocking(reader_fd[1])<0)){
1937
+			LOG(L_ERR, "ERROR: tcp_main: failed to set non blocking"
1938
+						"on child sockets\n");
1939
+			/* continue, it's not critical (it will go slower under
1940
+			 * very high connection rates) */
1941
+		}
1942
+#endif
1896 1943
 		
1897 1944
 		process_no++;
1898 1945
 		child_rank++;
... ...
@@ -6,6 +6,7 @@ egcs-2.91.66
6 6
 3.0.4
7 7
 gcc (GCC) 3.2
8 8
 gcc-3.2 (GCC) 3.2.3 (Debian)
9
+gcc (GCC) 3.3 20030304 (Apple Computer, Inc. build 1809)
9 10
 gcc (GCC) 3.3.3 (NetBSD nb3 20040520)
10 11
 gcc (GCC) 3.3.5 (propolice)
11 12
 gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-6)
... ...
@@ -13,6 +14,7 @@ gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-13)
13 14
 gcc (GCC) 3.4.2 [FreeBSD] 20040728
14 15
 gcc (GCC) 3.4.3 (csl-sol210-3_4-branch+sol_rpath)
15 16
 gcc-3.4 (GCC) 3.4.4 20041218 (prerelease) (Debian 3.4.3-7)
17
+powerpc-apple-darwin8-gcc-3.5.0 (GCC) 3.5.0 20041026 (Apple Computer, Inc. build 4059)
16 18
 powerpc-apple-darwin8-gcc-4.0.0 (GCC) 4.0.0 20041026 (Apple Computer, Inc. build 4061)
17 19
 gcc-4.0 (GCC) 4.0.2 20050821 (prerelease) (Debian 4.0.1-6)
18 20
 gcc-4.0.1 (GCC) 3.3.6 (Gentoo 3.3.6, ssp-3.3.6-1.0, pie-8.7.8)