Browse code

- merge from testing-0.8.12-r0: - tcp updates (lots) - makefile mips support - tm timer workarround (present also in stable), t_relay_tls changed to t_relay_to_tls - udp_flood sleep & throttle support

Andrei Pelinescu-Onciul authored on 11/11/2003 15:32:36
Showing 12 changed files
... ...
@@ -29,6 +29,7 @@
29 29
 #  2003-09-25  added -pthread into LIBS when compiling on FreeBSD/alpha
30 30
 #              and other FreeBSD arches for which no fast locking assembly
31 31
 #              code exists (sobomax)
32
+#  2003-11-08  mips1 support introduced (andrei)
32 33
 
33 34
 
34 35
 # check if already included/exported
... ...
@@ -43,7 +44,7 @@ export makefile_defs
43 44
 VERSION = 0
44 45
 PATCHLEVEL = 8
45 46
 SUBLEVEL =   12
46
-EXTRAVERSION = dev-22-tcp_aliases
47
+EXTRAVERSION = -dev-23-merged
47 48
 
48 49
 RELEASE=$(VERSION).$(PATCHLEVEL).$(SUBLEVEL)$(EXTRAVERSION)
49 50
 OS = $(shell uname -s | sed -e s/SunOS/solaris/ | tr "[A-Z]" "[a-z]")
... ...
@@ -51,7 +52,7 @@ ARCH = $(shell uname -m |sed -e s/i.86/i386/ -e s/sun4u/sparc64/  \
51 52
 			-e s/armv4l/arm/)
52 53
 
53 54
 # TLS support
54
-TLS ?=
55
+TLS ?= 
55 56
 ifneq ($(TLS),)
56 57
 	RELEASE:=$(RELEASE)-tls
57 58
 endif
... ...
@@ -240,16 +241,16 @@ endif
240 241
 # -DUSE_POSIX_SEM
241 242
 #		uses posix semaphores for locking (faster than sys v)
242 243
 # -DBUSY_WAIT
243
-#		uses busy waiting on the lock
244
+#		uses busy waiting on the lock (FAST_LOCK)
244 245
 # -DADAPTIVE_WAIT
245 246
 #		try busy waiting for a while and if the lock is still held go to
246
-#		force reschedule
247
+#		force reschedule (FAST_LOCK)
247 248
 # -DADAPTIVE_WAIT_LOOPS=number
248 249
 #		number of loops we busy wait, after "number" loops have elapsed we 
249
-#		force a reschedule
250
+#		force a reschedule (FAST_LOCK)
250 251
 # -DNOSMP
251 252
 #		don't use smp compliant locking (faster but won't work on SMP machines)
252
-#		(not yet enabled)
253
+#		(not yet enabled) (FAST_LOCK)
253 254
 # -DNO_PINGTEL_TAG_HACK
254 255
 #		if enabled, To-header-field will be less liberal and will not accept
255 256
 #		'tag=' (tag parameter with equal sign and without value); it is called
... ...
@@ -273,12 +274,11 @@ DEFS+= $(extra_defs) \
273 274
 	 -DCFG_DIR='"$(cfg-target)"'\
274 275
 	 -DPKG_MALLOC \
275 276
 	 -DSHM_MEM  -DSHM_MMAP \
276
-	 -DADAPTIVE_WAIT -DADAPTIVE_WAIT_LOOPS=1024 \
277 277
 	 -DDNS_IP_HACK \
278 278
 	 -DUSE_IPV6 \
279 279
 	 -DUSE_TCP \
280 280
 	 -DDISABLE_NAGLE \
281
-	 -DDBG_QM_MALLOC \
281
+	# -DDBG_QM_MALLOC \
282 282
 	# -DF_MALLOC \
283 283
 	# -DDBG_F_MALLOC \
284 284
 	# -DDBG_QM_MALLOC \
... ...
@@ -386,8 +386,13 @@ ifeq ($(ARCH), ppc)
386 386
 	use_fast_lock=yes
387 387
 endif
388 388
 
389
+ifeq ($(ARCH), mips)
390
+# mips1 arch. (e.g. R3000) - no hardware locking support
391
+	use_fast_lock=no
392
+endif
393
+
389 394
 ifeq ($(use_fast_lock), yes)
390
-	DEFS+= -DFAST_LOCK
395
+	DEFS+= -DFAST_LOCK -DADAPTIVE_WAIT -DADAPTIVE_WAIT_LOOPS=1024 
391 396
 	found_lock_method=yes
392 397
 endif
393 398
 
... ...
@@ -519,15 +524,46 @@ else
519 524
 				#really old version
520 525
 $(warning			You are using an old and unsupported gcc \
521 526
 					 version ($(CC_SHORTVER)), compile at your own risk!)
522
-
527
+	
523 528
 endif			# CC_SHORTVER, 2.9x
524 529
 endif			# CC_SHORTVER, 3.0
530
+	
531
+else		# CC_NAME, gcc
532
+				#other compilers
533
+$(error 			Unsupported compiler ($(CC):$(CC_NAME)), try gcc)
534
+endif		#CC_NAME, gcc
535
+endif	#ARCH, arm 
525 536
 
537
+	#if  mips (R3000)
538
+ifeq	($(ARCH), mips)
539
+		# if gcc 
540
+ifeq		($(CC_NAME), gcc)
541
+				#common stuff
542
+				CFLAGS=-O9 -funroll-loops  -Wcast-align $(PROFILE) \
543
+					-Wall   \
544
+			#if gcc 3.0
545
+ifeq			($(CC_SHORTVER), 3.0)
546
+					CFLAGS+= -mcpu=r3000
547
+							#-mcpu=athlon
548
+else
549
+ifeq			($(CC_SHORTVER), 2.9x) #older gcc version (2.9[1-5])
550
+$(warning 			Old gcc detected ($(CC_SHORTVER)), use  gcc 3.0.x \
551
+					for better results)
552
+					
553
+					CFLAGS+=-mcpu=r3000
554
+else
555
+				#really old version
556
+$(warning			You are using an old and unsupported gcc \
557
+					 version ($(CC_SHORTVER)), compile at your own risk!)
558
+	
559
+endif			# CC_SHORTVER, 2.9x
560
+endif			# CC_SHORTVER, 3.0
561
+	
526 562
 else		# CC_NAME, gcc
527 563
 				#other compilers
528 564
 $(error 			Unsupported compiler ($(CC):$(CC_NAME)), try gcc)
529 565
 endif		#CC_NAME, gcc
530
-endif	#ARCH, i386
566
+endif	#ARCH, mips
531 567
 
532 568
 
533 569
 CFLAGS+= $(CC_EXTRA_OPTS)
... ...
@@ -6,9 +6,9 @@
6 6
 
7 7
 # ----------- global configuration parameters ------------------------
8 8
 
9
-debug=3         # debug level (cmd line: -dddddddddd)
10
-fork=yes
11
-log_stderror=no	# (cmd line: -E)
9
+#debug=3         # debug level (cmd line: -dddddddddd)
10
+#fork=yes
11
+#log_stderror=no	# (cmd line: -E)
12 12
 
13 13
 /* Uncomment these lines to enter debugging mode 
14 14
 fork=no
... ...
@@ -18,8 +18,8 @@ log_stderror=yes
18 18
 check_via=no	# (cmd. line: -v)
19 19
 dns=no           # (cmd. line: -r)
20 20
 rev_dns=no      # (cmd. line: -R)
21
-port=5060
22
-children=4
21
+#port=5060
22
+#children=4
23 23
 fifo="/tmp/ser_fifo"
24 24
 
25 25
 # ------------------ module loading ----------------------------------
... ...
@@ -349,8 +349,8 @@ static int fifo_check(int fd, char* fname)
349 349
 	 */
350 350
 	if ((lst.st_dev!=fst.st_dev)||(lst.st_ino!=fst.st_ino)){
351 351
 		LOG(L_ERR, "ERROR: security: fifo_check: inode/dev number differ"
352
-				": %ld %ld (%s)\n",
353
-				 fst.st_ino, lst.st_ino, fname);
352
+				": %d %d (%s)\n",
353
+				 (int)fst.st_ino, (int)lst.st_ino, fname);
354 354
 		return -1;
355 355
 	}
356 356
 	/* success */
... ...
@@ -326,6 +326,11 @@ inline static void final_response_handler( void *attr)
326 326
 	struct cell *t;
327 327
 
328 328
 	r_buf = (struct retr_buf*)attr;
329
+	if (r_buf==0){
330
+		/* or BUG?, ignoring it for now */
331
+		LOG(L_CRIT, "ERROR: final_response_handler(0) called\n");
332
+		return;
333
+	}
329 334
 	t=r_buf->my_T;
330 335
 
331 336
 #	ifdef EXTRA_DEBUG
... ...
@@ -52,7 +52,7 @@
52 52
 #define T_RELAY_TO           "t_relay_to"
53 53
 #define T_RELAY_TO_UDP       "t_relay_to_udp"
54 54
 #define T_RELAY_TO_TCP       "t_relay_to_tcp"
55
-#define T_RELAY_TO_TLS       "t_relay_tls"
55
+#define T_RELAY_TO_TLS       "t_relay_to_tls"
56 56
 #define T_RELAY              "t_relay"
57 57
 #define T_REPLY              "t_reply"
58 58
 #define T_REPLY_WB           "t_reply_with_body"
... ...
@@ -29,6 +29,8 @@
29 29
   * --------
30 30
   *  2002-11-29  created by andrei
31 31
   *  2003-02-20  added solaris support (! HAVE_MSGHDR_MSG_CONTROL) (andrei)
32
+  *  2003-11-03  added send_all, recv_all  and updated send/get_fd
33
+  *               to handle signals  (andrei)
32 34
   */
33 35
 
34 36
 #ifdef USE_TCP
... ...
@@ -37,10 +39,55 @@
37 39
 #include <sys/socket.h>
38 40
 #include <sys/uio.h>
39 41
 #include <stdlib.h> /* for NULL definition on openbsd */
42
+#include <errno.h>
43
+#include <string.h>
40 44
 
41 45
 #include "dprint.h"
42 46
 
43 47
 
48
+
49
+/* receive all the data or returns error (handles EINTR etc.)
50
+ * returns: bytes read or error (<0)
51
+ * can return < data_len if EOF */
52
+int recv_all(int socket, void* data, int data_len)
53
+{
54
+	int b_read;
55
+	int n;
56
+	
57
+	b_read=0;
58
+	do{
59
+		n=recv(socket, data+b_read, data_len-b_read, MSG_WAITALL);
60
+		if (n<0){
61
+			/* error */
62
+			if (errno==EINTR) continue; /* signal, try again */
63
+			LOG(L_CRIT, "ERROR: recv_all: recv on %d failed: %s\n",
64
+					socket, strerror(errno));
65
+			return n;
66
+		}
67
+		b_read+=n;
68
+	}while( (b_read!=data_len) && (n));
69
+	return b_read;
70
+}
71
+
72
+
73
+/* sends all data (takes care of signals) (assumes blocking fd)
74
+ * returns number of bytes sent or < 0 for an error */
75
+int send_all(int socket, void* data, int data_len)
76
+{
77
+	int n;
78
+	
79
+again:
80
+	n=send(socket, data, data_len, 0);
81
+	if (n<0){
82
+			/* error */
83
+		if (errno==EINTR) goto again; /* signal, try again */
84
+		LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
85
+					socket, strerror(errno));
86
+	}
87
+	return n;
88
+}
89
+
90
+
44 91
 /* at least 1 byte must be sent! */
45 92
 int send_fd(int unix_socket, void* data, int data_len, int fd)
46 93
 {
... ...
@@ -76,8 +123,13 @@ int send_fd(int unix_socket, void* data, int data_len, int fd)
76 123
 	msg.msg_iov=iov;
77 124
 	msg.msg_iovlen=1;
78 125
 	
79
-	
126
+again:
80 127
 	ret=sendmsg(unix_socket, &msg, 0);
128
+	if (ret<0){
129
+		if (errno==EINTR) goto again;
130
+		LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
131
+				unix_socket, strerror(errno));
132
+	}
81 133
 	
82 134
 	return ret;
83 135
 }
... ...
@@ -90,6 +142,7 @@ int receive_fd(int unix_socket, void* data, int data_len, int* fd)
90 142
 	struct iovec iov[1];
91 143
 	int new_fd;
92 144
 	int ret;
145
+	int n;
93 146
 #ifdef HAVE_MSGHDR_MSG_CONTROL
94 147
 	struct cmsghdr* cmsg;
95 148
 	union{
... ...
@@ -112,26 +165,47 @@ int receive_fd(int unix_socket, void* data, int data_len, int* fd)
112 165
 	msg.msg_iov=iov;
113 166
 	msg.msg_iovlen=1;
114 167
 	
115
-	ret=recvmsg(unix_socket, &msg, 0);
116
-	if (ret<=0) goto error;
168
+again:
169
+	ret=recvmsg(unix_socket, &msg, MSG_WAITALL);
170
+	if (ret<0){
171
+		if (errno==EINTR) goto again;
172
+		LOG(L_CRIT, "ERROR: receive_fd: recvmsg on %d failed: %s\n",
173
+				unix_socket, strerror(errno));
174
+		goto error;
175
+	}
176
+	if (ret==0){
177
+		/* EOF */
178
+		LOG(L_CRIT, "ERROR: receive_fd: EOF on %d\n", unix_socket);
179
+		goto error;
180
+	}
181
+	if (ret<data_len){
182
+		LOG(L_WARN, "WARNING: receive_fd: too few bytes read (%d from %d)"
183
+				    "trying to fix...\n", ret, data_len);
184
+		n=recv_all(unix_socket, (char*)data+ret, data_len-ret);
185
+		if (n>=0) ret+=n;
186
+		else{
187
+			ret=n;
188
+			goto error;
189
+		}
190
+	}
117 191
 	
118 192
 #ifdef HAVE_MSGHDR_MSG_CONTROL
119 193
 	cmsg=CMSG_FIRSTHDR(&msg);
120 194
 	if ((cmsg!=0) && (cmsg->cmsg_len==CMSG_LEN(sizeof(new_fd)))){
121 195
 		if (cmsg->cmsg_type!= SCM_RIGHTS){
122
-			LOG(L_ERR, "receive_fd: msg control type != SCM_RIGHTS\n");
196
+			LOG(L_ERR, "ERROR: receive_fd: msg control type != SCM_RIGHTS\n");
123 197
 			ret=-1;
124 198
 			goto error;
125 199
 		}
126 200
 		if (cmsg->cmsg_level!= SOL_SOCKET){
127
-			LOG(L_ERR, "receive_fd: msg level != SOL_SOCKET\n");
201
+			LOG(L_ERR, "ERROR: receive_fd: msg level != SOL_SOCKET\n");
128 202
 			ret=-1;
129 203
 			goto error;
130 204
 		}
131 205
 		*fd=*((int*) CMSG_DATA(cmsg));
132 206
 	}else{
133
-		LOG(L_ERR, "receive_fd: no descriptor passed, cmsg=%p, len=%d\n",
134
-				cmsg, cmsg->cmsg_len);
207
+		LOG(L_ERR, "ERROR: receive_fd: no descriptor passed, cmsg=%p,"
208
+				"len=%d\n", cmsg, cmsg->cmsg_len);
135 209
 		*fd=-1;
136 210
 		/* it's not really an error */
137 211
 	}
... ...
@@ -139,8 +213,8 @@ int receive_fd(int unix_socket, void* data, int data_len, int* fd)
139 213
 	if (msg.msg_accrightslen==sizeof(int)){
140 214
 		*fd=new_fd;
141 215
 	}else{
142
-		LOG(L_ERR, "receive_fd: no descriptor passed, accrightslen=%d\n",
143
-				msg.msg_accrightslen);
216
+		LOG(L_ERR, "ERROR: receive_fd: no descriptor passed,"
217
+				" accrightslen=%d\n", msg.msg_accrightslen);
144 218
 		*fd=-1;
145 219
 	}
146 220
 #endif
... ...
@@ -148,5 +222,4 @@ int receive_fd(int unix_socket, void* data, int data_len, int* fd)
148 222
 error:
149 223
 	return ret;
150 224
 }
151
-
152 225
 #endif
... ...
@@ -32,6 +32,8 @@
32 32
 int send_fd(int unix_socket, void* data, int data_len, int fd);
33 33
 int receive_fd(int unix_socket, void* data, int data_len, int* fd);
34 34
 
35
+int recv_all(int socket, void* data, int data_len);
36
+int send_all(int socket, void* data, int data_len);
35 37
 
36 38
 
37 39
 #endif
... ...
@@ -44,8 +44,8 @@
44 44
 #define TCP_CON_MAX_ALIASES 4 /* maximum number of port aliases */
45 45
 
46 46
 #define TCP_BUF_SIZE 65535
47
-#define TCP_CON_TIMEOUT 60 /* in  seconds */
48
-#define TCP_CON_SEND_TIMEOUT 30 /* timeout after a send */
47
+#define TCP_CON_TIMEOUT 120 /* in  seconds */
48
+#define TCP_CON_SEND_TIMEOUT 120 /* timeout after a send */
49 49
 #define TCP_CHILD_TIMEOUT 5 /* after 5 seconds, the child "returns" 
50 50
 							 the connection to the tcp master process */
51 51
 #define TCP_MAIN_SELECT_TIMEOUT 5 /* how often "tcp main" checks for timeout*/
... ...
@@ -73,6 +73,8 @@ enum tcp_conn_states { S_CONN_ERROR=-2, S_CONN_BAD=-1, S_CONN_OK=0,
73 73
 /* fd communication commands */
74 74
 enum conn_cmds { CONN_DESTROY=-3, CONN_ERROR=-2, CONN_EOF=-1, CONN_RELEASE, 
75 75
 					CONN_GET_FD, CONN_NEW };
76
+/* CONN_RELEASE, EOF, ERROR, DESTROY can be used by "reader" processes
77
+ * CONN_GET_FD, NEW, ERROR only by writers */
76 78
 
77 79
 struct tcp_req{
78 80
 	struct tcp_req* next;
... ...
@@ -44,6 +44,10 @@
44 44
  *  2003-07-09  tls_close called before closing the tcp connection (andrei)
45 45
  *  2003-10-24  converted to the new socket_info lists (andrei)
46 46
  *  2003-10-27  tcp port aliases support added (andrei)
47
+ *  2003-11-04  always lock before manipulating refcnt; sendchild
48
+ *              does not inc refcnt by itself anymore (andrei)
49
+ *  2003-11-07  different unix sockets are used for fd passing
50
+ *              to/from readers/writers (andrei)
47 51
  */
48 52
 
49 53
 
... ...
@@ -100,7 +104,7 @@
100 104
 struct tcp_child{
101 105
 	pid_t pid;
102 106
 	int proc_no; /* ser proc_no, for debugging */
103
-	int unix_sock; /* unix sock fd, copied from pt*/
107
+	int unix_sock; /* unix "read child" sock fd */
104 108
 	int busy;
105 109
 	int n_reqs; /* number of requests serviced so far */
106 110
 };
... ...
@@ -242,7 +246,13 @@ struct tcp_connection* tcpconn_connect(union sockaddr_union* server, int type)
242 246
 				strerror(errno), errno);
243 247
 		si=0; /* try to go on */
244 248
 	}
245
-	si=find_tcp_si(&my_name);
249
+#ifdef USE_TLS
250
+	if (type==PROTO_TLS)
251
+		si=find_tls_si(&my_name);
252
+	else
253
+#endif
254
+		si=find_tcp_si(&my_name);
255
+
246 256
 	if (si==0){
247 257
 		LOG(L_ERR, "ERROR: tcp_connect: could not find coresponding"
248 258
 				" listening socket, using default...\n");
... ...
@@ -443,9 +453,20 @@ error_sec:
443 453
 
444 454
 
445 455
 
456
+void tcpconn_ref(struct tcp_connection* c)
457
+{
458
+	TCPCONN_LOCK;
459
+	c->refcnt++; /* FIXME: atomic_dec */
460
+	TCPCONN_UNLOCK;
461
+}
462
+
463
+
464
+
446 465
 void tcpconn_put(struct tcp_connection* c)
447 466
 {
467
+	TCPCONN_LOCK;
448 468
 	c->refcnt--; /* FIXME: atomic_dec */
469
+	TCPCONN_UNLOCK;
449 470
 }
450 471
 
451 472
 
... ...
@@ -455,6 +476,7 @@ int tcp_send(int type, char* buf, unsigned len, union sockaddr_union* to,
455 476
 				int id)
456 477
 {
457 478
 	struct tcp_connection *c;
479
+	struct tcp_connection *tmp;
458 480
 	struct ip_addr ip;
459 481
 	int port;
460 482
 	int fd;
... ...
@@ -494,22 +516,25 @@ no_id:
494 516
 				LOG(L_ERR, "ERROR: tcp_send: connect failed\n");
495 517
 				return -1;
496 518
 			}
497
-			c->refcnt++;
519
+			c->refcnt++; /* safe to do it w/o locking, it's not yet
520
+							available to the rest of the world */
498 521
 			fd=c->s;
499 522
 			
500 523
 			/* send the new tcpconn to "tcp main" */
501 524
 			response[0]=(long)c;
502 525
 			response[1]=CONN_NEW;
503
-			n=write(unix_tcp_sock, response, sizeof(response));
504
-			if (n<0){
526
+			n=send_all(unix_tcp_sock, response, sizeof(response));
527
+			if (n<=0){
505 528
 				LOG(L_ERR, "BUG: tcp_send: failed write: %s (%d)\n",
506 529
 						strerror(errno), errno);
530
+				n=-1;
507 531
 				goto end;
508 532
 			}	
509 533
 			n=send_fd(unix_tcp_sock, &c, sizeof(c), c->s);
510
-			if (n<0){
534
+			if (n<=0){
511 535
 				LOG(L_ERR, "BUG: tcp_send: failed send_fd: %s (%d)\n",
512 536
 						strerror(errno), errno);
537
+				n=-1;
513 538
 				goto end;
514 539
 			}
515 540
 			goto send_it;
... ...
@@ -517,21 +542,34 @@ no_id:
517 542
 get_fd:
518 543
 			/* todo: see if this is not the same process holding
519 544
 			 *  c  and if so send directly on c->fd */
520
-			DBG("tcp_send: tcp connection found, acquiring fd\n");
545
+			DBG("tcp_send: tcp connection found (%p), acquiring fd\n", c);
521 546
 			/* get the fd */
522 547
 			response[0]=(long)c;
523 548
 			response[1]=CONN_GET_FD;
524
-			n=write(unix_tcp_sock, response, sizeof(response));
525
-			if (n<0){
549
+			n=send_all(unix_tcp_sock, response, sizeof(response));
550
+			if (n<=0){
526 551
 				LOG(L_ERR, "BUG: tcp_send: failed to get fd(write):%s (%d)\n",
527 552
 						strerror(errno), errno);
553
+				n=-1;
528 554
 				goto release_c;
529 555
 			}
530 556
 			DBG("tcp_send, c= %p, n=%d\n", c, n);
557
+			tmp=c;
531 558
 			n=receive_fd(unix_tcp_sock, &c, sizeof(c), &fd);
532
-			if (n<0){
559
+			if (n<=0){
533 560
 				LOG(L_ERR, "BUG: tcp_send: failed to get fd(receive_fd):"
534 561
 							" %s (%d)\n", strerror(errno), errno);
562
+				n=-1;
563
+				goto release_c;
564
+			}
565
+			if (c!=tmp){
566
+				LOG(L_CRIT, "BUG: tcp_send: get_fd: got different connection:"
567
+						"  %p (id= %d, refcnt=%d state=%d != "
568
+						"  %p (id= %d, refcnt=%d state=%d (n=%d)\n",
569
+						  c,   c->id,   c->refcnt,   c->state,
570
+						  tmp, tmp->id, tmp->refcnt, tmp->state, n
571
+				   );
572
+				n=-1; /* fail */
535 573
 				goto release_c;
536 574
 			}
537 575
 			DBG("tcp_send: after receive_fd: c= %p n=%d fd=%d\n",c, n, fd);
... ...
@@ -567,11 +605,12 @@ send_it:
567 605
 		/* tell "main" it should drop this (optional it will t/o anyway?)*/
568 606
 		response[0]=(long)c;
569 607
 		response[1]=CONN_ERROR;
570
-		n=write(unix_tcp_sock, response, sizeof(response));
571
-		/* CONN_ERROR wil auto-dec refcnt => we must not call tcpconn_put !!*/
572
-		if (n<0){
608
+		n=send_all(unix_tcp_sock, response, sizeof(response));
609
+		/* CONN_ERROR will auto-dec refcnt => we must not call tcpconn_put !!*/
610
+		if (n<=0){
573 611
 			LOG(L_ERR, "BUG: tcp_send: error return failed (write):%s (%d)\n",
574 612
 					strerror(errno), errno);
613
+			n=-1;
575 614
 		}
576 615
 		close(fd);
577 616
 		return n; /* error return, no tcpconn_put */
... ...
@@ -737,19 +776,21 @@ static int send2child(struct tcp_connection* tcpconn)
737 776
 	
738 777
 	tcp_children[idx].busy++;
739 778
 	tcp_children[idx].n_reqs++;
740
-	tcpconn->refcnt++;
741 779
 	if (min_busy){
742
-		LOG(L_WARN, "WARNING: send2child:no free tcp receiver, "
780
+		LOG(L_WARN, "WARNING: send2child: no free tcp receiver, "
743 781
 				" connection passed to the least busy one (%d)\n",
744 782
 				min_busy);
745 783
 	}
746 784
 	DBG("send2child: to tcp child %d %d(%d), %p\n", idx, 
747 785
 					tcp_children[idx].proc_no,
748 786
 					tcp_children[idx].pid, tcpconn);
749
-	send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
750
-			tcpconn->s);
787
+	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
788
+			tcpconn->s)<=0){
789
+		LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
790
+		return -1;
791
+	}
751 792
 	
752
-	return 0; /* just to fix a warning*/
793
+	return 0;
753 794
 }
754 795
 
755 796
 
... ...
@@ -776,6 +817,8 @@ static inline void handle_new_connect(struct socket_info* si,
776 817
 		/* add socket to list */
777 818
 		tcpconn=tcpconn_new(new_sock, &su, si, si->proto, S_CONN_ACCEPT);
778 819
 		if (tcpconn){
820
+			tcpconn->refcnt++; /* safe, not yet available to the
821
+								  outside world */
779 822
 			tcpconn_add(tcpconn);
780 823
 			DBG("tcp_main_loop: new connection: %p %d\n",
781 824
 				tcpconn, tcpconn->s);
... ...
@@ -784,6 +827,7 @@ static inline void handle_new_connect(struct socket_info* si,
784 827
 				LOG(L_ERR,"ERROR: tcp_main_loop: no children "
785 828
 						"available\n");
786 829
 				TCPCONN_LOCK;
830
+				tcpconn->refcnt--;
787 831
 				if (tcpconn->refcnt==0){
788 832
 					close(tcpconn->s);
789 833
 					_tcpconn_rm(tcpconn);
... ...
@@ -795,6 +839,34 @@ static inline void handle_new_connect(struct socket_info* si,
795 839
 }
796 840
 
797 841
 
842
+/* used internally by tcp_main_loop() */
843
+static void tcpconn_destroy(struct tcp_connection* tcpconn)
844
+{
845
+	int fd;
846
+
847
+	TCPCONN_LOCK; /*avoid races w/ tcp_send*/
848
+	tcpconn->refcnt--;
849
+	if (tcpconn->refcnt==0){ 
850
+		DBG("tcp_main_loop: destroying connection\n");
851
+		fd=tcpconn->s;
852
+#ifdef USE_TLS
853
+		/*FIXME: lock ->writelock ? */
854
+		if (tcpconn->type==PROTO_TLS)
855
+			tls_close(tcpconn, fd);
856
+#endif
857
+		_tcpconn_rm(tcpconn);
858
+		close(fd);
859
+	}else{
860
+		/* force timeout */
861
+		tcpconn->timeout=0;
862
+		tcpconn->state=S_CONN_BAD;
863
+		DBG("tcp_main_loop: delaying ...\n");
864
+		
865
+	}
866
+	TCPCONN_UNLOCK;
867
+}
868
+
869
+
798 870
 void tcp_main_loop()
799 871
 {
800 872
 	int r;
... ...
@@ -843,6 +915,14 @@ void tcp_main_loop()
843 915
 			if (pt[r].unix_sock>maxfd) maxfd=pt[r].unix_sock;
844 916
 		}
845 917
 	}
918
+	for (r=0; r<tcp_children_no; r++){
919
+		if (tcp_children[r].unix_sock>0){ /* we can't have 0, 
920
+											 we never close it!*/
921
+			FD_SET(tcp_children[r].unix_sock, &master_set);
922
+			if (tcp_children[r].unix_sock>maxfd)
923
+				maxfd=tcp_children[r].unix_sock;
924
+		}
925
+	}
846 926
 	
847 927
 	
848 928
 	/* main loop*/
... ...
@@ -872,6 +952,7 @@ void tcp_main_loop()
872 952
 		for (h=0; h<TCP_ID_HASH_SIZE; h++){
873 953
 			for(tcpconn=tcpconn_id_hash[h]; tcpconn && n; 
874 954
 					tcpconn=tcpconn->id_next){
955
+				/* FIXME: is refcnt==0 really necessary? */
875 956
 				if ((tcpconn->refcnt==0)&&(FD_ISSET(tcpconn->s, &sel_set))){
876 957
 					/* new data available */
877 958
 					n--;
... ...
@@ -879,10 +960,12 @@ void tcp_main_loop()
879 960
 					DBG("tcp_main_loop: data available on %p [h:%d] %d\n",
880 961
 							tcpconn, h, tcpconn->s);
881 962
 					FD_CLR(tcpconn->s, &master_set);
963
+					tcpconn_ref(tcpconn); /* refcnt ++ */
882 964
 					if (send2child(tcpconn)<0){
883 965
 						LOG(L_ERR,"ERROR: tcp_main_loop: no "
884 966
 									"children available\n");
885 967
 						TCPCONN_LOCK;
968
+						tcpconn->refcnt--;
886 969
 						if (tcpconn->refcnt==0){
887 970
 							fd=tcpconn->s;
888 971
 							_tcpconn_rm(tcpconn);
... ...
@@ -894,109 +977,138 @@ void tcp_main_loop()
894 977
 			}
895 978
 		}
896 979
 		/* check unix sockets & listen | destroy connections */
897
-		/* start from 1, the "main" process does not transmit anything*/
898
-		for (r=1; r<process_no && n; r++){
899
-			if ( (pt[r].unix_sock>0) && FD_ISSET(pt[r].unix_sock, &sel_set)){
980
+		/* tcp_children readers first */
981
+		for (r=0; r<tcp_children_no && n; r++){
982
+			if ( (tcp_children[r].unix_sock>0) && 
983
+					FD_ISSET(tcp_children[r].unix_sock, &sel_set)){
900 984
 				/* (we can't have a fd==0, 0 is never closed )*/
901 985
 				n--;
902
-				/* errno==EINTR !!! TODO*/
903
-read_again:
904
-				bytes=read(pt[r].unix_sock, response, sizeof(response));
986
+				/* read until sizeof(response)
987
+				 * (this is a SOCK_STREAM so read is not atomic */
988
+				bytes=recv_all(tcp_children[r].unix_sock, response,
989
+								sizeof(response));
905 990
 				if (bytes==0){
906 991
 					/* EOF -> bad, child has died */
907
-					LOG(L_CRIT, "BUG: tcp_main_loop: dead child %d\n", r);
992
+					LOG(L_CRIT, "BUG: tcp_main_loop: dead tcp child %d\n", r);
908 993
 					/* don't listen on it any more */
909
-					FD_CLR(pt[r].unix_sock, &master_set);
994
+					FD_CLR(tcp_children[r].unix_sock, &master_set);
910 995
 					/*exit(-1);*/
911
-					continue;
996
+					continue; /* skip this and try the next one */
912 997
 				}else if (bytes<0){
913
-					if (errno==EINTR) goto read_again;
914
-					else{
915
-						LOG(L_CRIT, "ERROR: tcp_main_loop: read from child: "
916
-								" %s\n", strerror(errno));
917
-						/* try to continue ? */
918
-						continue;
919
-					}
998
+					LOG(L_CRIT, "ERROR: tcp_main_loop: read from tcp child %d "
999
+							"%s\n", r, strerror(errno));
1000
+					/* try to ignore ? */
1001
+					continue; /* skip this and try the next one */
920 1002
 				}
921 1003
 					
922
-				DBG("tcp_main_loop: read response= %lx, %ld from %d (%d)\n",
923
-						response[0], response[1], r, pt[r].pid);
1004
+				DBG("tcp_main_loop: reader response= %lx, %ld from %d \n",
1005
+						response[0], response[1], r);
924 1006
 				cmd=response[1];
1007
+				tcpconn=(struct tcp_connection*)response[0];
925 1008
 				switch(cmd){
926 1009
 					case CONN_RELEASE:
927
-						if (pt[r].idx>=0){
928
-							tcp_children[pt[r].idx].busy--;
929
-						}else{
930
-							LOG(L_CRIT, "BUG: tcp_main_loop: CONN_RELEASE\n");
931
-						}
932
-						tcpconn=(struct tcp_connection*)response[0];
1010
+						tcp_children[r].busy--;
933 1011
 						if (tcpconn){
934
-								if (tcpconn->state==S_CONN_BAD) 
935
-									goto tcpconn_destroy;
1012
+								if (tcpconn->state==S_CONN_BAD){ 
1013
+									tcpconn_destroy(tcpconn);
1014
+									break;
1015
+								}
936 1016
 								FD_SET(tcpconn->s, &master_set);
937 1017
 								if (maxfd<tcpconn->s) maxfd=tcpconn->s;
938 1018
 								/* update the timeout*/
939 1019
 								tcpconn->timeout=get_ticks()+TCP_CON_TIMEOUT;
940 1020
 								tcpconn_put(tcpconn);
941
-								DBG("tcp_main_loop: %p refcnt= %d\n", 
942
-									tcpconn, tcpconn->refcnt);
1021
+								DBG("tcp_main_loop: CONN_RELEASE  %p"
1022
+										" refcnt= %d\n", 
1023
+										tcpconn, tcpconn->refcnt);
943 1024
 						}
944 1025
 						break;
945 1026
 					case CONN_ERROR:
946 1027
 					case CONN_DESTROY:
947 1028
 					case CONN_EOF:
948 1029
 						/* WARNING: this will auto-dec. refcnt! */
949
-						if (pt[r].idx>=0){
950
-							tcp_children[pt[r].idx].busy--;
951
-						}else{
952
-							LOG(L_CRIT, "BUG: tcp_main_loop: CONN_RELEASE\n");
1030
+						tcp_children[pt[r].idx].busy--;
1031
+						if (tcpconn){
1032
+							if (tcpconn->s!=-1)
1033
+								FD_CLR(tcpconn->s, &master_set);
1034
+							tcpconn_destroy(tcpconn);
953 1035
 						}
954
-						tcpconn=(struct tcp_connection*)response[0];
1036
+						break;
1037
+					default:
1038
+							LOG(L_CRIT, "BUG: tcp_main_loop:  unknown cmd %d"
1039
+										" from tcp reader %d\n",
1040
+									cmd, r);
1041
+				}
1042
+			}
1043
+		}
1044
+		/* check "send" unix sockets & listen | destroy connections */
1045
+		/* start from 1, the "main" process does not transmit anything*/
1046
+		for (r=1; r<process_no && n; r++){
1047
+			if ( (pt[r].unix_sock>0) && FD_ISSET(pt[r].unix_sock, &sel_set)){
1048
+				/* (we can't have a fd==0, 0 is never closed )*/
1049
+				n--;
1050
+				/* read until sizeof(response)
1051
+				 * (this is a SOCK_STREAM so read is not atomic */
1052
+				bytes=recv_all(pt[r].unix_sock, response, sizeof(response));
1053
+				if (bytes==0){
1054
+					/* EOF -> bad, child has died */
1055
+					LOG(L_CRIT, "BUG: tcp_main_loop: dead child %d\n", r);
1056
+					/* don't listen on it any more */
1057
+					FD_CLR(pt[r].unix_sock, &master_set);
1058
+					/*exit(-1);*/
1059
+					continue; /* skip this and try the next one */
1060
+				}else if (bytes<0){
1061
+					LOG(L_CRIT, "ERROR: tcp_main_loop: read from child:  %s\n",
1062
+							strerror(errno));
1063
+					/* try to ignore ? */
1064
+					continue; /* skip this and try the next one */
1065
+				}
1066
+					
1067
+				DBG("tcp_main_loop: read response= %lx, %ld from %d (%d)\n",
1068
+						response[0], response[1], r, pt[r].pid);
1069
+				cmd=response[1];
1070
+				tcpconn=(struct tcp_connection*)response[0];
1071
+				switch(cmd){
1072
+					case CONN_ERROR:
955 1073
 						if (tcpconn){
956 1074
 							if (tcpconn->s!=-1)
957 1075
 								FD_CLR(tcpconn->s, &master_set);
958
-		tcpconn_destroy:
959
-							TCPCONN_LOCK; /*avoid races w/ tcp_send*/
960
-							tcpconn->refcnt--;
961
-							if (tcpconn->refcnt==0){ 
962
-								DBG("tcp_main_loop: destroying connection\n");
963
-								fd=tcpconn->s;
964
-#ifdef USE_TLS
965
-								if (tcpconn->type==PROTO_TLS)
966
-									tls_close(tcpconn, fd);
967
-#endif
968
-								_tcpconn_rm(tcpconn);
969
-								close(fd);
970
-							}else{
971
-								/* force timeout */
972
-								tcpconn->timeout=0;
973
-								tcpconn->state=S_CONN_BAD;
974
-								DBG("tcp_main_loop: delaying ...\n");
975
-								
976
-							}
977
-							TCPCONN_UNLOCK;
1076
+							tcpconn_destroy(tcpconn);
978 1077
 						}
979 1078
 						break;
980 1079
 					case CONN_GET_FD:
981 1080
 						/* send the requested FD  */
982
-						tcpconn=(struct tcp_connection*)response[0];
983 1081
 						/* WARNING: take care of setting refcnt properly to
984 1082
 						 * avoid race condition */
985 1083
 						if (tcpconn){
986
-							send_fd(pt[r].unix_sock, &tcpconn,
987
-									sizeof(tcpconn), tcpconn->s);
1084
+							if (send_fd(pt[r].unix_sock, &tcpconn,
1085
+										sizeof(tcpconn), tcpconn->s)<=0){
1086
+								LOG(L_ERR, "ERROR: tcp_main_loop:"
1087
+										"send_fd failed\n");
1088
+							}
988 1089
 						}else{
989 1090
 							LOG(L_CRIT, "BUG: tcp_main_loop: null pointer\n");
990 1091
 						}
991 1092
 						break;
992 1093
 					case CONN_NEW:
993 1094
 						/* update the fd in the requested tcpconn*/
994
-						tcpconn=(struct tcp_connection*)response[0];
995 1095
 						/* WARNING: take care of setting refcnt properly to
996 1096
 						 * avoid race condition */
997 1097
 						if (tcpconn){
998
-							receive_fd(pt[r].unix_sock, &tcpconn,
999
-										sizeof(tcpconn), &tcpconn->s);
1098
+							bytes=receive_fd(pt[r].unix_sock, &tcpconn,
1099
+									sizeof(tcpconn), &tcpconn->s);
1100
+								if (bytes<sizeof(tcpconn)){
1101
+									if (bytes<0){
1102
+										LOG(L_CRIT, "BUG: tcp_main_loop:"
1103
+												" CONN_NEW: receive_fd "
1104
+												"failed\n");
1105
+									}else{
1106
+										LOG(L_CRIT, "BUG: tcp_main_loop:"
1107
+												" CONN_NEW: to few bytes "
1108
+												"received (%d)\n", bytes );
1109
+									}
1110
+									break; /* try to ignore */
1111
+								}
1000 1112
 							/* add tcpconn to the list*/
1001 1113
 							tcpconn_add(tcpconn);
1002 1114
 							FD_SET(tcpconn->s, &master_set);
... ...
@@ -1012,7 +1124,7 @@ read_again:
1012 1124
 									cmd);
1013 1125
 				}
1014 1126
 			}
1015
-		}
1127
+		} /* for */
1016 1128
 		
1017 1129
 		/* remove old connections */
1018 1130
 		tcpconn_timeout(&master_set);
... ...
@@ -1114,6 +1226,7 @@ int tcp_init_children()
1114 1226
 {
1115 1227
 	int r;
1116 1228
 	int sockfd[2];
1229
+	int reader_fd[2]; /* for comm. with the tcp children read  */
1117 1230
 	pid_t pid;
1118 1231
 	
1119 1232
 	
... ...
@@ -1127,6 +1240,11 @@ int tcp_init_children()
1127 1240
 					strerror(errno));
1128 1241
 			goto error;
1129 1242
 		}
1243
+		if (socketpair(AF_UNIX, SOCK_STREAM, 0, reader_fd)<0){
1244
+			LOG(L_ERR, "ERROR: tcp_main: socketpair failed: %s\n",
1245
+					strerror(errno));
1246
+			goto error;
1247
+		}
1130 1248
 		
1131 1249
 		process_no++;
1132 1250
 		pid=fork();
... ...
@@ -1137,11 +1255,12 @@ int tcp_init_children()
1137 1255
 		}else if (pid>0){
1138 1256
 			/* parent */
1139 1257
 			close(sockfd[1]);
1258
+			close(reader_fd[1]);
1140 1259
 			tcp_children[r].pid=pid;
1141 1260
 			tcp_children[r].proc_no=process_no;
1142 1261
 			tcp_children[r].busy=0;
1143 1262
 			tcp_children[r].n_reqs=0;
1144
-			tcp_children[r].unix_sock=sockfd[0];
1263
+			tcp_children[r].unix_sock=reader_fd[0];
1145 1264
 			pt[process_no].pid=pid;
1146 1265
 			pt[process_no].unix_sock=sockfd[0];
1147 1266
 			pt[process_no].idx=r;
... ...
@@ -1156,7 +1275,7 @@ int tcp_init_children()
1156 1275
 				LOG(L_ERR, "init_children failed\n");
1157 1276
 				goto error;
1158 1277
 			}
1159
-			tcp_receive_loop(sockfd[1]);
1278
+			tcp_receive_loop(reader_fd[1]);
1160 1279
 		}
1161 1280
 	}
1162 1281
 	return 0;
... ...
@@ -399,20 +399,11 @@ int tcp_read_req(struct tcp_connection* con)
399 399
 		req=&con->req;
400 400
 #ifdef USE_TLS
401 401
 		if (con->type==PROTO_TLS){
402
-			if (con->state==S_CONN_ACCEPT){
403
-				if (tls_accept(con, 0)!=0){
404
-					resp=CONN_ERROR;
405
-					goto end_req;
406
-				}
407
-				if(con->state!=S_CONN_OK) goto end_req; /* not enough data */
408
-			}
409
-			if(con->state==S_CONN_CONNECT){
410
-				if (tls_connect(con, 0)!=0){
411
-					resp=CONN_ERROR;
412
-					goto end_req;
413
-				}
414
-				if(con->state!=S_CONN_OK) goto end_req; /* not enough data */
402
+			if (tls_fix_read_conn(con)!=0){
403
+				resp=CONN_ERROR;
404
+				goto end_req;
415 405
 			}
406
+			if(con->state!=S_CONN_OK) goto end_req; /* not enough data */
416 407
 		}
417 408
 #endif
418 409
 
... ...
@@ -547,7 +538,8 @@ void release_tcpconn(struct tcp_connection* c, long state, int unix_sock)
547 538
 		/* errno==EINTR, EWOULDBLOCK a.s.o todo */
548 539
 		response[0]=(long)c;
549 540
 		response[1]=state;
550
-		write(unix_sock, response, sizeof(response));
541
+		if (send_all(unix_sock, response, sizeof(response))<=0)
542
+			LOG(L_ERR, "ERROR: release_tcpconn: send_all failed\n");
551 543
 }
552 544
 
553 545
 
... ...
@@ -625,12 +617,18 @@ void tcp_receive_loop(int unix_sock)
625 617
 					release_tcpconn(con, resp, unix_sock);
626 618
 					goto skip;
627 619
 				}
628
-#ifdef USE_TLS
629
-				if (con->type==PROTO_TLS) tls_tcpconn_update_fd(con, s);
630
-#endif
631 620
 				con->timeout=get_ticks()+TCP_CHILD_TIMEOUT;
632 621
 				FD_SET(s, &master_set);
633 622
 				if (maxfd<s) maxfd=s;
623
+				if (con==list){
624
+					LOG(L_CRIT, "BUG: tcp_receive_loop: duplicate"
625
+							" connection recevied: %p, id %d, fd %d, refcnt %d"
626
+							" state %d (n=%d)\n", con, con->id, con->fd,
627
+							con->refcnt, con->state, n);
628
+					resp=CONN_ERROR;
629
+					release_tcpconn(con, resp, unix_sock);
630
+					goto skip; /* try to recover */
631
+				}
634 632
 				tcpconn_listadd(list, con, c_next, c_prev);
635 633
 			}
636 634
 skip:
... ...
@@ -641,12 +639,22 @@ skip:
641 639
 				DBG("tcp receive: list fd=%d, id=%d, timeout=%d, refcnt=%d\n",
642 640
 						con->fd, con->id, con->timeout, con->refcnt);
643 641
 #endif
642
+				if (con->state<0){
643
+					/* S_CONN_BAD or S_CONN_ERROR, remove it */
644
+					resp=CONN_ERROR;
645
+					FD_CLR(con->fd, &master_set);
646
+					tcpconn_listrm(list, con, c_next, c_prev);
647
+					con->state=S_CONN_BAD;
648
+					release_tcpconn(con, resp, unix_sock);
649
+					continue;
650
+				}
644 651
 				if (nfds && FD_ISSET(con->fd, &sel_set)){
645 652
 #ifdef EXTRA_DEBUG
646 653
 					DBG("tcp receive: match, fd:isset\n");
647 654
 #endif
648 655
 					nfds--;
649 656
 					resp=tcp_read_req(con);
657
+					
650 658
 					if (resp<0){
651 659
 						FD_CLR(con->fd, &master_set);
652 660
 						tcpconn_listrm(list, con, c_next, c_prev);
... ...
@@ -26,7 +26,7 @@ rev_dns=off      # (cmd. line: -R)
26 26
 alias=iptel.org
27 27
 alias="foo.bar"
28 28
 fifo="/tmp/ser_fifo"
29
-listen= tcp:10.0.0.179:5065
29
+#listen= tcp:10.0.0.179:5065
30 30
 alias=  tcp:all:5065
31 31
 tcp_accept_aliases=yes
32 32
 
... ...
@@ -50,6 +50,8 @@ Options:\n\
50 50
     -d address    destination address\n\
51 51
     -p port       destination port\n\
52 52
     -c count      number of packets to be sent\n\
53
+    -s usec       microseconds to sleep before sending \"throttle\" packets\n\
54
+    -t throttle   number of packets to send before sleeping\n\
53 55
     -v            increase verbosity level\n\
54 56
     -V            version number\n\
55 57
     -h            this help message\n\
... ...
@@ -74,6 +76,9 @@ int main (int argc, char** argv)
74 76
 	char *fname;
75 77
 	char *dst;
76 78
 	int port;
79
+	long usec;
80
+	int throttle;
81
+	int t;
77 82
 	
78 83
 	/* init */
79 84
 	count=0;
... ...
@@ -81,9 +86,11 @@ int main (int argc, char** argv)
81 86
 	fname=0;
82 87
 	dst=0;
83 88
 	port=0;
89
+	usec=0;
90
+	throttle=0;
84 91
 
85 92
 	opterr=0;
86
-	while ((c=getopt(argc,argv, "f:c:d:p:vhV"))!=-1){
93
+	while ((c=getopt(argc,argv, "f:c:d:p:s:t:vhV"))!=-1){
87 94
 		switch(c){
88 95
 			case 'f':
89 96
 				fname=optarg;
... ...
@@ -108,6 +115,20 @@ int main (int argc, char** argv)
108 115
 					goto error;
109 116
 				}
110 117
 				break;
118
+			case 's':
119
+				usec=strtol(optarg, &tmp, 10);
120
+				if ((tmp==0)||(*tmp)){
121
+					fprintf(stderr, "bad count: -c %s\n", optarg);
122
+					goto error;
123
+				}
124
+				break;
125
+			case 't':
126
+				throttle=strtol(optarg, &tmp, 10);
127
+				if ((tmp==0)||(*tmp)){
128
+					fprintf(stderr, "bad count: -c %s\n", optarg);
129
+					goto error;
130
+				}
131
+				break;
111 132
 			case 'V':
112 133
 				printf("version: %s\n", version);
113 134
 				printf("%s\n",id);
... ...
@@ -197,12 +218,20 @@ int main (int argc, char** argv)
197 218
 
198 219
 
199 220
 	/* flood loop */
221
+	t=throttle;
200 222
 	for (r=0; r<count; r++){
201 223
 		if ((verbose>1)&&(r%1000))  putchar('.');
202 224
 		if (send(sock, buf, n, 0)==-1) {
203 225
 			fprintf(stderr, "Error: send: %s\n",  strerror(errno));
204 226
 			exit(1);
205 227
 		}
228
+		if (usec){
229
+			t--;
230
+			if (t==0){
231
+				usleep(usec);
232
+				t=throttle;
233
+			}
234
+		}
206 235
 	}
207 236
 	printf("\n%d packets sent, %d bytes each => total %d bytes\n",
208 237
 			count, n, n*count);