Browse code

- tcp: close connection immediately if the write buf. timeouts (timeout fixes) + some cleanups

Andrei Pelinescu-Onciul authored on 12/12/2007 19:11:25
Showing 2 changed files
... ...
@@ -135,6 +135,7 @@ struct tcp_conn_alias{
135 135
 	struct tcp_wbuffer_queue{
136 136
 		struct tcp_wbuffer* first;
137 137
 		struct tcp_wbuffer* last;
138
+		ticks_t wr_timeout; /* write timeout*/
138 139
 		unsigned int queued; /* total size */
139 140
 		unsigned int offset; /* offset in the first wbuffer were data
140 141
 								starts */
... ...
@@ -167,7 +168,6 @@ struct tcp_connection{
167 168
 	struct tcp_conn_alias con_aliases[TCP_CON_MAX_ALIASES];
168 169
 	int aliases; /* aliases number, at least 1 */
169 170
 #ifdef TCP_BUF_WRITE
170
-	ticks_t last_write; /* time when the last write took place */
171 171
 	struct tcp_wbuffer_queue wbuf_q;
172 172
 #endif
173 173
 };
... ...
@@ -88,6 +88,7 @@
88 88
  *  2007-11-28  added support for TCP_DEFER_ACCEPT, KEEPALIVE, KEEPINTVL,
89 89
  *               KEEPCNT, QUICKACK, SYNCNT, LINGER2 (andrei)
90 90
  *  2007-12-04  support for queueing write requests (andrei)
91
+ *  2007-12-12  destroy connection asap on wbuf. timeout (andrei)
91 92
  */
92 93
 
93 94
 
... ...
@@ -558,7 +559,14 @@ inline static int _tcpconn_write_nb(int fd, struct tcp_connection* c,
558 559
 #ifdef TCP_BUF_WRITE
559 560
 
560 561
 
561
-inline static int wbufq_add(struct  tcp_connection* c, char* data, 
562
+/* unsafe version */
563
+#define _wbufq_empty(con) ((con)->wbuf_q.first==0)
564
+/* unsafe version */
565
+#define _wbufq_non_empty(con) ((con)->wbuf_q.first!=0)
566
+
567
+
568
+/* unsafe version, call while holding the connection write lock */
569
+inline static int _wbufq_add(struct  tcp_connection* c, char* data, 
562 570
 							unsigned int size)
563 571
 {
564 572
 	struct tcp_wbuffer_queue* q;
... ...
@@ -573,11 +581,11 @@ inline static int wbufq_add(struct  tcp_connection* c, char* data,
573 581
 	if (unlikely(	((q->queued+size)>tcp_options.tcpconn_wq_max) ||
574 582
 					((*tcp_total_wq+size)>tcp_options.tcp_wq_max) ||
575 583
 					(q->first &&
576
-					TICKS_GT(t, c->last_write+tcp_options.tcp_wq_timeout)) )){
584
+					TICKS_LT(q->wr_timeout, t)) )){
577 585
 		LOG(L_ERR, "ERROR: wbufq_add(%d bytes): write queue full or timeout "
578 586
 					" (%d, total %d, last write %d s ago)\n",
579 587
 					size, q->queued, *tcp_total_wq,
580
-					TICKS_TO_S(t-c->last_write));
588
+					TICKS_TO_S(t-q->wr_timeout-tcp_options.tcp_wq_timeout));
581 589
 		goto error;
582 590
 	}
583 591
 	
... ...
@@ -592,7 +600,7 @@ inline static int wbufq_add(struct  tcp_connection* c, char* data,
592 600
 		q->first=wb;
593 601
 		q->last_used=0;
594 602
 		q->offset=0;
595
-		c->last_write=get_ticks_raw(); /* start with the crt. time */
603
+		q->wr_timeout=get_ticks_raw()+tcp_options.tcp_wq_timeout;
596 604
 	}else{
597 605
 		wb=q->last;
598 606
 	}
... ...
@@ -626,7 +634,8 @@ error:
626 634
 
627 635
 
628 636
 
629
-inline static void wbufq_destroy( struct  tcp_wbuffer_queue* q)
637
+/* unsafe version, call while holding the connection write lock */
638
+inline static void _wbufq_destroy( struct  tcp_wbuffer_queue* q)
630 639
 {
631 640
 	struct tcp_wbuffer* wb;
632 641
 	struct tcp_wbuffer* next_wb;
... ...
@@ -650,7 +659,7 @@ inline static void wbufq_destroy( struct  tcp_wbuffer_queue* q)
650 659
 
651 660
 
652 661
 
653
-/* tries to empty the queue
662
+/* tries to empty the queue  (safe version, c->write_lock must not be hold)
654 663
  * returns -1 on error, bytes written on success (>=0) 
655 664
  * if the whole queue is emptied => sets *empty*/
656 665
 inline static int wbufq_run(int fd, struct tcp_connection* c, int* empty)
... ...
@@ -688,7 +697,7 @@ inline static int wbufq_run(int fd, struct tcp_connection* c, int* empty)
688 697
 				atomic_add_int((int*)tcp_total_wq, -n);
689 698
 				break;
690 699
 			}
691
-			c->last_write=t;
700
+			q->wr_timeout=t+tcp_options.tcp_wq_timeout;
692 701
 			c->state=S_CONN_OK;
693 702
 		}else{
694 703
 			if (n<0){
... ...
@@ -1018,8 +1027,8 @@ static inline void _tcpconn_detach(struct tcp_connection *c)
1018 1027
 static inline void _tcpconn_free(struct tcp_connection* c)
1019 1028
 {
1020 1029
 #ifdef TCP_BUF_WRITE
1021
-	if (unlikely(c->wbuf_q.first))
1022
-		wbufq_destroy(&c->wbuf_q);
1030
+	if (unlikely(_wbufq_non_empty(c)))
1031
+		_wbufq_destroy(&c->wbuf_q);
1023 1032
 #endif
1024 1033
 	lock_destroy(&c->write_lock);
1025 1034
 #ifdef USE_TLS
... ...
@@ -1412,11 +1421,11 @@ no_id:
1412 1421
 get_fd:
1413 1422
 #ifdef TCP_BUF_WRITE
1414 1423
 		/* if data is already queued, we don't need the fd any more */
1415
-		if (unlikely(tcp_options.tcp_buf_write && c->wbuf_q.first)){
1424
+		if (unlikely(tcp_options.tcp_buf_write && _wbufq_non_empty(c))){
1416 1425
 			lock_get(&c->write_lock);
1417
-				if (likely(c->wbuf_q.first)){
1426
+				if (likely(_wbufq_non_empty(c))){
1418 1427
 					do_close_fd=0;
1419
-					if (unlikely(wbufq_add(c, buf, len)<0)){
1428
+					if (unlikely(_wbufq_add(c, buf, len)<0)){
1420 1429
 						lock_release(&c->write_lock);
1421 1430
 						n=-1;
1422 1431
 						goto error;
... ...
@@ -1483,8 +1492,8 @@ send_it:
1483 1492
 	lock_get(&c->write_lock);
1484 1493
 #ifdef TCP_BUF_WRITE
1485 1494
 	if (likely(tcp_options.tcp_buf_write)){
1486
-		if (c->wbuf_q.first){
1487
-			if (unlikely(wbufq_add(c, buf, len)<0)){
1495
+		if (_wbufq_non_empty(c)){
1496
+			if (unlikely(_wbufq_add(c, buf, len)<0)){
1488 1497
 				lock_release(&c->write_lock);
1489 1498
 				n=-1;
1490 1499
 				goto error;
... ...
@@ -1514,8 +1523,8 @@ send_it:
1514 1523
 		if (tcp_options.tcp_buf_write && 
1515 1524
 				(errno==EAGAIN || errno==EWOULDBLOCK)){
1516 1525
 			lock_get(&c->write_lock);
1517
-			enable_write_watch=(c->wbuf_q.first==0);
1518
-			if (unlikely(wbufq_add(c, buf, len)<0)){
1526
+			enable_write_watch=_wbufq_empty(c);
1527
+			if (unlikely(_wbufq_add(c, buf, len)<0)){
1519 1528
 				lock_release(&c->write_lock);
1520 1529
 				n=-1;
1521 1530
 				goto error;
... ...
@@ -1567,7 +1576,6 @@ error:
1567 1576
 	if (likely(tcp_options.tcp_buf_write)){
1568 1577
 		if (unlikely(c->state==S_CONN_CONNECT))
1569 1578
 			c->state=S_CONN_OK;
1570
-		c->last_write=get_ticks_raw();
1571 1579
 	}
1572 1580
 #endif /* TCP_BUF_WRITE */
1573 1581
 end:
... ...
@@ -1747,11 +1755,11 @@ static void tcpconn_destroy(struct tcp_connection* tcpconn)
1747 1755
 		LOG(L_CRIT, "tcpconn_destroy: possible BUG: flags = %0x\n",
1748 1756
 					tcpconn->flags);
1749 1757
 	}
1750
-	if (unlikely(tcpconn->wbuf_q.first)){
1758
+	if (unlikely(_wbufq_non_empty(tcpconn))){
1751 1759
 		lock_get(&tcpconn->write_lock);
1752 1760
 			/* check again, while holding the lock */
1753
-			if (likely(tcpconn->wbuf_q.first))
1754
-				wbufq_destroy(&tcpconn->wbuf_q);
1761
+			if (likely(_wbufq_non_empty(tcpconn)))
1762
+				_wbufq_destroy(&tcpconn->wbuf_q);
1755 1763
 		lock_release(&tcpconn->write_lock);
1756 1764
 	}
1757 1765
 #endif /* TCP_BUF_WRITE */
... ...
@@ -1986,6 +1994,7 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
1986 1994
 	int bytes;
1987 1995
 	int n;
1988 1996
 	ticks_t t;
1997
+	ticks_t crt_timeout;
1989 1998
 	
1990 1999
 	if (unlikely(tcp_c->unix_sock<=0)){
1991 2000
 		/* (we can't have a fd==0, 0 is never closed )*/
... ...
@@ -2059,11 +2068,31 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2059 2068
 			t=get_ticks_raw();
2060 2069
 			tcpconn->timeout=t+tcp_con_lifetime;
2061 2070
 			tcpconn_put(tcpconn);
2071
+			crt_timeout=tcp_con_lifetime;
2072
+#ifdef TCP_BUF_WRITE
2073
+			if (unlikely(tcp_options.tcp_buf_write && 
2074
+							_wbufq_non_empty(tcpconn) )){
2075
+				if (unlikely(TICKS_LE(t, tcpconn->wbuf_q.wr_timeout))){
2076
+					DBG("handle_tcp_child: wr. timeout on CONN_RELEASE for %p "
2077
+							"refcnt= %d\n", tcpconn,
2078
+							atomic_get(&tcpconn->refcnt));
2079
+					/* timeout */
2080
+					if (unlikely(tcpconn->flags & F_CONN_WRITE_W)){
2081
+						io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
2082
+						tcpconn->flags&=~F_CONN_WRITE_W;
2083
+					}
2084
+					tcpconn_destroy(tcpconn); /* closes also the fd */
2085
+					break;
2086
+				}else{
2087
+					crt_timeout=MIN_unsigned(tcp_con_lifetime,
2088
+											tcpconn->wbuf_q.wr_timeout-t);
2089
+				}
2090
+			}
2091
+#endif /* TCP_BUF_WRITE */
2062 2092
 			/* re-activate the timer */
2063 2093
 			tcpconn->timer.f=tcpconn_main_timeout;
2064 2094
 			local_timer_reinit(&tcpconn->timer);
2065
-			local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
2066
-								tcp_con_lifetime, t);
2095
+			local_timer_add(&tcp_main_ltimer, &tcpconn->timer, crt_timeout, t);
2067 2096
 			/* must be after the de-ref*/
2068 2097
 			tcpconn->flags&=~(F_CONN_REMOVED|F_CONN_READER);
2069 2098
 #ifdef TCP_BUF_WRITE
... ...
@@ -2234,7 +2263,8 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2234 2263
 			/* update the timeout*/
2235 2264
 			t=get_ticks_raw();
2236 2265
 			tcpconn->timeout=t+tcp_con_lifetime;
2237
-			/* activate the timer (already properly init. in tcpconn_new() */
2266
+			/* activate the timer (already properly init. in tcpconn_new())
2267
+			 * no need for */
2238 2268
 			local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
2239 2269
 								tcp_con_lifetime, t);
2240 2270
 			tcpconn->flags&=~F_CONN_REMOVED;
... ...
@@ -2279,6 +2309,20 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2279 2309
 					}
2280 2310
 				}
2281 2311
 				tcpconn->flags|=F_CONN_WRITE_W;
2312
+				t=get_ticks_raw();
2313
+				if (likely(!(tcpconn->flags & F_CONN_READER) && 
2314
+					(TICKS_LT(tcpconn->wbuf_q.wr_timeout, tcpconn->timeout)) &&
2315
+						TICKS_LT(t, tcpconn->wbuf_q.wr_timeout) )){
2316
+					/* _wbufq_nonempty() is guaranteed here */
2317
+					/* update the timer */
2318
+					local_timer_del(&tcp_main_ltimer, &tcpconn->timer);
2319
+					local_timer_reinit(&tcpconn->timer);
2320
+					local_timer_add(&tcp_main_ltimer, &tcpconn->timer,
2321
+										tcpconn->wbuf_q.wr_timeout-t, t);
2322
+					DBG("tcp_main: handle_ser_child: CONN_QUEUED_WRITE; %p "
2323
+							"timeout adjusted to %d s\n", tcpconn, 
2324
+							TICKS_TO_S(tcpconn->wbuf_q.wr_timeout-t));
2325
+				}
2282 2326
 			}else{
2283 2327
 				LOG(L_WARN, "tcp_main: hanlder_ser_child: connection %p"
2284 2328
 							" already watched for write\n", tcpconn);
... ...
@@ -2621,10 +2665,28 @@ static ticks_t tcpconn_main_timeout(ticks_t t, struct timer_ln* tl, void* data)
2621 2665
 	c=(struct tcp_connection*)data; 
2622 2666
 	/* or (struct tcp...*)(tl-offset(c->timer)) */
2623 2667
 	
2668
+#ifdef TCP_BUF_WRITE
2669
+	DBG( "tcp_main: entering timer for %p (ticks=%d, timeout=%d (%d s), "
2670
+			"wr_timeout=%d (%d s)), write queue: %d bytes\n",
2671
+			c, t, c->timeout, TICKS_TO_S(c->timeout-t),
2672
+			c->wbuf_q.wr_timeout, TICKS_TO_S(c->wbuf_q.wr_timeout-t),
2673
+			c->wbuf_q.queued);
2674
+	
2675
+	if (TICKS_LT(t, c->timeout) && 
2676
+			(!tcp_options.tcp_buf_write | _wbufq_empty(c) |
2677
+				TICKS_LT(t, c->wbuf_q.wr_timeout)) ){
2678
+		if (unlikely(tcp_options.tcp_buf_write && _wbufq_non_empty(c)))
2679
+			return (ticks_t)MIN_unsigned(c->timeout-t, c->wbuf_q.wr_timeout-t);
2680
+		else
2681
+			return (ticks_t)(c->timeout - t);
2682
+	}
2683
+#else /* ! TCP_BUF_WRITE */
2624 2684
 	if (TICKS_LT(t, c->timeout)){
2625 2685
 		/* timeout extended, exit */
2626 2686
 		return (ticks_t)(c->timeout - t);
2627 2687
 	}
2688
+#endif /* TCP_BUF_WRITE */
2689
+	DBG("tcp_main: timeout for %p\n", c);
2628 2690
 	if (likely(atomic_get(&c->refcnt)==0)){
2629 2691
 		TCPCONN_LOCK;
2630 2692
 			/* check again to avoid races with tcp_send() */