Browse code

- tcp: - try to close connections faster even if still referenced - changed refcnt usage (on 0 refcnt free, experimental)

Andrei Pelinescu-Onciul authored on 14/12/2007 01:40:12
Showing 2 changed files
... ...
@@ -71,6 +71,8 @@
71 71
 #define F_CONN_REMOVED      2 /* no longer  in "main" listen fd list */
72 72
 #define F_CONN_READER       4 /* handled by a tcp reader */
73 73
 #define F_CONN_WRITE_W      8 /* watched for write (main) */
74
+#define F_CONN_HASHED      16 /* in tcp_main hash */
75
+#define F_CONN_FD_CLOSED   32 /* in tcp_main hash */
74 76
 
75 77
 
76 78
 enum tcp_req_errors {	TCP_REQ_INIT, TCP_REQ_OK, TCP_READ_ERROR,
... ...
@@ -177,7 +179,7 @@ struct tcp_connection{
177 179
 
178 180
 
179 181
 #define tcpconn_ref(c) atomic_inc(&((c)->refcnt))
180
-#define tcpconn_put(c) atomic_dec(&((c)->refcnt))
182
+#define tcpconn_put(c) atomic_dec_and_test(&((c)->refcnt))
181 183
 
182 184
 
183 185
 #define init_tcp_req( r) \
... ...
@@ -89,6 +89,8 @@
89 89
  *               KEEPCNT, QUICKACK, SYNCNT, LINGER2 (andrei)
90 90
  *  2007-12-04  support for queueing write requests (andrei)
91 91
  *  2007-12-12  destroy connection asap on wbuf. timeout (andrei)
92
+ *  2007-12-13  changed the refcnt and destroy scheme, now refcnt is 1 if
93
+ *                linked into the hash tables (was 0)  (andrei)
92 94
  */
93 95
 
94 96
 
... ...
@@ -980,6 +982,7 @@ inline static struct tcp_connection*  tcpconn_add(struct tcp_connection *c)
980 982
 		c->id_hash=tcp_id_hash(c->id);
981 983
 		c->aliases=0;
982 984
 		TCPCONN_LOCK;
985
+		c->flags|=F_CONN_HASHED;
983 986
 		/* add it at the begining of the list*/
984 987
 		tcpconn_listadd(tcpconn_id_hash[c->id_hash], c, id_next, id_prev);
985 988
 		/* set the aliases */
... ...
@@ -1330,6 +1333,11 @@ inline static void tcp_fd_cache_add(struct tcp_connection *c, int fd)
1330 1333
 #endif /* TCP_FD_CACHE */
1331 1334
 
1332 1335
 
1336
+
1337
+inline static int tcpconn_chld_put(struct tcp_connection* tcpconn);
1338
+
1339
+
1340
+
1333 1341
 /* finds a tcpconn & sends on it
1334 1342
  * uses the dst members to, proto (TCP|TLS) and id and tries to send
1335 1343
  *  from the "from" address (if non null and id==0)
... ...
@@ -1403,7 +1411,8 @@ no_id:
1403 1411
 				LOG(L_ERR, "ERROR: tcp_send: connect failed\n");
1404 1412
 				return -1;
1405 1413
 			}
1406
-			atomic_set(&c->refcnt, 1); /* ref. only from here for now */
1414
+			atomic_set(&c->refcnt, 2); /* ref. from here and it will also
1415
+			                              be added in the tcp_main hash */
1407 1416
 			fd=c->s;
1408 1417
 			
1409 1418
 			/* send the new tcpconn to "tcp main" */
... ...
@@ -1413,9 +1422,11 @@ no_id:
1413 1422
 			if (unlikely(n<=0)){
1414 1423
 				LOG(L_ERR, "BUG: tcp_send: failed send_fd: %s (%d)\n",
1415 1424
 						strerror(errno), errno);
1425
+				/* we can safely delete it, it's not referenced by anybody */
1426
+				_tcpconn_free(c);
1416 1427
 				n=-1;
1417
-				goto end;
1418
-			}	
1428
+				goto end_no_conn;
1429
+			}
1419 1430
 			goto send_it;
1420 1431
 		}
1421 1432
 get_fd:
... ...
@@ -1514,15 +1525,15 @@ send_it:
1514 1525
 		n=tsend_stream(fd, buf, len, tcp_send_timeout*1000); 
1515 1526
 #ifdef TCP_BUF_WRITE
1516 1527
 	}
1517
-#endif /* TCP_BUF_WRITE */
1528
+#else /* ! TCP_BUF_WRITE */
1518 1529
 	lock_release(&c->write_lock);
1530
+#endif /* TCP_BUF_WRITE */
1519 1531
 	DBG("tcp_send: after write: c= %p n=%d fd=%d\n",c, n, fd);
1520 1532
 	DBG("tcp_send: buf=\n%.*s\n", (int)len, buf);
1521 1533
 	if (unlikely(n<0)){
1522 1534
 #ifdef TCP_BUF_WRITE
1523 1535
 		if (tcp_options.tcp_buf_write && 
1524 1536
 				(errno==EAGAIN || errno==EWOULDBLOCK)){
1525
-			lock_get(&c->write_lock);
1526 1537
 			enable_write_watch=_wbufq_empty(c);
1527 1538
 			if (unlikely(_wbufq_add(c, buf, len)<0)){
1528 1539
 				lock_release(&c->write_lock);
... ...
@@ -1542,6 +1553,8 @@ send_it:
1542 1553
 				}
1543 1554
 			}
1544 1555
 			goto end;
1556
+		}else{
1557
+			lock_release(&c->write_lock);
1545 1558
 		}
1546 1559
 error:
1547 1560
 #endif /* TCP_BUF_WRITE */
... ...
@@ -1555,7 +1568,7 @@ error:
1555 1568
 		if (send_all(unix_tcp_sock, response, sizeof(response))<=0){
1556 1569
 			LOG(L_ERR, "BUG: tcp_send: error return failed (write):%s (%d)\n",
1557 1570
 					strerror(errno), errno);
1558
-			tcpconn_put(c); /* deref. it manually */
1571
+			tcpconn_chld_put(c); /* deref. it manually & free on 0 */
1559 1572
 			n=-1;
1560 1573
 		}
1561 1574
 		/* CONN_ERROR will auto-dec refcnt => we must not call tcpconn_put 
... ...
@@ -1572,7 +1585,9 @@ error:
1572 1585
 		if (do_close_fd) close(fd);
1573 1586
 		return n; /* error return, no tcpconn_put */
1574 1587
 	}
1588
+	
1575 1589
 #ifdef TCP_BUF_WRITE
1590
+	lock_release(&c->write_lock);
1576 1591
 	if (likely(tcp_options.tcp_buf_write)){
1577 1592
 		if (unlikely(c->state==S_CONN_CONNECT))
1578 1593
 			c->state=S_CONN_OK;
... ...
@@ -1586,7 +1601,8 @@ end:
1586 1601
 #endif /* TCP_FD_CACHE */
1587 1602
 	if (do_close_fd) close(fd);
1588 1603
 release_c:
1589
-	tcpconn_put(c); /* release c (lock; dec refcnt; unlock) */
1604
+	tcpconn_chld_put(c); /* release c (dec refcnt & free on 0) */
1605
+end_no_conn:
1590 1606
 	return n;
1591 1607
 }
1592 1608
 
... ...
@@ -1736,6 +1752,7 @@ error:
1736 1752
 
1737 1753
 
1738 1754
 
1755
+#if 0
1739 1756
 /* used internally by tcp_main_loop()
1740 1757
  * tries to destroy a tcp connection (if it cannot it will force a timeout)
1741 1758
  * Note: it's called _only_ from the tcp_main process */
... ...
@@ -1802,6 +1819,167 @@ static void tcpconn_destroy(struct tcp_connection* tcpconn)
1802 1819
 				tcpconn, tcpconn->flags);
1803 1820
 	}
1804 1821
 }
1822
+#endif
1823
+
1824
+
1825
+
1826
+/* close tcp_main's fd from a tcpconn
1827
+ * call only in tcp_main context */
1828
+inline static void tcpconn_close_main_fd(struct tcp_connection* tcpconn)
1829
+{
1830
+	int fd;
1831
+	
1832
+	
1833
+	fd=tcpconn->s;
1834
+#ifdef USE_TLS
1835
+	/*FIXME: lock ->writelock ? */
1836
+	if (tcpconn->type==PROTO_TLS)
1837
+		tls_close(tcpconn, fd);
1838
+#endif
1839
+#ifdef TCP_FD_CACHE
1840
+	if (likely(tcp_options.fd_cache)) shutdown(fd, SHUT_RDWR);
1841
+#endif /* TCP_FD_CACHE */
1842
+close_again:
1843
+	if (unlikely(close(fd)<0)){
1844
+		if (errno==EINTR)
1845
+			goto close_again;
1846
+		LOG(L_ERR, "ERROR: tcpconn_put_destroy; close() failed: %s (%d)\n",
1847
+				strerror(errno), errno);
1848
+	}
1849
+}
1850
+
1851
+
1852
+
1853
+/* dec refcnt & frees the connection if refcnt==0
1854
+ * returns 1 if the connection is freed, 0 otherwise
1855
+ *
1856
+ * WARNING: use only from child processes */
1857
+inline static int tcpconn_chld_put(struct tcp_connection* tcpconn)
1858
+{
1859
+	if (unlikely(atomic_dec_and_test(&tcpconn->refcnt))){
1860
+		DBG("tcpconn_put_chld: destroying connection %p (%d, %d) "
1861
+				"flags %04x\n", tcpconn, tcpconn->id,
1862
+				tcpconn->s, tcpconn->flags);
1863
+		/* sanity checks */
1864
+		if (unlikely(!(tcpconn->flags & F_CONN_FD_CLOSED) || 
1865
+					 !(tcpconn->flags & F_CONN_REMOVED) || 
1866
+					(tcpconn->flags & 
1867
+					 		(F_CONN_HASHED| F_CONN_WRITE_W)) )){
1868
+			LOG(L_CRIT, "BUG: tcpconn_put_chld: %p bad flags = %0x\n",
1869
+					tcpconn, tcpconn->flags);
1870
+			abort();
1871
+		}
1872
+		_tcpconn_free(tcpconn); /* destroys also the wbuf_q if still present*/
1873
+		return 1;
1874
+	}
1875
+	return 0;
1876
+}
1877
+
1878
+
1879
+
1880
+/* simple destroy function (the connection should be already removed
1881
+ * from the hashes and the fds should not be watched anymore for IO)
1882
+ */
1883
+inline static void tcpconn_destroy(struct tcp_connection* tcpconn)
1884
+{
1885
+		DBG("tcpconn_destroy: destroying connection %p (%d, %d) "
1886
+				"flags %04x\n", tcpconn, tcpconn->id,
1887
+				tcpconn->s, tcpconn->flags);
1888
+		if (unlikely(tcpconn->flags & F_CONN_HASHED)){
1889
+			LOG(L_CRIT, "BUG: tcpconn_destroy: called with hashed"
1890
+						" connection (%p)\n", tcpconn);
1891
+			/* try to continue */
1892
+			local_timer_del(&tcp_main_ltimer, &tcpconn->timer);
1893
+			TCPCONN_LOCK;
1894
+				_tcpconn_detach(tcpconn);
1895
+			TCPCONN_UNLOCK;
1896
+		}
1897
+		if (likely(!(tcpconn->flags & F_CONN_FD_CLOSED))){
1898
+			tcpconn_close_main_fd(tcpconn);
1899
+			(*tcp_connections_no)--;
1900
+		}
1901
+		_tcpconn_free(tcpconn); /* destroys also the wbuf_q if still present*/
1902
+}
1903
+
1904
+
1905
+
1906
+/* tries to destroy the connection: dec. refcnt and if 0 destroys the
1907
+ *  connection, else it will mark it as BAD and close the main fds
1908
+ *
1909
+ * returns 1 if the connection was destroyed, 0 otherwise
1910
+ *
1911
+ * WARNING: - the connection _has_ to be removed from the hash and timer
1912
+ *  first (use tcpconn_try_unhash() for this )
1913
+ *         - the fd should not be watched anymore (io_watch_del()...)
1914
+ *         - must be called _only_ from the tcp_main process context
1915
+ *          (or else the fd will remain open)
1916
+ */
1917
+inline static int tcpconn_put_destroy(struct tcp_connection* tcpconn)
1918
+{
1919
+	if (unlikely((tcpconn->flags & F_CONN_WRITE_W) ||
1920
+				!(tcpconn->flags & F_CONN_REMOVED))){
1921
+		/* sanity check */
1922
+		LOG(L_CRIT, "BUG: tcpconn_put_destroy: %p flags = %0x\n",
1923
+					tcpconn, tcpconn->flags);
1924
+	}
1925
+	if (unlikely(atomic_dec_and_test(&tcpconn->refcnt))){
1926
+		tcpconn_destroy(tcpconn);
1927
+		return 1;
1928
+	}else{
1929
+		if (likely(!(tcpconn->flags & F_CONN_FD_CLOSED))){
1930
+			tcpconn_close_main_fd(tcpconn);
1931
+			tcpconn->flags|=F_CONN_FD_CLOSED;
1932
+			(*tcp_connections_no)--;
1933
+		}
1934
+		tcpconn->state=S_CONN_BAD;
1935
+		/* in case it's still in a reader timer */
1936
+		tcpconn->timeout=get_ticks_raw();
1937
+	}
1938
+	return 0;
1939
+}
1940
+
1941
+
1942
+
1943
+/* try to remove a connection from the hashes and timer.
1944
+ * returns 1 if the connection was removed, 0 if not (connection not in
1945
+ *  hash)
1946
+ *
1947
+ * WARNING: call it only in the  tcp_main process context or else the
1948
+ *  timer removal won't work.
1949
+ */
1950
+inline static int tcpconn_try_unhash(struct tcp_connection* tcpconn)
1951
+{
1952
+	if (unlikely((tcpconn->flags & F_CONN_WRITE_W) ||
1953
+				!(tcpconn->flags & F_CONN_REMOVED))){
1954
+		/* sanity check */
1955
+		LOG(L_CRIT, "BUG: tcpconn_put_destroy: %p flags = %0x\n",
1956
+					tcpconn, tcpconn->flags);
1957
+	}
1958
+	if (likely(tcpconn->flags & F_CONN_HASHED)){
1959
+		tcpconn->flags&=~F_CONN_HASHED;
1960
+		tcpconn->state=S_CONN_BAD;
1961
+		if (likely(!(tcpconn->flags & F_CONN_READER)))
1962
+			local_timer_del(&tcp_main_ltimer, &tcpconn->timer);
1963
+		else
1964
+			/* in case it's still in a reader timer */
1965
+			tcpconn->timeout=get_ticks_raw();
1966
+		TCPCONN_LOCK;
1967
+			_tcpconn_detach(tcpconn);
1968
+		TCPCONN_UNLOCK;
1969
+#ifdef TCP_BUF_WRITE
1970
+		/* empty possible write buffers (optional) */
1971
+		if (unlikely(_wbufq_non_empty(tcpconn))){
1972
+			lock_get(&tcpconn->write_lock);
1973
+				/* check again, while holding the lock */
1974
+				if (likely(_wbufq_non_empty(tcpconn)))
1975
+					_wbufq_destroy(&tcpconn->wbuf_q);
1976
+			lock_release(&tcpconn->write_lock);
1977
+		}
1978
+#endif /* TCP_BUF_WRITE */
1979
+		return 1;
1980
+	}
1981
+	return 0;
1982
+}
1805 1983
 
1806 1984
 
1807 1985
 
... ...
@@ -1935,7 +2113,9 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
1935 2113
 				}
1936 2114
 #endif
1937 2115
 				p->tcp_conn->flags &= ~F_CONN_READER;
1938
-				tcpconn_destroy(p->tcp_conn);
2116
+				if (likely(tcpconn_try_unhash(p->tcp_conn)))
2117
+					tcpconn_put(p->tcp_conn);
2118
+				tcpconn_put_destroy(p->tcp_conn); /* dec refcnt & destroy */
1939 2119
 			}
1940 2120
 		}
1941 2121
 	}
... ...
@@ -2054,6 +2234,10 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2054 2234
 	switch(cmd){
2055 2235
 		case CONN_RELEASE:
2056 2236
 			tcp_c->busy--;
2237
+			if (unlikely(tcpconn_put(tcpconn))){
2238
+				tcpconn_destroy(tcpconn);
2239
+				break;
2240
+			}
2057 2241
 			if (unlikely(tcpconn->state==S_CONN_BAD)){ 
2058 2242
 #ifdef TCP_BUF_WRITE
2059 2243
 				if (unlikely(tcpconn->flags & F_CONN_WRITE_W)){
... ...
@@ -2061,13 +2245,13 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2061 2245
 					tcpconn->flags &= ~F_CONN_WRITE_W;
2062 2246
 				}
2063 2247
 #endif /* TCP_BUF_WRITE */
2064
-				tcpconn_destroy(tcpconn);
2248
+				if (tcpconn_try_unhash(tcpconn))
2249
+					tcpconn_put_destroy(tcpconn);
2065 2250
 				break;
2066 2251
 			}
2067 2252
 			/* update the timeout*/
2068 2253
 			t=get_ticks_raw();
2069 2254
 			tcpconn->timeout=t+tcp_con_lifetime;
2070
-			tcpconn_put(tcpconn);
2071 2255
 			crt_timeout=tcp_con_lifetime;
2072 2256
 #ifdef TCP_BUF_WRITE
2073 2257
 			if (unlikely(tcp_options.tcp_buf_write && 
... ...
@@ -2081,7 +2265,8 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2081 2265
 						io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
2082 2266
 						tcpconn->flags&=~F_CONN_WRITE_W;
2083 2267
 					}
2084
-					tcpconn_destroy(tcpconn); /* closes also the fd */
2268
+					if (tcpconn_try_unhash(tcpconn))
2269
+						tcpconn_put_destroy(tcpconn);
2085 2270
 					break;
2086 2271
 				}else{
2087 2272
 					crt_timeout=MIN_unsigned(tcp_con_lifetime,
... ...
@@ -2111,7 +2296,9 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2111 2296
 					tcpconn->flags&=~F_CONN_WRITE_W;
2112 2297
 				}
2113 2298
 #endif /* TCP_BUF_WRITE */
2114
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2299
+				if (tcpconn_try_unhash(tcpconn));
2300
+					tcpconn_put_destroy(tcpconn);
2301
+				break;
2115 2302
 			}
2116 2303
 			DBG("handle_tcp_child: CONN_RELEASE  %p refcnt= %d\n", 
2117 2304
 							tcpconn, atomic_get(&tcpconn->refcnt));
... ...
@@ -2131,7 +2318,9 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2131 2318
 					tcpconn->flags&=~F_CONN_WRITE_W;
2132 2319
 				}
2133 2320
 #endif /* TCP_BUF_WRITE */
2134
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2321
+				if (tcpconn_try_unhash(tcpconn))
2322
+					tcpconn_put(tcpconn);
2323
+				tcpconn_put_destroy(tcpconn); /* deref & delete if refcnt==0 */
2135 2324
 				break;
2136 2325
 		default:
2137 2326
 				LOG(L_CRIT, "BUG: handle_tcp_child:  unknown cmd %d"
... ...
@@ -2236,12 +2425,14 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2236 2425
 			LOG(L_ERR, "handle_ser_child: ERROR: received CON_ERROR for %p"
2237 2426
 					" (id %d), refcnt %d\n", 
2238 2427
 					tcpconn, tcpconn->id, atomic_get(&tcpconn->refcnt));
2239
-			tcpconn_destroy(tcpconn); /* will close also the fd */
2428
+			if (tcpconn_try_unhash(tcpconn))
2429
+				tcpconn_put(tcpconn);
2430
+			tcpconn_put_destroy(tcpconn); /* dec refcnt & destroy on 0 */
2240 2431
 			break;
2241 2432
 		case CONN_GET_FD:
2242 2433
 			/* send the requested FD  */
2243 2434
 			/* WARNING: take care of setting refcnt properly to
2244
-			 * avoid race condition */
2435
+			 * avoid race conditions */
2245 2436
 			if (unlikely(send_fd(p->unix_sock, &tcpconn, sizeof(tcpconn),
2246 2437
 								tcpconn->s)<=0)){
2247 2438
 				LOG(L_ERR, "ERROR: handle_ser_child: send_fd failed\n");
... ...
@@ -2250,10 +2441,12 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2250 2441
 		case CONN_NEW:
2251 2442
 			/* update the fd in the requested tcpconn*/
2252 2443
 			/* WARNING: take care of setting refcnt properly to
2253
-			 * avoid race condition */
2444
+			 * avoid race conditions */
2254 2445
 			if (unlikely(fd==-1)){
2255 2446
 				LOG(L_CRIT, "BUG: handle_ser_child: CONN_NEW:"
2256 2447
 							" no fd received\n");
2448
+				tcpconn->flags|=F_CONN_FD_CLOSED;
2449
+				tcpconn_put_destroy(tcpconn);
2257 2450
 				break;
2258 2451
 			}
2259 2452
 			(*tcp_connections_no)++;
... ...
@@ -2264,7 +2457,7 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2264 2457
 			t=get_ticks_raw();
2265 2458
 			tcpconn->timeout=t+tcp_con_lifetime;
2266 2459
 			/* activate the timer (already properly init. in tcpconn_new())
2267
-			 * no need for */
2460
+			 * no need for reinit */
2268 2461
 			local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
2269 2462
 								tcp_con_lifetime, t);
2270 2463
 			tcpconn->flags&=~F_CONN_REMOVED;
... ...
@@ -2283,18 +2476,23 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2283 2476
 						" new socket to the fd list\n");
2284 2477
 				tcpconn->flags|=F_CONN_REMOVED;
2285 2478
 				tcpconn->flags&=~F_CONN_WRITE_W;
2286
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2479
+				tcpconn_try_unhash(tcpconn); /*  unhash & dec refcnt */
2480
+				tcpconn_put_destroy(tcpconn);
2287 2481
 			}
2288 2482
 			break;
2289 2483
 #ifdef TCP_BUF_WRITE
2290 2484
 		case CONN_QUEUED_WRITE:
2485
+			if (unlikely((tcpconn->state==S_CONN_BAD) || 
2486
+							!(tcpconn->flags & F_CONN_HASHED) ))
2487
+				break;
2291 2488
 			if (!(tcpconn->flags & F_CONN_WRITE_W)){
2292 2489
 				if (tcpconn->flags& F_CONN_REMOVED){
2293 2490
 					if (unlikely(io_watch_add(&io_h, tcpconn->s, POLLOUT,
2294 2491
 												F_TCPCONN, tcpconn)<0)){
2295 2492
 						LOG(L_CRIT, "ERROR: tcp_main: handle_ser_child: failed"
2296 2493
 								    " to enable write watch on socket\n");
2297
-						tcpconn_destroy(tcpconn);
2494
+						if (tcpconn_try_unhash(tcpconn))
2495
+							tcpconn_put_destroy(tcpconn);
2298 2496
 						break;
2299 2497
 					}
2300 2498
 				}else{
... ...
@@ -2304,7 +2502,8 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2304 2502
 								    " to change socket watch events\n");
2305 2503
 						io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
2306 2504
 						tcpconn->flags|=F_CONN_REMOVED;
2307
-						tcpconn_destroy(tcpconn);
2505
+						if (tcpconn_try_unhash(tcpconn))
2506
+							tcpconn_put_destroy(tcpconn);
2308 2507
 						break;
2309 2508
 					}
2310 2509
 				}
... ...
@@ -2474,6 +2673,8 @@ static inline int handle_new_connect(struct socket_info* si)
2474 2673
 	tcpconn=tcpconn_new(new_sock, &su, dst_su, si, si->proto, S_CONN_ACCEPT);
2475 2674
 	if (likely(tcpconn)){
2476 2675
 #ifdef TCP_PASS_NEW_CONNECTION_ON_DATA
2676
+		atomic_set(&tcpconn->refcnt, 1); /* safe, not yet available to the
2677
+											outside world */
2477 2678
 		tcpconn_add(tcpconn);
2478 2679
 		/* activate the timer */
2479 2680
 		local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
... ...
@@ -2484,21 +2685,24 @@ static inline int handle_new_connect(struct socket_info* si)
2484 2685
 			LOG(L_CRIT, "ERROR: tcp_main: handle_new_connect: failed to add"
2485 2686
 						" new socket to the fd list\n");
2486 2687
 			tcpconn->flags|=F_CONN_REMOVED;
2487
-			tcpconn_destroy(tcpconn); /* closes also the fd */
2688
+			if (tcpconn_try_unhash(tcpconn))
2689
+				tcpconn_put_destroy(tcpconn);
2488 2690
 		}
2489 2691
 #else
2490
-		atomic_set(&tcpconn->refcnt, 1); /* safe, not yet available to the
2692
+		atomic_set(&tcpconn->refcnt, 2); /* safe, not yet available to the
2491 2693
 											outside world */
2694
+		/* prepare it for passing to a child */
2695
+		tcpconn->flags|=F_CONN_READER;
2492 2696
 		tcpconn_add(tcpconn);
2493 2697
 		DBG("handle_new_connect: new connection: %p %d flags: %04x\n",
2494 2698
 			tcpconn, tcpconn->s, tcpconn->flags);
2495
-		/* pass it to a child */
2496
-		tcpconn->flags|=F_CONN_READER;
2497 2699
 		if(unlikely(send2child(tcpconn)<0)){
2498 2700
 			LOG(L_ERR,"ERROR: handle_new_connect: no children "
2499 2701
 					"available\n");
2500 2702
 			tcpconn->flags&=~F_CONN_READER;
2501
-			tcpconn_destroy(tcpconn);
2703
+			tcpconn_put(tcpconn);
2704
+			tcpconn_try_unhash(tcpconn);
2705
+			tcpconn_put_destroy(tcpconn);
2502 2706
 		}
2503 2707
 #endif
2504 2708
 	}else{ /*tcpconn==0 */
... ...
@@ -2554,7 +2758,11 @@ inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, short ev,
2554 2758
 			io_watch_del(&io_h, tcpconn->s, fd_i, 0);
2555 2759
 			tcpconn->flags|=F_CONN_REMOVED;
2556 2760
 			tcpconn->flags&=~F_CONN_WRITE_W;
2557
-			tcpconn_destroy(tcpconn);
2761
+			if (unlikely(!tcpconn_try_unhash(tcpconn))){
2762
+				LOG(L_CRIT, "BUG: tcpconn_ev: unhashed connection %p\n",
2763
+							tcpconn);
2764
+			}
2765
+			tcpconn_put_destroy(tcpconn);
2558 2766
 			goto error;
2559 2767
 		}
2560 2768
 		if (empty_q){
... ...
@@ -2596,7 +2804,9 @@ inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, short ev,
2596 2804
 				tcpconn->flags&=~F_CONN_WRITE_W;
2597 2805
 			}
2598 2806
 #endif /* TCP_BUF_WRITE */
2599
-			tcpconn_destroy(tcpconn);
2807
+			tcpconn_put(tcpconn);
2808
+			tcpconn_try_unhash(tcpconn); 
2809
+			tcpconn_put_destroy(tcpconn); /* because of the tcpconn_ref() */
2600 2810
 		}
2601 2811
 	}
2602 2812
 	return 0; /* we are not interested in possibly queued io events, 
... ...
@@ -2687,45 +2897,33 @@ static ticks_t tcpconn_main_timeout(ticks_t t, struct timer_ln* tl, void* data)
2687 2897
 	}
2688 2898
 #endif /* TCP_BUF_WRITE */
2689 2899
 	DBG("tcp_main: timeout for %p\n", c);
2690
-	if (likely(atomic_get(&c->refcnt)==0)){
2900
+	if (likely(c->flags & F_CONN_HASHED)){
2901
+		c->flags&=~F_CONN_HASHED;
2902
+		c->state=S_CONN_BAD;
2691 2903
 		TCPCONN_LOCK;
2692
-			/* check again to avoid races with tcp_send() */
2693
-			if (likely(atomic_get(&c->refcnt)==0)){
2694
-				/* delete */
2695
-				_tcpconn_detach(c);
2696
-				TCPCONN_UNLOCK; /* unlock as soon as possible */
2697
-				fd=c->s;
2698
-				if (likely(fd>0)){
2699
-					if (likely(!(c->flags & F_CONN_REMOVED)
2904
+			_tcpconn_detach(c);
2905
+		TCPCONN_UNLOCK;
2906
+	}else{
2907
+		LOG(L_CRIT, "BUG: tcp_main: timer: called with unhashed connection %p"
2908
+				"\n", c);
2909
+		tcpconn_ref(c); /* ugly hack to try to go on */
2910
+	}
2911
+	fd=c->s;
2912
+	if (likely(fd>0)){
2913
+		if (likely(!(c->flags & F_CONN_REMOVED)
2700 2914
 #ifdef TCP_BUF_WRITE
2701
-								|| (c->flags & F_CONN_WRITE_W)
2915
+			|| (c->flags & F_CONN_WRITE_W)
2702 2916
 #endif /* TCP_BUF_WRITE */
2703
-								)){
2704
-						io_watch_del(&io_h, fd, -1, IO_FD_CLOSING);
2705
-						c->flags|=F_CONN_REMOVED;
2917
+			)){
2918
+			io_watch_del(&io_h, fd, -1, IO_FD_CLOSING);
2919
+			c->flags|=F_CONN_REMOVED;
2706 2920
 #ifdef TCP_BUF_WRITE
2707
-						c->flags&=~F_CONN_WRITE_W;
2921
+			c->flags&=~F_CONN_WRITE_W;
2708 2922
 #endif /* TCP_BUF_WRITE */
2709
-					}
2710
-#ifdef USE_TLS
2711
-					if (unlikely(c->type==PROTO_TLS ))
2712
-						tls_close(c, fd);
2713
-#endif /* USE_TLS */
2714
-					_tcpconn_free(c);
2715
-#ifdef TCP_FD_CACHE
2716
-					if (likely(tcp_options.fd_cache)) shutdown(fd, SHUT_RDWR);
2717
-#endif /* TCP_FD_CACHE */
2718
-					close(fd);
2719
-				}
2720
-				(*tcp_connections_no)--; /* modified only in tcp_main
2721
-											 => no lock needed */
2722
-				return 0; /* don't prolong the timer anymore */
2723
-			}
2724
-		TCPCONN_UNLOCK;
2923
+		}
2725 2924
 	}
2726
-	/* if we are here we can't delete the connection, it's still referenced
2727
-	 *  => we just delay deleting it */
2728
-	return TCPCONN_WAIT_TIMEOUT;
2925
+	tcpconn_put_destroy(c);
2926
+	return 0;
2729 2927
 }
2730 2928
 
2731 2929