Browse code

- tcp: - try to close connections faster even if still referenced - changed refcnt usage (on 0 refcnt free, experimental)

Andrei Pelinescu-Onciul authored on 14/12/2007 01:40:12
Showing 2 changed files
... ...
@@ -71,6 +71,8 @@
71 71
 #define F_CONN_REMOVED      2 /* no longer  in "main" listen fd list */
72 72
 #define F_CONN_READER       4 /* handled by a tcp reader */
73 73
 #define F_CONN_WRITE_W      8 /* watched for write (main) */
74
+#define F_CONN_HASHED      16 /* in tcp_main hash */
75
+#define F_CONN_FD_CLOSED   32 /* in tcp_main hash */
74 76
 
75 77
 
76 78
 enum tcp_req_errors {	TCP_REQ_INIT, TCP_REQ_OK, TCP_READ_ERROR,
... ...
@@ -177,7 +179,7 @@ struct tcp_connection{
177 177
 
178 178
 
179 179
 #define tcpconn_ref(c) atomic_inc(&((c)->refcnt))
180
-#define tcpconn_put(c) atomic_dec(&((c)->refcnt))
180
+#define tcpconn_put(c) atomic_dec_and_test(&((c)->refcnt))
181 181
 
182 182
 
183 183
 #define init_tcp_req( r) \
... ...
@@ -89,6 +89,8 @@
89 89
  *               KEEPCNT, QUICKACK, SYNCNT, LINGER2 (andrei)
90 90
  *  2007-12-04  support for queueing write requests (andrei)
91 91
  *  2007-12-12  destroy connection asap on wbuf. timeout (andrei)
92
+ *  2007-12-13  changed the refcnt and destroy scheme, now refcnt is 1 if
93
+ *                linked into the hash tables (was 0)  (andrei)
92 94
  */
93 95
 
94 96
 
... ...
@@ -980,6 +982,7 @@ inline static struct tcp_connection*  tcpconn_add(struct tcp_connection *c)
980 980
 		c->id_hash=tcp_id_hash(c->id);
981 981
 		c->aliases=0;
982 982
 		TCPCONN_LOCK;
983
+		c->flags|=F_CONN_HASHED;
983 984
 		/* add it at the begining of the list*/
984 985
 		tcpconn_listadd(tcpconn_id_hash[c->id_hash], c, id_next, id_prev);
985 986
 		/* set the aliases */
... ...
@@ -1330,6 +1333,11 @@ inline static void tcp_fd_cache_add(struct tcp_connection *c, int fd)
1330 1330
 #endif /* TCP_FD_CACHE */
1331 1331
 
1332 1332
 
1333
+
1334
+inline static int tcpconn_chld_put(struct tcp_connection* tcpconn);
1335
+
1336
+
1337
+
1333 1338
 /* finds a tcpconn & sends on it
1334 1339
  * uses the dst members to, proto (TCP|TLS) and id and tries to send
1335 1340
  *  from the "from" address (if non null and id==0)
... ...
@@ -1403,7 +1411,8 @@ no_id:
1403 1403
 				LOG(L_ERR, "ERROR: tcp_send: connect failed\n");
1404 1404
 				return -1;
1405 1405
 			}
1406
-			atomic_set(&c->refcnt, 1); /* ref. only from here for now */
1406
+			atomic_set(&c->refcnt, 2); /* ref. from here and it will also
1407
+			                              be added in the tcp_main hash */
1407 1408
 			fd=c->s;
1408 1409
 			
1409 1410
 			/* send the new tcpconn to "tcp main" */
... ...
@@ -1413,9 +1422,11 @@ no_id:
1413 1413
 			if (unlikely(n<=0)){
1414 1414
 				LOG(L_ERR, "BUG: tcp_send: failed send_fd: %s (%d)\n",
1415 1415
 						strerror(errno), errno);
1416
+				/* we can safely delete it, it's not referenced by anybody */
1417
+				_tcpconn_free(c);
1416 1418
 				n=-1;
1417
-				goto end;
1418
-			}	
1419
+				goto end_no_conn;
1420
+			}
1419 1421
 			goto send_it;
1420 1422
 		}
1421 1423
 get_fd:
... ...
@@ -1514,15 +1525,15 @@ send_it:
1514 1514
 		n=tsend_stream(fd, buf, len, tcp_send_timeout*1000); 
1515 1515
 #ifdef TCP_BUF_WRITE
1516 1516
 	}
1517
-#endif /* TCP_BUF_WRITE */
1517
+#else /* ! TCP_BUF_WRITE */
1518 1518
 	lock_release(&c->write_lock);
1519
+#endif /* TCP_BUF_WRITE */
1519 1520
 	DBG("tcp_send: after write: c= %p n=%d fd=%d\n",c, n, fd);
1520 1521
 	DBG("tcp_send: buf=\n%.*s\n", (int)len, buf);
1521 1522
 	if (unlikely(n<0)){
1522 1523
 #ifdef TCP_BUF_WRITE
1523 1524
 		if (tcp_options.tcp_buf_write && 
1524 1525
 				(errno==EAGAIN || errno==EWOULDBLOCK)){
1525
-			lock_get(&c->write_lock);
1526 1526
 			enable_write_watch=_wbufq_empty(c);
1527 1527
 			if (unlikely(_wbufq_add(c, buf, len)<0)){
1528 1528
 				lock_release(&c->write_lock);
... ...
@@ -1542,6 +1553,8 @@ send_it:
1542 1542
 				}
1543 1543
 			}
1544 1544
 			goto end;
1545
+		}else{
1546
+			lock_release(&c->write_lock);
1545 1547
 		}
1546 1548
 error:
1547 1549
 #endif /* TCP_BUF_WRITE */
... ...
@@ -1555,7 +1568,7 @@ error:
1555 1555
 		if (send_all(unix_tcp_sock, response, sizeof(response))<=0){
1556 1556
 			LOG(L_ERR, "BUG: tcp_send: error return failed (write):%s (%d)\n",
1557 1557
 					strerror(errno), errno);
1558
-			tcpconn_put(c); /* deref. it manually */
1558
+			tcpconn_chld_put(c); /* deref. it manually & free on 0 */
1559 1559
 			n=-1;
1560 1560
 		}
1561 1561
 		/* CONN_ERROR will auto-dec refcnt => we must not call tcpconn_put 
... ...
@@ -1572,7 +1585,9 @@ error:
1572 1572
 		if (do_close_fd) close(fd);
1573 1573
 		return n; /* error return, no tcpconn_put */
1574 1574
 	}
1575
+	
1575 1576
 #ifdef TCP_BUF_WRITE
1577
+	lock_release(&c->write_lock);
1576 1578
 	if (likely(tcp_options.tcp_buf_write)){
1577 1579
 		if (unlikely(c->state==S_CONN_CONNECT))
1578 1580
 			c->state=S_CONN_OK;
... ...
@@ -1586,7 +1601,8 @@ end:
1586 1586
 #endif /* TCP_FD_CACHE */
1587 1587
 	if (do_close_fd) close(fd);
1588 1588
 release_c:
1589
-	tcpconn_put(c); /* release c (lock; dec refcnt; unlock) */
1589
+	tcpconn_chld_put(c); /* release c (dec refcnt & free on 0) */
1590
+end_no_conn:
1590 1591
 	return n;
1591 1592
 }
1592 1593
 
... ...
@@ -1736,6 +1752,7 @@ error:
1736 1736
 
1737 1737
 
1738 1738
 
1739
+#if 0
1739 1740
 /* used internally by tcp_main_loop()
1740 1741
  * tries to destroy a tcp connection (if it cannot it will force a timeout)
1741 1742
  * Note: it's called _only_ from the tcp_main process */
... ...
@@ -1802,6 +1819,167 @@ static void tcpconn_destroy(struct tcp_connection* tcpconn)
1802 1802
 				tcpconn, tcpconn->flags);
1803 1803
 	}
1804 1804
 }
1805
+#endif
1806
+
1807
+
1808
+
1809
+/* close tcp_main's fd from a tcpconn
1810
+ * call only in tcp_main context */
1811
+inline static void tcpconn_close_main_fd(struct tcp_connection* tcpconn)
1812
+{
1813
+	int fd;
1814
+	
1815
+	
1816
+	fd=tcpconn->s;
1817
+#ifdef USE_TLS
1818
+	/*FIXME: lock ->writelock ? */
1819
+	if (tcpconn->type==PROTO_TLS)
1820
+		tls_close(tcpconn, fd);
1821
+#endif
1822
+#ifdef TCP_FD_CACHE
1823
+	if (likely(tcp_options.fd_cache)) shutdown(fd, SHUT_RDWR);
1824
+#endif /* TCP_FD_CACHE */
1825
+close_again:
1826
+	if (unlikely(close(fd)<0)){
1827
+		if (errno==EINTR)
1828
+			goto close_again;
1829
+		LOG(L_ERR, "ERROR: tcpconn_put_destroy; close() failed: %s (%d)\n",
1830
+				strerror(errno), errno);
1831
+	}
1832
+}
1833
+
1834
+
1835
+
1836
+/* dec refcnt & frees the connection if refcnt==0
1837
+ * returns 1 if the connection is freed, 0 otherwise
1838
+ *
1839
+ * WARNING: use only from child processes */
1840
+inline static int tcpconn_chld_put(struct tcp_connection* tcpconn)
1841
+{
1842
+	if (unlikely(atomic_dec_and_test(&tcpconn->refcnt))){
1843
+		DBG("tcpconn_put_chld: destroying connection %p (%d, %d) "
1844
+				"flags %04x\n", tcpconn, tcpconn->id,
1845
+				tcpconn->s, tcpconn->flags);
1846
+		/* sanity checks */
1847
+		if (unlikely(!(tcpconn->flags & F_CONN_FD_CLOSED) || 
1848
+					 !(tcpconn->flags & F_CONN_REMOVED) || 
1849
+					(tcpconn->flags & 
1850
+					 		(F_CONN_HASHED| F_CONN_WRITE_W)) )){
1851
+			LOG(L_CRIT, "BUG: tcpconn_put_chld: %p bad flags = %0x\n",
1852
+					tcpconn, tcpconn->flags);
1853
+			abort();
1854
+		}
1855
+		_tcpconn_free(tcpconn); /* destroys also the wbuf_q if still present*/
1856
+		return 1;
1857
+	}
1858
+	return 0;
1859
+}
1860
+
1861
+
1862
+
1863
+/* simple destroy function (the connection should be already removed
1864
+ * from the hashes and the fds should not be watched anymore for IO)
1865
+ */
1866
+inline static void tcpconn_destroy(struct tcp_connection* tcpconn)
1867
+{
1868
+		DBG("tcpconn_destroy: destroying connection %p (%d, %d) "
1869
+				"flags %04x\n", tcpconn, tcpconn->id,
1870
+				tcpconn->s, tcpconn->flags);
1871
+		if (unlikely(tcpconn->flags & F_CONN_HASHED)){
1872
+			LOG(L_CRIT, "BUG: tcpconn_destroy: called with hashed"
1873
+						" connection (%p)\n", tcpconn);
1874
+			/* try to continue */
1875
+			local_timer_del(&tcp_main_ltimer, &tcpconn->timer);
1876
+			TCPCONN_LOCK;
1877
+				_tcpconn_detach(tcpconn);
1878
+			TCPCONN_UNLOCK;
1879
+		}
1880
+		if (likely(!(tcpconn->flags & F_CONN_FD_CLOSED))){
1881
+			tcpconn_close_main_fd(tcpconn);
1882
+			(*tcp_connections_no)--;
1883
+		}
1884
+		_tcpconn_free(tcpconn); /* destroys also the wbuf_q if still present*/
1885
+}
1886
+
1887
+
1888
+
1889
+/* tries to destroy the connection: dec. refcnt and if 0 destroys the
1890
+ *  connection, else it will mark it as BAD and close the main fds
1891
+ *
1892
+ * returns 1 if the connection was destroyed, 0 otherwise
1893
+ *
1894
+ * WARNING: - the connection _has_ to be removed from the hash and timer
1895
+ *  first (use tcpconn_try_unhash() for this )
1896
+ *         - the fd should not be watched anymore (io_watch_del()...)
1897
+ *         - must be called _only_ from the tcp_main process context
1898
+ *          (or else the fd will remain open)
1899
+ */
1900
+inline static int tcpconn_put_destroy(struct tcp_connection* tcpconn)
1901
+{
1902
+	if (unlikely((tcpconn->flags & F_CONN_WRITE_W) ||
1903
+				!(tcpconn->flags & F_CONN_REMOVED))){
1904
+		/* sanity check */
1905
+		LOG(L_CRIT, "BUG: tcpconn_put_destroy: %p flags = %0x\n",
1906
+					tcpconn, tcpconn->flags);
1907
+	}
1908
+	if (unlikely(atomic_dec_and_test(&tcpconn->refcnt))){
1909
+		tcpconn_destroy(tcpconn);
1910
+		return 1;
1911
+	}else{
1912
+		if (likely(!(tcpconn->flags & F_CONN_FD_CLOSED))){
1913
+			tcpconn_close_main_fd(tcpconn);
1914
+			tcpconn->flags|=F_CONN_FD_CLOSED;
1915
+			(*tcp_connections_no)--;
1916
+		}
1917
+		tcpconn->state=S_CONN_BAD;
1918
+		/* in case it's still in a reader timer */
1919
+		tcpconn->timeout=get_ticks_raw();
1920
+	}
1921
+	return 0;
1922
+}
1923
+
1924
+
1925
+
1926
+/* try to remove a connection from the hashes and timer.
1927
+ * returns 1 if the connection was removed, 0 if not (connection not in
1928
+ *  hash)
1929
+ *
1930
+ * WARNING: call it only in the  tcp_main process context or else the
1931
+ *  timer removal won't work.
1932
+ */
1933
+inline static int tcpconn_try_unhash(struct tcp_connection* tcpconn)
1934
+{
1935
+	if (unlikely((tcpconn->flags & F_CONN_WRITE_W) ||
1936
+				!(tcpconn->flags & F_CONN_REMOVED))){
1937
+		/* sanity check */
1938
+		LOG(L_CRIT, "BUG: tcpconn_put_destroy: %p flags = %0x\n",
1939
+					tcpconn, tcpconn->flags);
1940
+	}
1941
+	if (likely(tcpconn->flags & F_CONN_HASHED)){
1942
+		tcpconn->flags&=~F_CONN_HASHED;
1943
+		tcpconn->state=S_CONN_BAD;
1944
+		if (likely(!(tcpconn->flags & F_CONN_READER)))
1945
+			local_timer_del(&tcp_main_ltimer, &tcpconn->timer);
1946
+		else
1947
+			/* in case it's still in a reader timer */
1948
+			tcpconn->timeout=get_ticks_raw();
1949
+		TCPCONN_LOCK;
1950
+			_tcpconn_detach(tcpconn);
1951
+		TCPCONN_UNLOCK;
1952
+#ifdef TCP_BUF_WRITE
1953
+		/* empty possible write buffers (optional) */
1954
+		if (unlikely(_wbufq_non_empty(tcpconn))){
1955
+			lock_get(&tcpconn->write_lock);
1956
+				/* check again, while holding the lock */
1957
+				if (likely(_wbufq_non_empty(tcpconn)))
1958
+					_wbufq_destroy(&tcpconn->wbuf_q);
1959
+			lock_release(&tcpconn->write_lock);
1960
+		}
1961
+#endif /* TCP_BUF_WRITE */
1962
+		return 1;
1963
+	}
1964
+	return 0;
1965
+}
1805 1966
 
1806 1967
 
1807 1968
 
... ...
@@ -1935,7 +2113,9 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
1935 1935
 				}
1936 1936
 #endif
1937 1937
 				p->tcp_conn->flags &= ~F_CONN_READER;
1938
-				tcpconn_destroy(p->tcp_conn);
1938
+				if (likely(tcpconn_try_unhash(p->tcp_conn)))
1939
+					tcpconn_put(p->tcp_conn);
1940
+				tcpconn_put_destroy(p->tcp_conn); /* dec refcnt & destroy */
1939 1941
 			}
1940 1942
 		}
1941 1943
 	}
... ...
@@ -2054,6 +2234,10 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2054 2054
 	switch(cmd){
2055 2055
 		case CONN_RELEASE:
2056 2056
 			tcp_c->busy--;
2057
+			if (unlikely(tcpconn_put(tcpconn))){
2058
+				tcpconn_destroy(tcpconn);
2059
+				break;
2060
+			}
2057 2061
 			if (unlikely(tcpconn->state==S_CONN_BAD)){ 
2058 2062
 #ifdef TCP_BUF_WRITE
2059 2063
 				if (unlikely(tcpconn->flags & F_CONN_WRITE_W)){
... ...
@@ -2061,13 +2245,13 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2061 2061
 					tcpconn->flags &= ~F_CONN_WRITE_W;
2062 2062
 				}
2063 2063
 #endif /* TCP_BUF_WRITE */
2064
-				tcpconn_destroy(tcpconn);
2064
+				if (tcpconn_try_unhash(tcpconn))
2065
+					tcpconn_put_destroy(tcpconn);
2065 2066
 				break;
2066 2067
 			}
2067 2068
 			/* update the timeout*/
2068 2069
 			t=get_ticks_raw();
2069 2070
 			tcpconn->timeout=t+tcp_con_lifetime;
2070
-			tcpconn_put(tcpconn);
2071 2071
 			crt_timeout=tcp_con_lifetime;
2072 2072
 #ifdef TCP_BUF_WRITE
2073 2073
 			if (unlikely(tcp_options.tcp_buf_write && 
... ...
@@ -2081,7 +2265,8 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2081 2081
 						io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
2082 2082
 						tcpconn->flags&=~F_CONN_WRITE_W;
2083 2083
 					}
2084
-					tcpconn_destroy(tcpconn); /* closes also the fd */
2084
+					if (tcpconn_try_unhash(tcpconn))
2085
+						tcpconn_put_destroy(tcpconn);
2085 2086
 					break;
2086 2087
 				}else{
2087 2088
 					crt_timeout=MIN_unsigned(tcp_con_lifetime,
... ...
@@ -2111,7 +2296,9 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2111 2111
 					tcpconn->flags&=~F_CONN_WRITE_W;
2112 2112
 				}
2113 2113
 #endif /* TCP_BUF_WRITE */
2114
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2114
+				if (tcpconn_try_unhash(tcpconn));
2115
+					tcpconn_put_destroy(tcpconn);
2116
+				break;
2115 2117
 			}
2116 2118
 			DBG("handle_tcp_child: CONN_RELEASE  %p refcnt= %d\n", 
2117 2119
 							tcpconn, atomic_get(&tcpconn->refcnt));
... ...
@@ -2131,7 +2318,9 @@ inline static int handle_tcp_child(struct tcp_child* tcp_c, int fd_i)
2131 2131
 					tcpconn->flags&=~F_CONN_WRITE_W;
2132 2132
 				}
2133 2133
 #endif /* TCP_BUF_WRITE */
2134
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2134
+				if (tcpconn_try_unhash(tcpconn))
2135
+					tcpconn_put(tcpconn);
2136
+				tcpconn_put_destroy(tcpconn); /* deref & delete if refcnt==0 */
2135 2137
 				break;
2136 2138
 		default:
2137 2139
 				LOG(L_CRIT, "BUG: handle_tcp_child:  unknown cmd %d"
... ...
@@ -2236,12 +2425,14 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2236 2236
 			LOG(L_ERR, "handle_ser_child: ERROR: received CON_ERROR for %p"
2237 2237
 					" (id %d), refcnt %d\n", 
2238 2238
 					tcpconn, tcpconn->id, atomic_get(&tcpconn->refcnt));
2239
-			tcpconn_destroy(tcpconn); /* will close also the fd */
2239
+			if (tcpconn_try_unhash(tcpconn))
2240
+				tcpconn_put(tcpconn);
2241
+			tcpconn_put_destroy(tcpconn); /* dec refcnt & destroy on 0 */
2240 2242
 			break;
2241 2243
 		case CONN_GET_FD:
2242 2244
 			/* send the requested FD  */
2243 2245
 			/* WARNING: take care of setting refcnt properly to
2244
-			 * avoid race condition */
2246
+			 * avoid race conditions */
2245 2247
 			if (unlikely(send_fd(p->unix_sock, &tcpconn, sizeof(tcpconn),
2246 2248
 								tcpconn->s)<=0)){
2247 2249
 				LOG(L_ERR, "ERROR: handle_ser_child: send_fd failed\n");
... ...
@@ -2250,10 +2441,12 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2250 2250
 		case CONN_NEW:
2251 2251
 			/* update the fd in the requested tcpconn*/
2252 2252
 			/* WARNING: take care of setting refcnt properly to
2253
-			 * avoid race condition */
2253
+			 * avoid race conditions */
2254 2254
 			if (unlikely(fd==-1)){
2255 2255
 				LOG(L_CRIT, "BUG: handle_ser_child: CONN_NEW:"
2256 2256
 							" no fd received\n");
2257
+				tcpconn->flags|=F_CONN_FD_CLOSED;
2258
+				tcpconn_put_destroy(tcpconn);
2257 2259
 				break;
2258 2260
 			}
2259 2261
 			(*tcp_connections_no)++;
... ...
@@ -2264,7 +2457,7 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2264 2264
 			t=get_ticks_raw();
2265 2265
 			tcpconn->timeout=t+tcp_con_lifetime;
2266 2266
 			/* activate the timer (already properly init. in tcpconn_new())
2267
-			 * no need for */
2267
+			 * no need for reinit */
2268 2268
 			local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
2269 2269
 								tcp_con_lifetime, t);
2270 2270
 			tcpconn->flags&=~F_CONN_REMOVED;
... ...
@@ -2283,18 +2476,23 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2283 2283
 						" new socket to the fd list\n");
2284 2284
 				tcpconn->flags|=F_CONN_REMOVED;
2285 2285
 				tcpconn->flags&=~F_CONN_WRITE_W;
2286
-				tcpconn_destroy(tcpconn); /* closes also the fd */
2286
+				tcpconn_try_unhash(tcpconn); /*  unhash & dec refcnt */
2287
+				tcpconn_put_destroy(tcpconn);
2287 2288
 			}
2288 2289
 			break;
2289 2290
 #ifdef TCP_BUF_WRITE
2290 2291
 		case CONN_QUEUED_WRITE:
2292
+			if (unlikely((tcpconn->state==S_CONN_BAD) || 
2293
+							!(tcpconn->flags & F_CONN_HASHED) ))
2294
+				break;
2291 2295
 			if (!(tcpconn->flags & F_CONN_WRITE_W)){
2292 2296
 				if (tcpconn->flags& F_CONN_REMOVED){
2293 2297
 					if (unlikely(io_watch_add(&io_h, tcpconn->s, POLLOUT,
2294 2298
 												F_TCPCONN, tcpconn)<0)){
2295 2299
 						LOG(L_CRIT, "ERROR: tcp_main: handle_ser_child: failed"
2296 2300
 								    " to enable write watch on socket\n");
2297
-						tcpconn_destroy(tcpconn);
2301
+						if (tcpconn_try_unhash(tcpconn))
2302
+							tcpconn_put_destroy(tcpconn);
2298 2303
 						break;
2299 2304
 					}
2300 2305
 				}else{
... ...
@@ -2304,7 +2502,8 @@ inline static int handle_ser_child(struct process_table* p, int fd_i)
2304 2304
 								    " to change socket watch events\n");
2305 2305
 						io_watch_del(&io_h, tcpconn->s, -1, IO_FD_CLOSING);
2306 2306
 						tcpconn->flags|=F_CONN_REMOVED;
2307
-						tcpconn_destroy(tcpconn);
2307
+						if (tcpconn_try_unhash(tcpconn))
2308
+							tcpconn_put_destroy(tcpconn);
2308 2309
 						break;
2309 2310
 					}
2310 2311
 				}
... ...
@@ -2474,6 +2673,8 @@ static inline int handle_new_connect(struct socket_info* si)
2474 2474
 	tcpconn=tcpconn_new(new_sock, &su, dst_su, si, si->proto, S_CONN_ACCEPT);
2475 2475
 	if (likely(tcpconn)){
2476 2476
 #ifdef TCP_PASS_NEW_CONNECTION_ON_DATA
2477
+		atomic_set(&tcpconn->refcnt, 1); /* safe, not yet available to the
2478
+											outside world */
2477 2479
 		tcpconn_add(tcpconn);
2478 2480
 		/* activate the timer */
2479 2481
 		local_timer_add(&tcp_main_ltimer, &tcpconn->timer, 
... ...
@@ -2484,21 +2685,24 @@ static inline int handle_new_connect(struct socket_info* si)
2484 2484
 			LOG(L_CRIT, "ERROR: tcp_main: handle_new_connect: failed to add"
2485 2485
 						" new socket to the fd list\n");
2486 2486
 			tcpconn->flags|=F_CONN_REMOVED;
2487
-			tcpconn_destroy(tcpconn); /* closes also the fd */
2487
+			if (tcpconn_try_unhash(tcpconn))
2488
+				tcpconn_put_destroy(tcpconn);
2488 2489
 		}
2489 2490
 #else
2490
-		atomic_set(&tcpconn->refcnt, 1); /* safe, not yet available to the
2491
+		atomic_set(&tcpconn->refcnt, 2); /* safe, not yet available to the
2491 2492
 											outside world */
2493
+		/* prepare it for passing to a child */
2494
+		tcpconn->flags|=F_CONN_READER;
2492 2495
 		tcpconn_add(tcpconn);
2493 2496
 		DBG("handle_new_connect: new connection: %p %d flags: %04x\n",
2494 2497
 			tcpconn, tcpconn->s, tcpconn->flags);
2495
-		/* pass it to a child */
2496
-		tcpconn->flags|=F_CONN_READER;
2497 2498
 		if(unlikely(send2child(tcpconn)<0)){
2498 2499
 			LOG(L_ERR,"ERROR: handle_new_connect: no children "
2499 2500
 					"available\n");
2500 2501
 			tcpconn->flags&=~F_CONN_READER;
2501
-			tcpconn_destroy(tcpconn);
2502
+			tcpconn_put(tcpconn);
2503
+			tcpconn_try_unhash(tcpconn);
2504
+			tcpconn_put_destroy(tcpconn);
2502 2505
 		}
2503 2506
 #endif
2504 2507
 	}else{ /*tcpconn==0 */
... ...
@@ -2554,7 +2758,11 @@ inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, short ev,
2554 2554
 			io_watch_del(&io_h, tcpconn->s, fd_i, 0);
2555 2555
 			tcpconn->flags|=F_CONN_REMOVED;
2556 2556
 			tcpconn->flags&=~F_CONN_WRITE_W;
2557
-			tcpconn_destroy(tcpconn);
2557
+			if (unlikely(!tcpconn_try_unhash(tcpconn))){
2558
+				LOG(L_CRIT, "BUG: tcpconn_ev: unhashed connection %p\n",
2559
+							tcpconn);
2560
+			}
2561
+			tcpconn_put_destroy(tcpconn);
2558 2562
 			goto error;
2559 2563
 		}
2560 2564
 		if (empty_q){
... ...
@@ -2596,7 +2804,9 @@ inline static int handle_tcpconn_ev(struct tcp_connection* tcpconn, short ev,
2596 2596
 				tcpconn->flags&=~F_CONN_WRITE_W;
2597 2597
 			}
2598 2598
 #endif /* TCP_BUF_WRITE */
2599
-			tcpconn_destroy(tcpconn);
2599
+			tcpconn_put(tcpconn);
2600
+			tcpconn_try_unhash(tcpconn); 
2601
+			tcpconn_put_destroy(tcpconn); /* because of the tcpconn_ref() */
2600 2602
 		}
2601 2603
 	}
2602 2604
 	return 0; /* we are not interested in possibly queued io events, 
... ...
@@ -2687,45 +2897,33 @@ static ticks_t tcpconn_main_timeout(ticks_t t, struct timer_ln* tl, void* data)
2687 2687
 	}
2688 2688
 #endif /* TCP_BUF_WRITE */
2689 2689
 	DBG("tcp_main: timeout for %p\n", c);
2690
-	if (likely(atomic_get(&c->refcnt)==0)){
2690
+	if (likely(c->flags & F_CONN_HASHED)){
2691
+		c->flags&=~F_CONN_HASHED;
2692
+		c->state=S_CONN_BAD;
2691 2693
 		TCPCONN_LOCK;
2692
-			/* check again to avoid races with tcp_send() */
2693
-			if (likely(atomic_get(&c->refcnt)==0)){
2694
-				/* delete */
2695
-				_tcpconn_detach(c);
2696
-				TCPCONN_UNLOCK; /* unlock as soon as possible */
2697
-				fd=c->s;
2698
-				if (likely(fd>0)){
2699
-					if (likely(!(c->flags & F_CONN_REMOVED)
2694
+			_tcpconn_detach(c);
2695
+		TCPCONN_UNLOCK;
2696
+	}else{
2697
+		LOG(L_CRIT, "BUG: tcp_main: timer: called with unhashed connection %p"
2698
+				"\n", c);
2699
+		tcpconn_ref(c); /* ugly hack to try to go on */
2700
+	}
2701
+	fd=c->s;
2702
+	if (likely(fd>0)){
2703
+		if (likely(!(c->flags & F_CONN_REMOVED)
2700 2704
 #ifdef TCP_BUF_WRITE
2701
-								|| (c->flags & F_CONN_WRITE_W)
2705
+			|| (c->flags & F_CONN_WRITE_W)
2702 2706
 #endif /* TCP_BUF_WRITE */
2703
-								)){
2704
-						io_watch_del(&io_h, fd, -1, IO_FD_CLOSING);
2705
-						c->flags|=F_CONN_REMOVED;
2707
+			)){
2708
+			io_watch_del(&io_h, fd, -1, IO_FD_CLOSING);
2709
+			c->flags|=F_CONN_REMOVED;
2706 2710
 #ifdef TCP_BUF_WRITE
2707
-						c->flags&=~F_CONN_WRITE_W;
2711
+			c->flags&=~F_CONN_WRITE_W;
2708 2712
 #endif /* TCP_BUF_WRITE */
2709
-					}
2710
-#ifdef USE_TLS
2711
-					if (unlikely(c->type==PROTO_TLS ))
2712
-						tls_close(c, fd);
2713
-#endif /* USE_TLS */
2714
-					_tcpconn_free(c);
2715
-#ifdef TCP_FD_CACHE
2716
-					if (likely(tcp_options.fd_cache)) shutdown(fd, SHUT_RDWR);
2717
-#endif /* TCP_FD_CACHE */
2718
-					close(fd);
2719
-				}
2720
-				(*tcp_connections_no)--; /* modified only in tcp_main
2721
-											 => no lock needed */
2722
-				return 0; /* don't prolong the timer anymore */
2723
-			}
2724
-		TCPCONN_UNLOCK;
2713
+		}
2725 2714
 	}
2726
-	/* if we are here we can't delete the connection, it's still referenced
2727
-	 *  => we just delay deleting it */
2728
-	return TCPCONN_WAIT_TIMEOUT;
2715
+	tcpconn_put_destroy(c);
2716
+	return 0;
2729 2717
 }
2730 2718
 
2731 2719