Browse code

core: Add PROXY protocol implementation (#1765)

- introduce new global variable `ksr_tcp_accept_haproxy`.
- this variable can be modified by using the `tcp_accept_haproxy=yes` core
configuration parameter.
- when active, inbound TCP connections are expected to behave according
to the PROXY protocol[1].
- Both the v1 (human-readable) and v2 (binary) versions of the protocol
are supported.
- [1]: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt

Sebastian Lauwers authored on 19/12/2018 08:05:14 • Henning Westerholt committed on 19/12/2018 08:05:14
Showing 5 changed files
... ...
@@ -399,6 +399,7 @@ TCP_OPT_KEEPCNT		"tcp_keepcnt"
399 399
 TCP_OPT_CRLF_PING	"tcp_crlf_ping"
400 400
 TCP_OPT_ACCEPT_NO_CL	"tcp_accept_no_cl"
401 401
 TCP_OPT_ACCEPT_HEP3	"tcp_accept_hep3"
402
+TCP_OPT_ACCEPT_HAPROXY	"tcp_accept_haproxy"
402 403
 TCP_CLONE_RCVBUF	"tcp_clone_rcvbuf"
403 404
 TCP_REUSE_PORT		"tcp_reuse_port"
404 405
 DISABLE_TLS		"disable_tls"|"tls_disable"
... ...
@@ -856,6 +857,8 @@ IMPORTFILE      "import_file"
856 857
 									return TCP_OPT_ACCEPT_NO_CL; }
857 858
 <INITIAL>{TCP_OPT_ACCEPT_HEP3}	{ count(); yylval.strval=yytext;
858 859
 									return TCP_OPT_ACCEPT_HEP3; }
860
+<INITIAL>{TCP_OPT_ACCEPT_HAPROXY}	{ count(); yylval.strval=yytext;
861
+									return TCP_OPT_ACCEPT_HAPROXY; }
859 862
 <INITIAL>{TCP_CLONE_RCVBUF}		{ count(); yylval.strval=yytext;
860 863
 									return TCP_CLONE_RCVBUF; }
861 864
 <INITIAL>{TCP_REUSE_PORT}	{ count(); yylval.strval=yytext; return TCP_REUSE_PORT; }
... ...
@@ -433,6 +433,7 @@ extern char *default_routename;
433 433
 %token TCP_OPT_CRLF_PING
434 434
 %token TCP_OPT_ACCEPT_NO_CL
435 435
 %token TCP_OPT_ACCEPT_HEP3
436
+%token TCP_OPT_ACCEPT_HAPROXY
436 437
 %token TCP_CLONE_RCVBUF
437 438
 %token TCP_REUSE_PORT
438 439
 %token DISABLE_TLS
... ...
@@ -1230,6 +1231,14 @@ assign_stm:
1230 1231
 		#endif
1231 1232
 	}
1232 1233
 	| TCP_OPT_ACCEPT_HEP3 EQUAL error { yyerror("boolean value expected"); }
1234
+	| TCP_OPT_ACCEPT_HAPROXY EQUAL NUMBER {
1235
+		#ifdef USE_TCP
1236
+			ksr_tcp_accept_haproxy=$3;
1237
+		#else
1238
+			warn("tcp support not compiled in");
1239
+		#endif
1240
+	}
1241
+	| TCP_OPT_ACCEPT_HAPROXY EQUAL error { yyerror("boolean value expected"); }
1233 1242
 
1234 1243
 	| TCP_CLONE_RCVBUF EQUAL NUMBER {
1235 1244
 		#ifdef USE_TCP
... ...
@@ -100,6 +100,7 @@ extern int tcp_max_connections; /* maximum tcp connections, hard limit */
100 100
 extern int tls_max_connections; /* maximum tls connections, hard limit */
101 101
 #endif
102 102
 extern int ksr_tcp_accept_hep3;
103
+extern int ksr_tcp_accept_haproxy;
103 104
 #ifdef USE_TLS
104 105
 extern int tls_disable;
105 106
 extern unsigned short tls_port_no;
... ...
@@ -46,6 +46,7 @@
46 46
 #define BSD_COMP  /* needed on older solaris for FIONREAD */
47 47
 #endif /* HAVE_FILIO_H / __OS_solaris */
48 48
 #include <sys/ioctl.h>  /* ioctl() used on write error */
49
+#include <arpa/inet.h>  /* for inet_pton() */
49 50
 #include <netinet/in.h>
50 51
 #include <netinet/in_systm.h>
51 52
 #include <netinet/ip.h>
... ...
@@ -957,7 +958,227 @@ end:
957 958
 }
958 959
 #endif
959 960
 
961
+/* Attempt to extract real connection information from an upstream load
962
+ * balancer or reverse proxy. This should be called right after accept()ing the
963
+ * connection, and before TLS negotiation.
964
+ *
965
+ * Returns:
966
+ *    -1 on parsing error (connection should be closed)
967
+ *    0 on parser success, and connection information was extracted
968
+ *    1 on parser success, but no connection information was provided by the
969
+ *      upstream load balancer or reverse proxy.
970
+ */
971
+int tcpconn_read_haproxy(struct tcp_connection *c) {
972
+	int bytes, retval = 0;
973
+	uint32_t size, port;
974
+	char *p, *end;
975
+	struct ip_addr *src_ip, *dst_ip;
976
+
977
+	const char v2sig[12] = "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A";
978
+
979
+	// proxy header union
980
+	union {
981
+		// v1 struct
982
+		struct {
983
+			char line[108];
984
+		} v1;
985
+
986
+		// v2 struct
987
+		struct {
988
+			uint8_t sig[12];
989
+			uint8_t ver_cmd;
990
+			uint8_t fam;
991
+			uint16_t len;
992
+
993
+			union {
994
+				struct { /* for TCP/UDP over IPv4, len = 12 */
995
+					uint32_t src_addr;
996
+					uint32_t dst_addr;
997
+					uint16_t src_port;
998
+					uint16_t dst_port;
999
+				} ip4;
1000
+
1001
+				struct { /* for TCP/UDP over IPv6, len = 36 */
1002
+					 uint8_t  src_addr[16];
1003
+					 uint8_t  dst_addr[16];
1004
+					 uint16_t src_port;
1005
+					 uint16_t dst_port;
1006
+				} ip6;
1007
+
1008
+				struct { /* for AF_UNIX sockets, len = 216 */
1009
+					 uint8_t src_addr[108];
1010
+					 uint8_t dst_addr[108];
1011
+				} unx;
1012
+			} addr;
1013
+		} v2;
1014
+
1015
+	} hdr;
1016
+
1017
+	do {
1018
+		bytes = recv(c->s, &hdr, sizeof(hdr), MSG_PEEK);
1019
+	} while (bytes == -1 && (errno == EINTR || errno == EAGAIN));
1020
+
1021
+	src_ip = &c->rcv.src_ip;
1022
+	dst_ip = &c->rcv.dst_ip;
1023
+
1024
+	if (bytes >= 16 && memcmp(&hdr.v2, v2sig, 12) == 0 &&
1025
+		(hdr.v2.ver_cmd & 0xF0) == 0x20) {
1026
+		LM_DBG("received PROXY protocol v2 header\n");
1027
+		size = 16 + ntohs(hdr.v2.len);
1028
+
1029
+		if (bytes < size) {
1030
+			return -1; /* truncated or too large header */
1031
+		}
1032
+
1033
+		switch (hdr.v2.ver_cmd & 0xF) {
1034
+			case 0x01: /* PROXY command */
1035
+				switch (hdr.v2.fam) {
1036
+					case 0x11: /* TCPv4 */
1037
+						src_ip->af = AF_INET;
1038
+						src_ip->len = 4;
1039
+						src_ip->u.addr32[0] =
1040
+							hdr.v2.addr.ip4.src_addr;
1041
+						c->rcv.src_port =
1042
+							hdr.v2.addr.ip4.src_port;
1043
+
1044
+						dst_ip->af = AF_INET;
1045
+						dst_ip->len = 4;
1046
+						dst_ip->u.addr32[0] =
1047
+							hdr.v2.addr.ip4.dst_addr;
1048
+						c->rcv.dst_port =
1049
+							hdr.v2.addr.ip4.dst_port;
1050
+
1051
+						goto done;
1052
+
1053
+					case 0x21: /* TCPv6 */
1054
+						src_ip->af = AF_INET6;
1055
+						src_ip->len = 16;
1056
+						memcpy(src_ip->u.addr,
1057
+							hdr.v2.addr.ip6.src_addr, 16);
1058
+						c->rcv.src_port =
1059
+							hdr.v2.addr.ip6.src_port;
1060
+
1061
+						dst_ip->af = AF_INET6;
1062
+						dst_ip->len = 16;
1063
+						memcpy(dst_ip->u.addr,
1064
+							hdr.v2.addr.ip6.src_addr, 16);
1065
+						c->rcv.dst_port =
1066
+							hdr.v2.addr.ip6.dst_port;
1067
+
1068
+						goto done;
1069
+
1070
+					default: /* unsupported protocol */
1071
+						return -1;
1072
+				}
1073
+
1074
+			case 0x00: /* LOCAL command */
1075
+				retval = 1; /* keep local connection address for LOCAL */
1076
+				goto done;
1077
+
1078
+			default:
1079
+				return -1; /* not a supported command */
1080
+		}
1081
+	}
1082
+	else if (bytes >= 8 && memcmp(hdr.v1.line, "PROXY", 5) == 0) {
1083
+		LM_DBG("received PROXY protocol v1 header\n");
1084
+		end = memchr(hdr.v1.line, '\r', bytes - 1);
1085
+		if (!end || end[1] != '\n') {
1086
+			return -1; /* partial or invalid header */
1087
+		}
1088
+		*end = '\0'; /* terminate the string to ease parsing */
1089
+		size = end + 2 - hdr.v1.line;
1090
+		p = hdr.v1.line + 5;
1091
+
1092
+		if (strncmp(p, " TCP", 4) == 0) {
1093
+			switch (p[4]) {
1094
+				case '4':
1095
+					src_ip->af  = dst_ip->af  = AF_INET;
1096
+					src_ip->len = dst_ip->len = 4;
1097
+					break;
1098
+				case '6':
1099
+					src_ip->af  = dst_ip->af  = AF_INET6;
1100
+					src_ip->len = dst_ip->len = 16;
1101
+					break;
1102
+				default:
1103
+					return -1; /* unknown TCP version */
1104
+			}
1105
+
1106
+			if (p[5] != ' ') {
1107
+				return -1; /* misformatted header */
1108
+			}
1109
+			p += 6; /* skip over the already-parsed bytes */
1110
+
1111
+			/* Parse the source IP address */
1112
+			end = strchr(p, ' ');
1113
+			if (!end) {
1114
+				return -1; /* truncated header */
1115
+			}
1116
+			*end = '\0'; /* mark the end of the IP address */
1117
+			if (inet_pton(src_ip->af, p, src_ip->u.addr) != 1) {
1118
+				return -1; /* missing IP address */
1119
+			}
1120
+			p = end + 1;
1121
+
1122
+			/* Parse the destination IP address */
1123
+			end = strchr(p, ' ');
1124
+			if (!end) {
1125
+				return -1;
1126
+			}
1127
+			*end = '\0'; /* mark the end of the IP address */
1128
+			if (inet_pton(dst_ip->af, p, dst_ip->u.addr) != 1) {
1129
+				return -1;
1130
+			}
1131
+			p = end + 1;
1132
+
1133
+			/* Parse the source port */
1134
+			port = strtoul(p, &end, 10);
1135
+			if (port == ULONG_MAX || port == 0 || port >= (1 << 16)) {
1136
+				return -1; /* invalid port number */
1137
+			}
1138
+			c->rcv.src_port = port;
1139
+
1140
+			if (*end != ' ') {
1141
+				return -1; /* invalid header */
1142
+			}
1143
+			p = end + 1;
1144
+
1145
+			/* Parse the destination port */
1146
+			port = strtoul(p, NULL, 10);
1147
+			if (port == ULONG_MAX || port == 0 || port >= (1 << 16)) {
1148
+				return -1; /* invalid port number */
1149
+			}
1150
+			c->rcv.dst_port = port;
1151
+
1152
+			goto done;
1153
+		}
1154
+		else if (strncmp(p, " UNKNOWN", 8) == 0) {
1155
+			/* We know that the sender speaks the correct PROXY protocol with the
1156
+			 * appropriate version, and we SHOULD accept the connection and use the
1157
+			 * real connection's parameters as if there were no PROXY protocol header
1158
+			 * on the wire.
1159
+			 */
1160
+			retval = 1; /* PROXY protocol parsed, but no IP override */
1161
+			goto done;
1162
+		}
1163
+		else {
1164
+			return -1; /* invalid header */
1165
+		}
1166
+	} else if (bytes == 0) {
1167
+		return 1; /* EOF? Return "no IP change" in any case */
1168
+	}
1169
+	else {
1170
+		/* Wrong protocol */
1171
+		return -1;
1172
+	}
960 1173
 
1174
+done:
1175
+	/* we need to consume the appropriate amount of data from the socket */
1176
+	do {
1177
+		bytes = recv(c->s, &hdr, size, 0);
1178
+	} while (bytes == -1 && errno == EINTR);
1179
+
1180
+	return (bytes >= 0) ? retval : -1;
1181
+}
961 1182
 
962 1183
 struct tcp_connection* tcpconn_new(int sock, union sockaddr_union* su,
963 1184
 									union sockaddr_union* local_addr,
... ...
@@ -965,7 +1186,7 @@ struct tcp_connection* tcpconn_new(int sock, union sockaddr_union* su,
965 1186
 									int state)
966 1187
 {
967 1188
 	struct tcp_connection *c;
968
-	int rd_b_size;
1189
+	int rd_b_size, ret;
969 1190
 
970 1191
 	rd_b_size=cfg_get(tcp, tcp_cfg, rd_buf_size);
971 1192
 	c=shm_malloc(sizeof(struct tcp_connection) + rd_b_size);
... ...
@@ -985,16 +1206,29 @@ struct tcp_connection* tcpconn_new(int sock, union sockaddr_union* su,
985 1206
 
986 1207
 	atomic_set(&c->refcnt, 0);
987 1208
 	local_timer_init(&c->timer, tcpconn_main_timeout, c, 0);
988
-	su2ip_addr(&c->rcv.src_ip, su);
989
-	c->rcv.src_port=su_getport(su);
990
-	c->rcv.bind_address=ba;
991
-	if (likely(local_addr)){
992
-		su2ip_addr(&c->rcv.dst_ip, local_addr);
993
-		c->rcv.dst_port=su_getport(local_addr);
994
-	}else if (ba){
995
-		c->rcv.dst_ip=ba->address;
996
-		c->rcv.dst_port=ba->port_no;
1209
+	if (unlikely(ksr_tcp_accept_haproxy && state == S_CONN_ACCEPT)) {
1210
+		ret = tcpconn_read_haproxy(c);
1211
+
1212
+		if (ret == -1) {
1213
+			LM_ERR("invalid PROXY protocol header\n");
1214
+			goto error;
1215
+		} else if (ret == 1) {
1216
+			LM_DBG("PROXY protocol did not override IP addresses\n");
1217
+			goto read_ip_info;
1218
+		}
1219
+	} else {
1220
+read_ip_info:
1221
+		su2ip_addr(&c->rcv.src_ip, su);
1222
+		c->rcv.src_port=su_getport(su);
1223
+		if (likely(local_addr)){
1224
+			su2ip_addr(&c->rcv.dst_ip, local_addr);
1225
+			c->rcv.dst_port=su_getport(local_addr);
1226
+		}else if (ba){
1227
+			c->rcv.dst_ip=ba->address;
1228
+			c->rcv.dst_port=ba->port_no;
1229
+		}
997 1230
 	}
1231
+	c->rcv.bind_address=ba;
998 1232
 	print_ip("tcpconn_new: new tcp connection: ", &c->rcv.src_ip, "\n");
999 1233
 	LM_DBG("on port %d, type %d\n", c->rcv.src_port, type);
1000 1234
 	init_tcp_req(&c->req, (char*)c+sizeof(struct tcp_connection), rd_b_size);
... ...
@@ -39,7 +39,6 @@
39 39
 #include <unistd.h>
40 40
 #include <stdlib.h> /* for abort() */
41 41
 
42
-
43 42
 #include "dprint.h"
44 43
 #include "tcp_conn.h"
45 44
 #include "tcp_read.h"
... ...
@@ -93,6 +92,7 @@ static ticks_t tcp_reader_prev_ticks;
93 92
 int is_msg_complete(struct tcp_req* r);
94 93
 
95 94
 int ksr_tcp_accept_hep3=0;
95
+int ksr_tcp_accept_haproxy=0;
96 96
 /**
97 97
  * control cloning of TCP receive buffer
98 98
  * - needed for operations working directly inside the buffer