Browse code

db_mysql: Fixes crash in libmysqlclient after connection reset.

Sometimes SIP-Router would crash in libmysqlclient after a connection
to the server has been reset. This is caused by mysql_stmt_prepare
which will reset the connection data structure if a connection has been
reset. Subsequent calls to other mysql functions (mysql_stmt_execute)
crash unless the connection has been re-connected. This is documented
as mysql bug #33384.

A workaround is to reset and reconnect the connection explicitly
immediately after mysql_stmt_prepare has failed with
CR_SERVER_GONE_ERROR. This change implements exactly that.

First of all, this patch fixes a minor bug in updating the variable
that keeps track of number of connection resets for each database
connection and pre-pared statement. The variable needs to be
incremented each time a connection is closed. Previously it was
incremented only if a connection was successfully reconnected. If the
reconnect attempt failed than the variable was not incremented. The
function that uploads commands to the server relies on the variable
to detect connection resets and may not have worked properly under
some circumstances (if a connection fails to reconnect).

Function upload_cmd has been modified to close the connection
explicitly if mysql_stmt_prepare fails with CR_SERVER_GONE_ERROR. This
ensures that subsequent calls to mysql_stmt_exec are forced to reconnect
and re-upload commands to the server. This is needed to prevent the
library from crashing.

exec_cmd_safe now checks if a connection has been disconnected and if
so it tries to reconnect it before executing a prepared statement. This
is used to recover from failing mysql_stmt_prepare.

Jan Janak authored on 17/11/2010 23:26:32
Showing 3 changed files
... ...
@@ -671,72 +671,79 @@ static inline int update_result(db_fld_t* result, MYSQL_STMT* st)
671 671
  */
672 672
 static int exec_cmd_safe(db_cmd_t* cmd)
673 673
 {
674
-	int i, err;
675
-	db_con_t* con;
676
-	struct my_cmd* mcmd;
677
-	struct my_con* mcon;
678
-
679
-	/* First things first: retrieve connection info
680
-	 * from the currently active connection and also
681
-	 * mysql payload from the database command
674
+    int i, err;
675
+    db_con_t* con;
676
+    struct my_cmd* mcmd;
677
+    struct my_con* mcon;
678
+	
679
+    /* First things first: retrieve connection info
680
+     * from the currently active connection and also
681
+     * mysql payload from the database command
682
+     */
683
+    mcmd = DB_GET_PAYLOAD(cmd);
684
+    con = cmd->ctx->con[db_payload_idx];
685
+    mcon = DB_GET_PAYLOAD(con);
686
+    
687
+    for(i = 0; i <= my_retries; i++) {
688
+	if ((mcon->flags & MY_CONNECTED) == 0) {
689
+	    /* The connection is disconnected, try to reconnect */
690
+	    if (my_con_connect(con)) {
691
+		INFO("mysql: exec_cmd_safe failed to re-connect\n");
692
+		continue;
693
+	    }
694
+	}	
695
+	
696
+	/* Next check the number of resets in the database connection, if this
697
+	 * number is higher than the number we keep in my_cmd structure in
698
+	 * last_reset variable then the connection was reset and we need to
699
+	 * upload the command again to the server before executing it, because
700
+	 * the server recycles all server side information upon disconnect.
682 701
 	 */
683
-	mcmd = DB_GET_PAYLOAD(cmd);
684
-	con = cmd->ctx->con[db_payload_idx];
685
-	mcon = DB_GET_PAYLOAD(con);
686
-
687
-	for(i = 0; i <= my_retries; i++) {
688
-		/* Next check the number of resets in the database connection,
689
-		 * if this number is higher than the number we keep in my_cmd
690
-		 * structure in last_reset variable then the connection was
691
-		 * reset and we need to upload the command again to the server
692
-		 * before executing it, because the server recycles all server
693
-		 * side information upon disconnect.
702
+	if (mcon->resets > mcmd->last_reset) {
703
+	    INFO("mysql: Connection reset detected, uploading command to server\n");
704
+	    err = upload_cmd(cmd);
705
+	    if (err < 0) {
706
+		INFO("mysql: Error while uploading command\n");
707
+		continue;
708
+	    } else if (err > 0) {
709
+		/* DB API error, this is a serious problem such as memory
710
+		 * allocation failure, bail out
694 711
 		 */
695
-		if (mcon->resets > mcmd->last_reset) {
696
-			INFO("mysql: Connection reset detected, uploading command to server\n");
697
-			err = upload_cmd(cmd);
698
-			if (err < 0) {
699
-				INFO("mysql: Error while uploading command\n");
700
-				/* MySQL error, skip execution and try again if we have attempts left */
701
-				continue;
702
-			} else if (err > 0) {
703
-				/* DB API error, this is a serious problem such
704
-				 * as memory allocation failure, bail out
705
-				 */
706
-				return 1;
707
-			}
708
-		}
709
-
710
-		set_mysql_params(cmd);
711
-		err = mysql_stmt_execute(mcmd->st);
712
-		if (err == 0) {
713
-			/* The command was executed successfully, now fetch all data
714
-			 * to the client if it was requested by the user */
715
-			if (mcmd->flags & MY_FETCH_ALL) {
716
-				err = mysql_stmt_store_result(mcmd->st);
717
-				if (err) {
718
-					INFO("mysql: Error while fetching data to client.\n");
719
-					goto error;
720
-				}
721
-			}
722
-			return 0;
723
-		}
724
-		
725
-	error:
726
-		/* Command execution failed, log a message and try to reconnect */
727
-		INFO("mysql: libmysql: %d, %s\n", mysql_stmt_errno(mcmd->st),
728
-			 mysql_stmt_error(mcmd->st));
729
-		INFO("mysql: Error while executing command on server, trying to reconnect\n");
730
-		my_con_disconnect(con);
731
-		if (my_con_connect(con)) {
732
-			INFO("mysql: Failed to reconnect server\n");
733
-		} else {
734
-			INFO("mysql: Successfully reconnected server\n");
712
+		return 1;
713
+	    }
714
+	}
715
+	
716
+	set_mysql_params(cmd);
717
+	err = mysql_stmt_execute(mcmd->st);
718
+	if (err == 0) {
719
+	    /* The command was executed successfully, now fetch all data to
720
+	     * the client if it was requested by the user */
721
+	    if (mcmd->flags & MY_FETCH_ALL) {
722
+		err = mysql_stmt_store_result(mcmd->st);
723
+		if (err) {
724
+		    INFO("mysql: Error while fetching data to client.\n");
725
+		    goto error;
735 726
 		}
727
+	    }
728
+	    return 0;
736 729
 	}
737
-
738
-	INFO("mysql: Failed to execute command, giving up\n");
739
-	return -1;
730
+	
731
+    error:
732
+	/* Command execution failed, log a message and try to reconnect */
733
+	INFO("mysql: libmysql: %d, %s\n", mysql_stmt_errno(mcmd->st),
734
+	     mysql_stmt_error(mcmd->st));
735
+	INFO("mysql: Error while executing command on server, trying to reconnect\n");
736
+
737
+	my_con_disconnect(con);
738
+	if (my_con_connect(con)) {
739
+	    INFO("mysql: Failed to reconnect server\n");
740
+	} else {
741
+	    INFO("mysql: Successfully reconnected server\n");
742
+	}
743
+    }
744
+    
745
+    INFO("mysql: Failed to execute command, giving up\n");
746
+    return -1;
740 747
 }
741 748
 
742 749
 
... ...
@@ -1089,58 +1096,73 @@ static int bind_result(MYSQL_STMT* st, db_fld_t* fld)
1089 1096
  */
1090 1097
 static int upload_cmd(db_cmd_t* cmd)
1091 1098
 {
1092
-	struct my_cmd* res;
1093
-	struct my_con* mcon;
1094
-	int err = 0;
1095
-
1096
-	res = DB_GET_PAYLOAD(cmd);
1097
-
1098
-	/* FIXME: The function should take the connection as one of parameters */
1099
-	mcon = DB_GET_PAYLOAD(cmd->ctx->con[db_payload_idx]);
1100
-
1101
-	/* If there is a previous pre-compiled statement, close it first */
1102
-	if (res->st) mysql_stmt_close(res->st);
1103
-	res->st = NULL;
1104
-
1105
-	/* Create a new pre-compiled statement data structure */
1106
-	res->st = mysql_stmt_init(mcon->con);
1107
-	if (res->st == NULL) {
1108
-		ERR("mysql: Error while creating new MySQL_STMT data structure (no memory left)\n");
1109
-		err = 1;
1110
-		goto error;
1111
-	}
1112
-
1113
-	/* Try to upload the command to the server */
1114
-	err = mysql_stmt_prepare(res->st, res->sql_cmd.s, res->sql_cmd.len);
1115
-	if (err) {
1116
-		ERR("mysql: libmysql: %d, %s\n", mysql_stmt_errno(res->st), 
1117
-			mysql_stmt_error(res->st));
1118
-		ERR("mysql: An error occurred while uploading a command to MySQL server\n");
1119
-		goto error;
1120
-	}
1121
-
1122
-	err = bind_mysql_params(res->st, cmd->vals, cmd->match);
1099
+    struct my_cmd* res;
1100
+    struct my_con* mcon;
1101
+    int err = 0;
1102
+    
1103
+    res = DB_GET_PAYLOAD(cmd);
1104
+    
1105
+    /* FIXME: The function should take the connection as one of parameters */
1106
+    mcon = DB_GET_PAYLOAD(cmd->ctx->con[db_payload_idx]);
1107
+    /* Do not upload the command if the connection is not connected */
1108
+    if ((mcon->flags & MY_CONNECTED) == 0) {
1109
+	err = 1;
1110
+	goto error;
1111
+    }
1112
+
1113
+    /* If there is a previous pre-compiled statement, close it first */
1114
+    if (res->st) mysql_stmt_close(res->st);
1115
+    res->st = NULL;
1116
+    
1117
+    /* Create a new pre-compiled statement data structure */
1118
+    res->st = mysql_stmt_init(mcon->con);
1119
+    if (res->st == NULL) {
1120
+	ERR("mysql: Error while creating new MySQL_STMT data structure (no memory left)\n");
1121
+	err = 1;
1122
+	goto error;
1123
+    }
1124
+    
1125
+    /* Try to upload the command to the server */
1126
+    if (mysql_stmt_prepare(res->st, res->sql_cmd.s, res->sql_cmd.len)) {
1127
+	err = mysql_stmt_errno(res->st);    
1128
+	ERR("mysql: libmysql: %d, %s\n", err, mysql_stmt_error(res->st));
1129
+	ERR("mysql: An error occurred while uploading command to server\n");
1130
+    }
1131
+    if (err == CR_SERVER_LOST ||
1132
+	err == CR_SERVER_GONE_ERROR) {
1133
+	/* Connection to the server was lost, mark the connection as
1134
+	 * disconnected. In this case mysql_stmt_prepare invalidates the
1135
+	 * connection internally and calling another mysql function on that
1136
+	 * connection would crash. To make sure that no other mysql function
1137
+	 * gets called unless the connection is reconnected we disconnect it
1138
+	 * explicitly here. This is a workaround for mysql bug #33384. */
1139
+	my_con_disconnect(cmd->ctx->con[db_payload_idx]);
1140
+    }
1141
+    if (err) {
1142
+	/* Report mysql error to the caller */
1143
+	err = -1;
1144
+	goto error;
1145
+    }
1146
+    
1147
+    err = bind_mysql_params(res->st, cmd->vals, cmd->match);
1148
+    if (err) goto error;
1149
+    
1150
+    if (cmd->type == DB_GET || cmd->type == DB_SQL) {
1151
+	err = check_result(cmd, res);
1123 1152
 	if (err) goto error;
1124
-
1125
-	if (cmd->type == DB_GET || cmd->type == DB_SQL) {
1126
-		err = check_result(cmd, res);
1127
-		if (err) goto error;
1128
-		err = bind_result(res->st, cmd->result);
1129
-		if (err) goto error;
1130
-	}
1131
-
1132
-	res->last_reset = mcon->resets;
1133
-	return 0;
1134
-
1135
- error:
1136
-	if (res->st) {
1137
-		ERR("mysql: libmysqlclient: %d, %s\n", 
1138
-			mysql_stmt_errno(res->st), 
1139
-			mysql_stmt_error(res->st));
1140
-		mysql_stmt_close(res->st);
1141
-		res->st = NULL;
1142
-	}
1143
-	return err;
1153
+	err = bind_result(res->st, cmd->result);
1154
+	if (err) goto error;
1155
+    }
1156
+    
1157
+    res->last_reset = mcon->resets;
1158
+    return 0;
1159
+    
1160
+error:
1161
+    if (res->st) {
1162
+	mysql_stmt_close(res->st);
1163
+	res->st = NULL;
1164
+    }
1165
+    return err;
1144 1166
 }
1145 1167
 
1146 1168
 
... ...
@@ -1192,7 +1214,15 @@ int my_cmd(db_cmd_t* cmd)
1192 1214
 	}
1193 1215
 
1194 1216
 	DB_SET_PAYLOAD(cmd, res);
1195
-	if (upload_cmd(cmd) != 0) goto error;
1217
+
1218
+	/* In order to check all the parameters and results, we need to upload
1219
+	 * the command to the server. We need to do that here before we report
1220
+	 * back that the command was created successfully. Hence, this
1221
+	 * function requires the corresponding connection be established. We
1222
+	 * would not be able to check parameters if we don't do that there and
1223
+	 * that could result in repeated execution failures at runtime.
1224
+	 */
1225
+	if (upload_cmd(cmd)) goto error;
1196 1226
 	return 0;
1197 1227
 
1198 1228
  error:
... ...
@@ -106,15 +106,6 @@ int my_con_connect(db_con_t* con)
106 106
 	DBG("mysql: Server version is %s\n", mysql_get_server_info(mcon->con));
107 107
 
108 108
 	mcon->flags |= MY_CONNECTED;
109
-
110
-	/* Increase the variable that keeps track of number of connects performed
111
-	 * on this connection. The mysql module uses the variable to determine
112
-	 * when a pre-compiled command needs to be uploaded to the server again.
113
-	 * If the number in the my_con structure is large than the number kept
114
-	 * in my_cmd then it means that we have to upload the command to the server
115
-	 * again because the connection was reconnected meanwhile.
116
-	 */
117
-	mcon->resets++;
118 109
 	return 0;
119 110
 }
120 111
 
... ...
@@ -133,6 +124,15 @@ void my_con_disconnect(db_con_t* con)
133 124
 
134 125
 	mysql_close(mcon->con);
135 126
 	mcon->flags &= ~MY_CONNECTED;
127
+
128
+	/* Increase the variable that keeps track of number of connection
129
+	 * resets on this connection. The mysql module uses the variable to
130
+	 * determine when a pre-compiled command needs to be uploaded to the
131
+	 * server again. If the number in the my_con structure is larger than
132
+	 * the number kept in my_cmd then it means that we have to upload the
133
+	 * command to the server again because the connection was reset.
134
+	 */
135
+	mcon->resets++;
136 136
 }
137 137
 
138 138
 
... ...
@@ -47,11 +47,11 @@ typedef struct my_con {
47 47
 	MYSQL* con;
48 48
 	unsigned int flags;
49 49
 	
50
-	/* We keep the number of connection resets in this variable,
51
-	 * this variable is incremented each time the module performs
52
-	 * a re-connect on the connection. This is used by my_cmd
53
-	 * related functions to check if a pre-compiled command needs
54
-	 * to be uploaded to the server before executing it.
50
+	/* We keep the number of connection resets in this variable, this
51
+	 * variable is incremented each time the module performs a disconnect
52
+	 * on the connection. This is used by my_cmd related functions to
53
+	 * check if a pre-compiled command needs to be uploaded to the server
54
+	 * before executing them.
55 55
 	 */
56 56
 	unsigned int resets;
57 57
 } my_con_t;