1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Patch for Bug#27863 (excessive memory usage for many small queries in a

Browse Source 
multiquery packet).

Background:

  - a query can contain multiple SQL statements;

  - the server frees resources allocated to process a query when the
    whole query is handled. In other words, resources allocated to process
    one SQL statement from a multi-statement query are freed when all SQL
    statements are handled.

The problem was that the parser allocated a buffer of size of the whole
query for each SQL statement in a multi-statement query. Thus, if a query
had many SQL-statements (so, the query was long), but each SQL statement
was short, ther parser tried to allocate huge amount of memory (number of
small SQL statements * length of the whole query).

The memory was allocated for a so-called "cpp buffer", which is intended to
store pre-processed SQL statement -- SQL text without version specific
comments.

The fix is to allocate memory for the "cpp buffer" once for all SQL
statements (once for a query).
This commit is contained in:
Alexander Nozdrin 2010-05-14 22:11:25 +04:00
parent 7752ccec48
commit 5c4333bc85
8 changed files with 103 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mysql-test/t/parser_stack.test
View File

@ -399,4 +399,12 @@ delimiter ;$$
drop procedure p_37228;
#
# Bug#27863 (excessive memory usage for many small queries in a multiquery
# packet).
#
let $i=`select repeat("set @a=1;", 65535)`;
--disable_query_log
eval $i;
--enable_query_log

4
sql/ha_ndbcluster_binlog.cc
View File

@ -263,7 +263,7 @@ static void run_query(THD *thd, char *buf, char *end,
ulonglong save_thd_options= thd->variables.option_bits;
DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->variables.option_bits));
NET save_thd_net= thd->net;
const char* found_semicolon= NULL;
Parser_state parser_state(thd, thd->query(), thd->query_length());
bzero((char*) &thd->net, sizeof(NET));
thd->set_query(buf, (uint) (end - buf));
@ -277,7 +277,7 @@ static void run_query(THD *thd, char *buf, char *end,
DBUG_ASSERT(!thd->in_sub_stmt);
DBUG_ASSERT(!thd->locked_tables_mode);
mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
if (no_print_error && thd->is_slave_error)
{

4
sql/log_event.cc
View File

@ -3290,8 +3290,8 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli,
thd->table_map_for_update= (table_map)table_map_for_update;
/* Execute the query (note that we bypass dispatch_command()) */
const char* found_semicolon= NULL;
mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
Parser_state parser_state(thd, thd->query(), thd->query_length());
mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
log_slow_statement(thd);
/*

79
sql/sql_lex.cc
View File

@ -142,37 +142,64 @@ st_parsing_options::reset()
allows_derived= TRUE;
}
/**
Perform initialization of Lex_input_stream instance.
Basically, a buffer for pre-processed query. This buffer should be large
enough to keep multi-statement query. The allocation is done once in the
Lex_input_stream constructor in order to prevent memory pollution when
the server is processing large multi-statement queries.
@todo Check return value of THD::alloc().
*/
Lex_input_stream::Lex_input_stream(THD *thd,
const char* buffer,
unsigned int length)
: m_thd(thd),
yylineno(1),
yytoklen(0),
yylval(NULL),
lookahead_token(-1),
lookahead_yylval(NULL),
m_ptr(buffer),
m_tok_start(NULL),
m_tok_end(NULL),
m_end_of_query(buffer + length),
m_tok_start_prev(NULL),
m_buf(buffer),
m_buf_length(length),
m_echo(TRUE),
m_cpp_tok_start(NULL),
m_cpp_tok_start_prev(NULL),
m_cpp_tok_end(NULL),
m_body_utf8(NULL),
m_cpp_utf8_processed_ptr(NULL),
next_state(MY_LEX_START),
found_semicolon(NULL),
ignore_space(test(thd->variables.sql_mode & MODE_IGNORE_SPACE)),
stmt_prepare_mode(FALSE),
multi_statements(TRUE),
in_comment(NO_COMMENT),
m_underscore_cs(NULL)
:m_thd(thd)
{
m_cpp_buf= (char*) thd->alloc(length + 1);
reset(buffer, length);
}
/**
Prepare Lex_input_stream instance state for use for handling next SQL statement.
It should be called between two statements in a multi-statement query.
The operation resets the input stream to the beginning-of-parse state,
but does not reallocate m_cpp_buf.
*/
void
Lex_input_stream::reset(const char *buffer, unsigned int length)
{
yylineno= 1;
yytoklen= 0;
yylval= NULL;
lookahead_token= -1;
lookahead_yylval= NULL;
m_ptr= buffer;
m_tok_start= NULL;
m_tok_end= NULL;
m_end_of_query= buffer + length;
m_tok_start_prev= NULL;
m_buf= buffer;
m_buf_length= length;
m_echo= TRUE;
m_cpp_tok_start= NULL;
m_cpp_tok_start_prev= NULL;
m_cpp_tok_end= NULL;
m_body_utf8= NULL;
m_cpp_utf8_processed_ptr= NULL;
next_state= MY_LEX_START;
found_semicolon= NULL;
ignore_space= test(m_thd->variables.sql_mode & MODE_IGNORE_SPACE);
stmt_prepare_mode= FALSE;
multi_statements= TRUE;
in_comment=NO_COMMENT;
m_underscore_cs= NULL;
m_cpp_ptr= m_cpp_buf;
}

26
sql/sql_lex.h
View File

@ -1377,6 +1377,8 @@ public:
Lex_input_stream(THD *thd, const char* buff, unsigned int length);
~Lex_input_stream();
void reset(const char *buff, unsigned int length);
/**
Set the echo mode.
@ -2206,8 +2208,8 @@ struct LEX: public Query_tables_list
class Set_signal_information
{
public:
/** Constructor. */
Set_signal_information();
/** Empty default constructor, use clear() */
Set_signal_information() {}
/** Copy constructor. */
Set_signal_information(const Set_signal_information& set);
@ -2220,7 +2222,7 @@ public:
void clear();
/**
For each contition item assignment, m_item[] contains the parsed tree
For each condition item assignment, m_item[] contains the parsed tree
that represents the expression assigned, if any.
m_item[] is an array indexed by Diag_condition_item_name.
*/
@ -2237,8 +2239,16 @@ class Yacc_state
{
public:
Yacc_state()
: yacc_yyss(NULL), yacc_yyvs(NULL)
{}
{
reset();
}
void reset()
{
yacc_yyss= NULL;
yacc_yyvs= NULL;
m_set_signal_info.clear();
}
~Yacc_state();
@ -2283,6 +2293,12 @@ public:
Lex_input_stream m_lip;
Yacc_state m_yacc;
void reset(const char *found_semicolon, unsigned int length)
{
m_lip.reset(found_semicolon, length);
m_yacc.reset();
}
};

36
sql/sql_parse.cc
View File

@ -458,7 +458,6 @@ static void handle_bootstrap_impl(THD *thd)
{
MYSQL_FILE *file= bootstrap_file;
char *buff;
const char* found_semicolon= NULL;
DBUG_ENTER("handle_bootstrap");
@ -531,7 +530,8 @@ static void handle_bootstrap_impl(THD *thd)
mode we have only one thread.
*/
thd->set_time();
mysql_parse(thd, thd->query(), length, & found_semicolon);
Parser_state parser_state(thd, thd->query(), length);
mysql_parse(thd, thd->query(), length, &parser_state);
close_thread_tables(thd); // Free tables
bootstrap_error= thd->is_error();
@ -1074,19 +1074,21 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
(char *) thd->security_ctx->host_or_ip);
char *packet_end= thd->query() + thd->query_length();
/* 'b' stands for 'buffer' parameter', special for 'my_snprintf' */
const char* end_of_stmt= NULL;
general_log_write(thd, command, thd->query(), thd->query_length());
DBUG_PRINT("query",("%-.4096s",thd->query()));
#if defined(ENABLED_PROFILING)
thd->profiling.set_query_source(thd->query(), thd->query_length());
#endif
Parser_state parser_state(thd, thd->query(), thd->query_length());
mysql_parse(thd, thd->query(), thd->query_length(), &end_of_stmt);
mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
while (!thd->killed && (end_of_stmt != NULL) && ! thd->is_error())
while (!thd->killed && (parser_state.m_lip.found_semicolon != NULL) &&
! thd->is_error())
{
char *beginning_of_next_stmt= (char*) end_of_stmt;
char *beginning_of_next_stmt= (char*)
parser_state.m_lip.found_semicolon;
thd->protocol->end_statement();
query_cache_end_of_result(thd);
@ -1127,8 +1129,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
*/
statistic_increment(thd->status_var.questions, &LOCK_status);
thd->set_time(); /* Reset the query start time. */
parser_state.reset(beginning_of_next_stmt, length);
/* TODO: set thd->lex->sql_command to SQLCOM_END here */
mysql_parse(thd, beginning_of_next_stmt, length, &end_of_stmt);
mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
}
DBUG_PRINT("info",("query ready"));
@ -5721,7 +5724,7 @@ void mysql_init_multi_delete(LEX *lex)
*/
void mysql_parse(THD *thd, const char *inBuf, uint length,
const char ** found_semicolon)
Parser_state *parser_state)
{
int error;
DBUG_ENTER("mysql_parse");
@ -5751,10 +5754,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
{
LEX *lex= thd->lex;
Parser_state parser_state(thd, inBuf, length);
bool err= parse_sql(thd, & parser_state, NULL);
*found_semicolon= parser_state.m_lip.found_semicolon;
bool err= parse_sql(thd, parser_state, NULL);
if (!err)
{
@ -5769,6 +5769,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
{
if (! thd->is_error())
{
const char *found_semicolon= parser_state->m_lip.found_semicolon;
/*
Binlog logs a string starting from thd->query and having length
thd->query_length; so we set thd->query_length correctly (to not
@ -5779,12 +5780,12 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
PROCESSLIST.
Note that we don't need LOCK_thread_count to modify query_length.
*/
if (*found_semicolon && (ulong) (*found_semicolon - thd->query()))
if (found_semicolon && (ulong) (found_semicolon - thd->query()))
thd->set_query_inner(thd->query(),
(uint32) (*found_semicolon -
(uint32) (found_semicolon -
thd->query() - 1));
/* Actually execute the query */
if (*found_semicolon)
if (found_semicolon)
{
lex->safe_to_cache_query= 0;
thd->server_status|= SERVER_MORE_RESULTS_EXISTS;
@ -5821,11 +5822,6 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
thd->cleanup_after_query();
DBUG_ASSERT(thd->change_list.is_empty());
}
else
{
/* There are no multi queries in the cache. */
*found_semicolon= NULL;
}
DBUG_VOID_RETURN;
}

2
sql/sql_parse.h
View File

@ -84,7 +84,7 @@ bool is_log_table_write_query(enum enum_sql_command command);
bool alloc_query(THD *thd, const char *packet, uint packet_length);
void mysql_init_select(LEX *lex);
void mysql_parse(THD *thd, const char *inBuf, uint length,
const char ** semicolon);
Parser_state *parser_state);
void mysql_reset_thd_for_next_command(THD *thd);
bool mysql_new_select(LEX *lex, bool move_down);
void create_select_for_variable(const char *var_name);

4
sql/sql_signal.cc
View File

@ -75,10 +75,6 @@ const LEX_STRING Diag_statement_item_names[]=
{ C_STRING_WITH_LEN("TRANSACTION_ACTIVE") }
};
Set_signal_information::Set_signal_information()
{
clear();
}
Set_signal_information::Set_signal_information(
const Set_signal_information& set)