From 9ec19b9b41804d8d4491f0f58ac06ae417e02ffa Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 1 Nov 2017 20:28:36 +0200 Subject: [PATCH] Reducing memory when using information schema The background is that one user had a lot of views and using some complex queries on information schema temporary memory of more than 2G was used. - Added new element 'total_alloc' to MEM_ROOT for easier debugging. - Added MAX_MEMORY_USED to information_schema.processlist. - Added new status variable "Memory_used_initial" that shows how much MariaDB uses at startup. This gives the base value for "Memory_used". - Reuse memory continuously for information schema queries instead of only freeing memory at query end. Other things - Removed some not needed set_notnull() calls for not null columns. --- include/my_alloc.h | 1 + include/mysql.h.pp | 1 + mysql-test/r/create.result | 2 + .../funcs_1/datadict/processlist_priv.inc | 26 ++-- .../funcs_1/datadict/processlist_val.inc | 14 +- .../suite/funcs_1/r/is_columns_is.result | 10 +- .../funcs_1/r/processlist_priv_no_prot.result | 138 +++++++++--------- .../funcs_1/r/processlist_priv_ps.result | 138 +++++++++--------- .../funcs_1/r/processlist_val_no_prot.result | 45 +++--- .../suite/funcs_1/r/processlist_val_ps.result | 45 +++--- mysys/my_alloc.c | 22 ++- sql/mysqld.cc | 7 + sql/sql_class.h | 2 +- sql/sql_const.h | 5 + sql/sql_show.cc | 37 +++-- 15 files changed, 274 insertions(+), 219 deletions(-) diff --git a/include/my_alloc.h b/include/my_alloc.h index 85ac75fc666..b461065eb7c 100644 --- a/include/my_alloc.h +++ b/include/my_alloc.h @@ -43,6 +43,7 @@ typedef struct st_mem_root /* if block have less memory it will be put in 'used' list */ size_t min_malloc; size_t block_size; /* initial block size */ + size_t total_alloc; unsigned int block_num; /* allocated blocks counter */ /* first free block in queue test counter (if it exceed diff --git a/include/mysql.h.pp b/include/mysql.h.pp index d061df71944..e88738be779 100644 --- a/include/mysql.h.pp +++ b/include/mysql.h.pp @@ -236,6 +236,7 @@ typedef struct st_mem_root USED_MEM *pre_alloc; size_t min_malloc; size_t block_size; + size_t total_alloc; unsigned int block_num; unsigned int first_block_usage; void (*error_handler)(void); diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index dca48c07fe9..b4636dca7e2 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -1173,6 +1173,7 @@ t1 CREATE TABLE `t1` ( `MAX_STAGE` tinyint(2) NOT NULL DEFAULT 0, `PROGRESS` decimal(7,3) NOT NULL DEFAULT 0.000, `MEMORY_USED` bigint(7) NOT NULL DEFAULT 0, + `MAX_MEMORY_USED` bigint(7) NOT NULL DEFAULT 0, `EXAMINED_ROWS` int(7) NOT NULL DEFAULT 0, `QUERY_ID` bigint(4) NOT NULL DEFAULT 0, `INFO_BINARY` blob DEFAULT NULL, @@ -1196,6 +1197,7 @@ t1 CREATE TEMPORARY TABLE `t1` ( `MAX_STAGE` tinyint(2) NOT NULL DEFAULT 0, `PROGRESS` decimal(7,3) NOT NULL DEFAULT 0.000, `MEMORY_USED` bigint(7) NOT NULL DEFAULT 0, + `MAX_MEMORY_USED` bigint(7) NOT NULL DEFAULT 0, `EXAMINED_ROWS` int(7) NOT NULL DEFAULT 0, `QUERY_ID` bigint(4) NOT NULL DEFAULT 0, `INFO_BINARY` blob DEFAULT NULL, diff --git a/mysql-test/suite/funcs_1/datadict/processlist_priv.inc b/mysql-test/suite/funcs_1/datadict/processlist_priv.inc index cd060e7a370..ae46dfd31c8 100644 --- a/mysql-test/suite/funcs_1/datadict/processlist_priv.inc +++ b/mysql-test/suite/funcs_1/datadict/processlist_priv.inc @@ -158,7 +158,7 @@ WHERE DB = 'information_schema' AND COMMAND = 'Sleep' AND USER = 'ddicttestuser1 eval SHOW CREATE TABLE $table; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS eval SHOW $table; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID eval SELECT * FROM $table $select_where ORDER BY id; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID eval SELECT $columns FROM $table $select_where ORDER BY id; @@ -178,7 +178,7 @@ connection con100; eval SHOW CREATE TABLE $table; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS eval SHOW $table; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID eval SELECT * FROM $table $select_where ORDER BY id; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID eval SELECT $columns FROM $table $select_where ORDER BY id; @@ -204,7 +204,7 @@ connection con100; SHOW GRANTS; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -216,7 +216,7 @@ connect (con101,localhost,ddicttestuser1,ddictpass,information_schema); SHOW GRANTS; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -239,7 +239,7 @@ connect (anonymous1,localhost,"''",,information_schema); SHOW GRANTS; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -261,7 +261,7 @@ connect (con102,localhost,ddicttestuser1,ddictpass,information_schema); SHOW GRANTS; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -287,7 +287,7 @@ if ($fixed_bug_30395) --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; } ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -308,7 +308,7 @@ connect (con103,localhost,ddicttestuser1,ddictpass,information_schema); SHOW GRANTS FOR 'ddicttestuser1'@'localhost'; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -330,7 +330,7 @@ connect (con104,localhost,ddicttestuser1,ddictpass,information_schema); SHOW GRANTS FOR 'ddicttestuser1'@'localhost'; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -377,7 +377,7 @@ connect (con200,localhost,ddicttestuser2,ddictpass,information_schema); SHOW GRANTS FOR 'ddicttestuser2'@'localhost'; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -398,7 +398,7 @@ connect (con201,localhost,ddicttestuser2,ddictpass,information_schema); SHOW GRANTS; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -421,7 +421,7 @@ SHOW GRANTS FOR 'ddicttestuser1'@'localhost'; GRANT PROCESS ON *.* TO 'ddicttestuser2'@'localhost'; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 @@ -445,7 +445,7 @@ connect (con108,localhost,ddicttestuser1,ddictpass,information_schema); SHOW GRANTS FOR 'ddicttestuser1'@'localhost'; --replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS SHOW processlist; ---replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 ROWS 15 QUERY_ID 17 TID +--replace_column 1 ID 3 HOST_NAME 6 TIME 9 TIME_MS 13 MEMORY 14 MAX_MEMORY 15 ROWS 16 QUERY_ID 18 TID SELECT * FROM information_schema.processlist; --real_sleep 0.3 diff --git a/mysql-test/suite/funcs_1/datadict/processlist_val.inc b/mysql-test/suite/funcs_1/datadict/processlist_val.inc index cfc3c5e751a..24f3eb76a1c 100644 --- a/mysql-test/suite/funcs_1/datadict/processlist_val.inc +++ b/mysql-test/suite/funcs_1/datadict/processlist_val.inc @@ -92,7 +92,7 @@ echo # - INFO must contain the corresponding SHOW/SELECT PROCESSLIST # # 1. Just dump what we get ---replace_column 1 3 6