1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added some fixes that should make MyISAM & Aria REPAIR work with more than 4G records

Browse Source 
- If one specifies --force twice to myisamchk and aria_chk, then we will try to finnish the repair even if sort_buffer would be too small.
  This was done by dynamically allocate buffer handler objects as long as memory lasts.
- New option for myisamchk and aria_chk: create-missing-keys
- Changed default size of myisam_sort_buffer_size from 8M to 128M.
- Changed default size of sort_buffer_size in aria_chk from 128M to 256M.
- Increased information in error message about 'sort_buffer_size' beeing to small.
- Print also to 'show warnings' if repair was retried.
- Increased size of internal sort-buffer-readers from 16K to 128K
- Changed printing of 'number of records' to use %ll instead of casting to long
- Changed buffer sizes for myisam and aria to use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines.





include/my_global.h:
  Added MY_ALIGN_DOWN() to get previous alignment (for big memory areas)
include/myisam.h:
  Increased size of types to be able to handle more records
include/myisamchk.h:
  Increased size of types to be able to handle more records
  Added T_FORCE_SORT_MEMORY to force repair to work even if sort_buffer would not be big enough
mysql-test/r/myisam.result:
  Updated result
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/repair.result:
  Updated result
mysql-test/suite/maria/maria.result:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria.test:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria3.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test:
  Updated result after sort buffer size increase
mysql-test/t/myisam.test:
  Fixed error messages to not print system specific data
mysql-test/t/repair.test:
  Fixed error messages to not print system specific data
storage/maria/ha_maria.cc:
  Print also to 'show warnings' if repair was retried
  Changed default size of sort_buffer_size from 128M to 256M (same as in mysqld)
storage/maria/ma_check.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/ma_sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug where maria_update_key_parts() was called too early.
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
storage/maria/maria_chk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
storage/maria/maria_def.h:
  Increased size of types to be able to handle more records
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_ftdump.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_read_log.c:
  Use PAGE_BUFFER_INIT for page cache
storage/myisam/ha_myisam.cc:
  Changed default size of myisam_sort_buffer_size from 8M to 128M
storage/myisam/mi_check.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisam_ftdump.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamchk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamdef.h:
  Increased SORT_BUFFER_INIT to 64M
  (speeds up repair a lot and most machines have nowadays a lot of memory)
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
This commit is contained in:
Michael Widenius 2013-05-10 12:32:34 +03:00
parent e5a0daae5a
commit 53d44ad18b
27 changed files with 651 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/my_global.h
View File

@ -847,7 +847,8 @@ typedef long my_ptrdiff_t;
typedef long long my_ptrdiff_t;
#endif
#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1))
#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1))
#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1))
#define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double))
#define ALIGN_MAX_UNIT (sizeof(double))
/* Size to make adressable obj. */

12
include/myisam.h
View File

@ -356,8 +356,10 @@ typedef struct st_mi_sort_param
ulonglong notnull[HA_MAX_KEY_SEG+1];
my_off_t pos,max_pos,filepos,start_recpos;
uint key, key_length,real_key_length,sortbuff_size;
uint maxbuffers, keys, find_length, sort_keys_length;
uint key, key_length,real_key_length;
uint maxbuffers, find_length;
ulonglong sortbuff_size;
ha_rows keys;
my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */
@ -366,10 +368,10 @@ typedef struct st_mi_sort_param
int (*key_write)(struct st_mi_sort_param *, const void *);
void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_mi_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
ulonglong , struct st_buffpek *, IO_CACHE *);
my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
uint, uint);
uint, ulonglong);
} MI_SORT_PARAM;
/* functions in mi_check */

11
include/myisamchk.h
View File

@ -62,9 +62,10 @@
#define T_ZEROFILL ((ulonglong) 1L << 32)
#define T_ZEROFILL_KEEP_LSN ((ulonglong) 1L << 33)
/** If repair should not bump create_rename_lsn */
#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 33)
#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34)
#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35)
#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 34)
#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35)
#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36)
#define T_FORCE_SORT_MEMORY ((ulonglong) 1L << 37)
#define T_REP_ANY (T_REP | T_REP_BY_SORT | T_REP_PARALLEL)
@ -180,8 +181,8 @@ typedef struct st_buffpek {
my_off_t file_pos; /* Where we are in the sort file */
uchar *base, *key; /* Key pointers */
ha_rows count; /* Number of rows in table */
ulong mem_count; /* numbers of keys in memory */
ulong max_keys; /* Max keys in buffert */
ha_rows mem_count; /* Numbers of keys in memory */
ha_rows max_keys; /* Max keys in buffert */
} BUFFPEK;
#endif /* _myisamchk_h */

4
mysql-test/r/myisam.result
View File

@ -2355,7 +2355,7 @@ Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '4'
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small
test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 7168
test.t1 repair status OK
SET myisam_repair_threads=2;
@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings:
Error 1034 myisam_sort_buffer_size is too small
Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
INSERT INTO t1 VALUES('1');

2
mysql-test/r/mysqld--help.result
View File

@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232
myisam-mmap-size 18446744073709551615
myisam-recover-options DEFAULT
myisam-repair-threads 1
myisam-sort-buffer-size 8388608
myisam-sort-buffer-size 134216704
myisam-stats-method nulls_unequal
myisam-use-mmap FALSE
net-buffer-length 16384

4
mysql-test/r/repair.result
View File

@ -77,12 +77,12 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings:
Error 1034 myisam_sort_buffer_size is too small
Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157
SET myisam_repair_threads=2;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small
test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 157
test.t1 repair status OK
SET myisam_repair_threads=@@global.myisam_repair_threads;

62
mysql-test/suite/maria/maria.result
View File

@ -2679,3 +2679,65 @@ select count(*) from t1;
count(*)
13
drop table t1;
#
# BUG#47444 - --myisam_repair_threads > 1 can result in all index
# cardinalities=1
#
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair status OK
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CARDINALITY
14
14
14
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
#
# BUG#47073 - valgrind errs, corruption,failed repair of partition,
# low myisam_sort_buffer_size
#
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=2;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;

46
mysql-test/suite/maria/maria.test
View File

@ -1963,6 +1963,52 @@ unlock tables;
select count(*) from t1;
drop table t1;
--echo #
--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index
--echo # cardinalities=1
--echo #
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CHECK TABLE t1;
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
--echo #
--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition,
--echo # low myisam_sort_buffer_size
--echo #
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=2;
# May report different values depending on threads activity.
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;
#
# End of test
#

2
mysql-test/suite/maria/maria3.result
View File

@ -317,7 +317,7 @@ aria_pagecache_division_limit 100
aria_page_checksum OFF
aria_recover NORMAL
aria_repair_threads 1
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
aria_stats_method nulls_unequal
aria_sync_log_dir NEWFILE
show status like 'aria%';

22
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result
View File

@ -1,30 +1,34 @@
SET @start_global_value = @@global.aria_sort_buffer_size;
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
134217728
268434432
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size
134217728
268434432
show global variables like 'aria_sort_buffer_size';
Variable_name Value
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
show session variables like 'aria_sort_buffer_size';
Variable_name Value
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
select * from information_schema.global_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728
ARIA_SORT_BUFFER_SIZE 268434432
select * from information_schema.session_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728
ARIA_SORT_BUFFER_SIZE 268434432
set global aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
10
4096
set session aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size
10
4096
set global aria_sort_buffer_size=1.1;
ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
set session aria_sort_buffer_size=1e1;
@ -36,7 +40,7 @@ Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '0'
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
4
4096
set session aria_sort_buffer_size=cast(-1 as unsigned int);
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size

20
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result
View File

@ -1,11 +1,11 @@
SET @start_global_value = @@global.myisam_sort_buffer_size ;
SELECT @start_global_value;
@start_global_value
8388608
134216704
SET @start_session_value = @@session.myisam_sort_buffer_size ;
SELECT @start_session_value;
@start_session_value
8388608
134216704
'#--------------------FN_DYNVARS_005_01-------------------------#'
SET @@global.myisam_sort_buffer_size = 100;
Warnings:
@ -13,22 +13,22 @@ Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '100'
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size
8388608
134216704
SET @@session.myisam_sort_buffer_size = 200;
Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '200'
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size
8388608
134216704
'#--------------------FN_DYNVARS_005_02-------------------------#'
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608;
@@global.myisam_sort_buffer_size = 8388608
SELECT @@global.myisam_sort_buffer_size = 134216704;
@@global.myisam_sort_buffer_size = 134216704
1
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608;
@@session.myisam_sort_buffer_size = 8388608
SELECT @@session.myisam_sort_buffer_size = 134216704;
@@session.myisam_sort_buffer_size = 134216704
1
'#--------------------FN_DYNVARS_005_03-------------------------#'
SET @@global.myisam_sort_buffer_size = 4;
@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list'
SET @@global.myisam_sort_buffer_size = @start_global_value;
SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size
8388608
134216704
SET @@session.myisam_sort_buffer_size = @start_session_value;
SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size
8388608
134216704

4
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test
View File

@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size ;
########################################################################
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608;
SELECT @@global.myisam_sort_buffer_size = 134216704;
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608;
SELECT @@session.myisam_sort_buffer_size = 134216704;
--echo '#--------------------FN_DYNVARS_005_03-------------------------#'

2
mysql-test/t/myisam.test
View File

@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1;
SET myisam_repair_threads=2;
@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3;
CREATE TABLE t1(a CHAR(255), KEY(a));
SELECT * FROM t1, t1 AS a1;
SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),

2
mysql-test/t/repair.test
View File

@ -61,6 +61,7 @@ DROP TABLE t1;
#
CREATE TABLE t1(a CHAR(255), KEY(a));
SET myisam_sort_buffer_size=4096;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
@ -79,6 +80,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
SET myisam_repair_threads=2;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1;
SET myisam_repair_threads=@@global.myisam_repair_threads;
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;

8
storage/maria/ha_maria.cc
View File

@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
"disables parallel repair.",
0, 0, 1, 1, 128, 1);
static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing a "
"REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.",
0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1);
"REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
"Specifies how Aria index statistics collection code should treat "
@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
if ((param.testflag & T_REP_BY_SORT))
{
param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP;
if (thd->vio_ok())
_ma_check_print_info(&param, "Retrying repair with keycache");
sql_print_information("Retrying repair of: '%s' with keycache",
table->s->path.str);
continue;

2
storage/maria/ma_check.c
View File

@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param)
param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR;
param->auto_increment_value= 0;
param->use_buffers=USE_BUFFER_INIT;
param->use_buffers= PAGE_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT;

344
storage/maria/ma_sort.c
View File

@ -36,8 +36,10 @@
#define MERGEBUFF2 31
#define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD)
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16)
#define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/*
Pointers of functions for store and read keys from temp file
@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info,uint keys,
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer,
DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions);
static int write_keys(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_keys(MARIA_SORT_PARAM *info,uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MARIA_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile);
static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count);
static int merge_many_buff(MARIA_SORT_PARAM *info,uint keys,
ha_keys count);
static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
BUFFPEK *buffpek,int *maxbuffer,
BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MARIA_SORT_PARAM *info,uint keys,
static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file,
uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MARIA_SORT_PARAM *,uint, uchar **,BUFFPEK *, int,
static int merge_index(MARIA_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *);
static int flush_maria_ft_buf(MARIA_SORT_PARAM *info);
static int write_keys_varlen(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count, BUFFPEK *buffpek,
static int write_keys_varlen(MARIA_SORT_PARAM *info,uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count);
uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count);
IO_CACHE *to_file,
uchar* key, uint sort_length,
ha_keys count);
static inline int
my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
size_t sortbuff_size)
{
int error,maxbuffer,skr;
size_t memavl,old_memavl;
uint keys,sort_length;
int error;
uint sort_length, maxbuffer;
size_t memavl, old_memavl;
DYNAMIC_ARRAY buffpek;
ha_rows records;
ha_rows records, keys;
uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_ma_create_index_by_sort");
@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{
info->write_keys= write_keys_varlen;
info->read_to_buffer=read_to_buffer_varlen;
info->read_to_buffer= read_to_buffer_varlen;
info->write_key=write_merge_key_varlen;
}
else
@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
while (memavl >= MIN_SORT_MEMORY)
{
if ((records < UINT_MAX32) &&
((my_off_t) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl))
keys= (uint)records+1;
/* Check if we can fit all keys into memory */
if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= memavl))
keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do
{
skr=maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
keys < maxbuffer)
{
_ma_check_print_error(info->sort_info->param,
"aria_sort_buffer_size is too small");
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM;
goto err;
}
}
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0))))
{
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0)))
min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= 0;
@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
}
if (memavl < MIN_SORT_MEMORY)
{
_ma_check_print_error(info->sort_info->param, "Aria sort buffer"
" too small"); /* purecov: tested */
goto err; /* purecov: tested */
/* purecov: begin inspected */
_ma_check_print_error(info->sort_info->param,
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys);
my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer == 0)
{
if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records);
if (write_index(info,sort_keys, (uint) records))
my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */
}
else
@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer >= MERGEBUFF2)
{
if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */
my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */
@ -266,13 +298,13 @@ err:
/* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, uint keys,
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile,
uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions)
{
int error;
uint idx;
ha_rows idx;
DBUG_ENTER("find_all_keys");
idx=error=0;
@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg;
int error;
size_t memavl,old_memavl;
size_t memavl, old_memavl;
longlong sortbuff_size;
ha_keys keys, idx;
uint sort_length;
ulong idx, maxbuffer, keys;
uint maxbuffer;
uchar **sort_keys=0;
LINT_INIT(keys);
@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek));
bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
memavl= max(sort_param->sortbuff_size, MIN_SORT_MEMORY);
idx= (uint)sort_param->sort_info->max_records;
sortbuff_size= sort_param->sortbuff_size;
memavl= max(sortbuff_size, MIN_SORT_MEMORY);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length;
maxbuffer= 1;
@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl)
keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else
{
ulong skr;
uint maxbuffer_org;
do
{
skr= maxbuffer;
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < maxbuffer)
{
_ma_check_print_error(sort_param->sort_info->param,
"aria_sort_buffer_size is too small");
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err;
}
}
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys= (uchar **)
my_malloc(keys*(sort_length+sizeof(char*))+
@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0))))
{
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0)))
maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */
@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
}
if (memavl < MIN_SORT_MEMORY)
{
/* purecov: begin inspected */
_ma_check_print_error(sort_param->sort_info->param,
"Aria sort buffer too small");
goto err; /* purecov: tested */
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %lu keys\n",
sort_param->key+1, (ulong) keys);
my_fprintf(stdout,
"Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys;
idx= error= 0;
@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
if (sort_param->real_key_length > sort_param->key_length)
{
if (write_key(sort_param,sort_keys[idx],
if (write_key(sort_param, sort_keys[idx],
&sort_param->tempfile_for_exceptions))
goto err;
continue;
@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
(size_t) sort_param->key_length);
idx= 1;
}
sort_keys[idx]=sort_keys[idx - 1] + sort_param->key_length;
sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
}
if (error > 0)
goto err;
@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
else
sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok;
err:
DBUG_PRINT("error", ("got some error"));
sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */
my_free(sort_keys);
sort_param->sort_keys=0;
sort_param->sort_keys= 0;
delete_dynamic(& sort_param->buffpek);
close_cached_file(&sort_param->tempfile);
close_cached_file(&sort_param->tempfile_for_exceptions);
@ -483,8 +535,12 @@ ok:
Detach from the share if the writer is involved. Avoid others to
be blocked. This includes a flush of the write buffer. This will
also indicate EOF to the readers.
That means that a writer always gets here first and readers -
only when they see EOF. But if a reader finishes prematurely
because of an error it may reach this earlier - don't allow it
to detach the writer thread.
*/
if (sort_param->sort_info->info->rec_cache.share)
if (sort_param->master && sort_param->sort_info->info->rec_cache.share)
remove_io_thread(&sort_param->sort_info->info->rec_cache);
/* Readers detach from the share if any. Avoid others to be blocked. */
@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
for (i= 0, sinfo= sort_param ;
i < sort_info->total_keys ;
i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++)
i++, sinfo++)
{
if (!sinfo->sort_keys)
{
@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
{
if (param->testflag & T_VERBOSE)
{
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout);
}
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo))
got_error=1;
}
if (!got_error && param->testflag & T_STATISTICS)
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
}
my_free(sinfo->sort_keys);
my_free(sinfo->rec_buff);
@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
delete_dynamic(&sinfo->buffpek),
close_cached_file(&sinfo->tempfile),
close_cached_file(&sinfo->tempfile_for_exceptions),
rec_per_key_part+= sinfo->keyinfo->keysegs,
sinfo++)
{
if (got_error)
@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2)
{
if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **) mergebuf,
my_fprintf(stdout,
"Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile))
&maxbuffer, &sinfo->tempfile))
{
got_error=1;
continue;
@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
}
}
}
if (!got_error && (param->testflag & T_STATISTICS))
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
}
my_free(mergebuf);
DBUG_RETURN(got_error);
@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */
static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{
uchar **end;
uint sort_length=info->key_length;
DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{
if (my_b_write(tempfile, *sort_keys, (uint) sort_length))
if (my_b_write(tempfile, *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */
}
DBUG_RETURN(0);
@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MARIA_SORT_PARAM *info,
register uchar **sort_keys,
uint count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
register uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
{
uchar **end;
int err;
DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key,
/* Write index */
static int write_index(MARIA_SORT_PARAM *info,
register uchar **sort_keys,
register uint count)
static int write_index(MARIA_SORT_PARAM *info, register uchar **sort_keys,
register ha_keys count)
{
DBUG_ENTER("write_index");
@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info,
/* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file)
static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek,
uint *maxbuffer, IO_CACHE *t_file)
{
int tmp, merges, max_merges;
uint tmp, merges, max_merges;
IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff");
@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
from_file= t_file ; to_file= &t_file2;
while (*maxbuffer >= MERGEBUFF2)
{
int i;
uint i;
reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1))
@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file))
break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1;
*maxbuffer= (uint) (lastbuff-buffpek)-1;
if (info->sort_info->param->max_stage != 1) /* If not parallel */
_ma_report_progress(info->sort_info->param, merges++, max_merges);
}
cleanup:
close_cached_file(to_file); /* This holds old result */
if (to_file == t_file)
{
DBUG_ASSERT(t_file2.type == WRITE_CACHE);
*t_file=t_file2; /* Copy result file */
t_file->current_pos= &t_file->write_pos;
t_file->current_end= &t_file->write_end;
}
DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */
} /* merge_many_buff */
@ -851,33 +923,35 @@ cleanup:
-1 Error
*/
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
uint length;
register ha_keys count;
my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
if (mysql_file_pread(fromfile->file, buffpek->base,
(length= sort_length*count),buffpek->file_pos,MYF_RW))
return((uint) -1); /* purecov: inspected */
if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length * count),
buffpek->file_pos, MYF_RW))
return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
register ha_keys count;
uint idx;
uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
buffp= buffpek->base;
@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint16 length_of_key;
if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key),
buffpek->file_pos,MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, buffp, length_of_key,
buffpek->file_pos,MYF_RW))
@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */
static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
uint idx;
ha_keys idx;
uchar *bufs = key;
for (idx=1;idx<=count;idx++)
@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count)
IO_CACHE *to_file, uchar *key,
uint sort_length, ha_keys count)
{
return my_b_write(to_file, key, (size_t) sort_length*count);
return my_b_write(to_file, key, ((size_t) sort_length) * count);
}
/*
Merge buffers to one buffer
If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/
static int
merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb)
{
int error;
uint sort_length,maxcount;
int error= 1;
uint sort_length;
ha_keys maxcount;
ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos);
my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos;
BUFFPEK *buffpek,**refpek;
QUEUE queue;
DBUG_ENTER("merge_buffers");
count=error=0;
maxcount=keys/((uint) (Tb-Fb) +1);
count= 0;
maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos);
if (to_file)
to_start_filepos=my_b_tell(to_file);
strpos= (uchar*) sort_keys;
@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
count+= buffpek->count;
buffpek->base= strpos;
buffpek->max_keys=maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
sort_length));
if (error == -1)
buffpek->max_keys= maxcount;
strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length));
if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek);
}
@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
if (info->write_key(info,to_file, buffpek->key,
(uint) sort_length,1))
{
error=1; goto err; /* purecov: inspected */
}
goto err; /* purecov: inspected */
}
else
{
if ((*info->key_write)(info,(void*) buffpek->key))
{
error=1; goto err; /* purecov: inspected */
}
goto err; /* purecov: inspected */
}
buffpek->key+=sort_length;
if (! --buffpek->mem_count)
{
/* It's enough to check for killedptr before a slow operation */
if (_ma_killed_ptr(info->sort_info->param))
{
error=1;
goto err;
}
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
{
uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys;
@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
break; /* One buffer have been removed */
}
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
}
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */
}
}
@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
}
}
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) !=
-1 && error != 0);
while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
if (read_length == 0)
error= 0;
lastbuff->count=count;
if (to_file)
@ -1076,8 +1150,8 @@ err:
/* Do a merge to output-file (save only positions) */
static int
merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
merge_index(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{
DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,

51
storage/maria/maria_chk.c
View File

@ -191,7 +191,7 @@ end:
enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE,
OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@ -229,6 +229,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
{"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.",
@ -362,8 +367,8 @@ static struct my_option my_long_options[] =
{ "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
"Size of page buffer. Used by --safe-repair",
&check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L,
(long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L,
SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
{ "read_buffer_size", OPT_READ_BUFFER_SIZE,
"Read buffer size for sequential reads during scanning",
&check_param.read_buffer_length,
@ -379,9 +384,8 @@ static struct my_option my_long_options[] =
{ "sort_buffer_size", OPT_SORT_BUFFER_SIZE,
"Size of sort buffer. Used by --recover",
&check_param.sort_buffer_length,
&check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
(long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
(long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
&check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0},
{ "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
"Internal buffer for sorting keys; Don't touch :)",
&check_param.sort_key_blocks,
@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
--correct-checksum Correct checksum information for table.\n\
-D, --data-file-length=# Max length of data file (when recreating data\n\
file when it's full).\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick.\n\
-e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\
-f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell Aria to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\
@ -664,10 +676,13 @@ get_one_option(int optid,
if (argument == disabled_my_option)
{
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
}
else
{
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
}
@ -720,8 +735,26 @@ get_one_option(int optid,
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break;
case 'u':
if (argument == disabled_my_option)

18
storage/maria/maria_def.h
View File

@ -94,9 +94,10 @@ typedef struct st_maria_sort_param
*/
ulonglong unique[HA_MAX_KEY_SEG+1];
ulonglong notnull[HA_MAX_KEY_SEG+1];
ulonglong sortbuff_size;
MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos;
uint key, key_length,real_key_length,sortbuff_size;
uint key, key_length,real_key_length;
uint maxbuffers, keys, find_length, sort_keys_length;
my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */
@ -107,10 +108,10 @@ typedef struct st_maria_sort_param
int (*key_write)(struct st_maria_sort_param *, const uchar *);
void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_maria_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
ulonglong , struct st_buffpek *, IO_CACHE *);
my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *,
uint, uint);
uint, ulonglong);
} MARIA_SORT_PARAM;
int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info
#define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192)
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD)
#define SORT_BUFFER_INIT (1024L*1024L*256-MALLOC_OVERHEAD)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD)
/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define PAGE_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192)
#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
#define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0)
#define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1)

2
storage/maria/maria_ftdump.c
View File

@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage();
}
init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0,
init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0,
MARIA_KEY_BLOCK_LENGTH, MY_WME);
if (!(info=maria_open(argv[0], O_RDONLY,

8
storage/maria/maria_read_log.c
View File

@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace";
static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
static my_bool opt_check;
static const char *opt_tmpdir;
static ulong opt_page_buffer_size, opt_translog_buffer_size;
static ulong opt_translog_buffer_size;
static ulonglong opt_page_buffer_size;
static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint;
static MY_TMPDIR maria_chk_tmpdir;
@ -204,9 +205,8 @@ static struct my_option my_long_options[] =
{ "page-buffer-size", 'P',
"The size of the buffer used for index blocks for Aria tables",
&opt_page_buffer_size, &opt_page_buffer_size, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT,
1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD,
(long) IO_SIZE, 0},
GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT,
PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0},
{ "start-from-lsn", 'o', "Start reading log from this lsn",
&opt_start_from_lsn, &opt_start_from_lsn,
0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },

2
storage/myisam/ha_myisam.cc
View File

@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing "
"a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1);
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
"Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);

2
storage/myisam/mi_check.c
View File

@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param)
param->opt_follow_links=1;
param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR;
param->use_buffers=USE_BUFFER_INIT;
param->use_buffers= KEY_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT;

2
storage/myisam/myisam_ftdump.c
View File

@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage();
}
init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0);
init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0);
if (!(info=mi_open(argv[0], O_RDONLY,
HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))

44
storage/myisam/myisamchk.c
View File

@ -134,7 +134,7 @@ int main(int argc, char **argv)
enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE,
OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@ -165,6 +165,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
{"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.",
@ -270,7 +275,7 @@ static struct my_option my_long_options[] =
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{ "key_buffer_size", OPT_KEY_BUFFER_SIZE, "",
&check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD,
GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD,
SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0},
{ "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "",
&opt_key_cache_block_size,
@ -395,10 +400,18 @@ static void usage(void)
-e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\
-f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'myisam_sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given myisam_sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick\n\
--max-record-length=#\n\
Skip rows bigger than this if myisamchk can't allocate\n\
memory to hold it.\n\
@ -541,10 +554,13 @@ get_one_option(int optid,
if (argument == disabled_my_option)
{
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
}
else
{
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
}
@ -597,8 +613,26 @@ get_one_option(int optid,
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break;
case 'u':
if (argument == disabled_my_option)

9
storage/myisam/myisamdef.h
View File

@ -629,10 +629,11 @@ typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */
#define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE)
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD)
#define SORT_BUFFER_INIT (2048L*1024L-MALLOC_OVERHEAD)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD)
/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define KEY_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE)
#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
enum myisam_log_commands
{

289
storage/myisam/sort.c
View File

@ -34,8 +34,10 @@
#define MERGEBUFF 15
#define MERGEBUFF2 31
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16)
#define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/*
Pointers of functions for store and read keys from temp file
@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */
static ha_rows find_all_keys(MI_SORT_PARAM *info,uint keys,
static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer,
DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions);
static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MI_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile);
static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys,
uint count);
static int merge_many_buff(MI_SORT_PARAM *info,uint keys,
ha_keys count);
static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar * *sort_keys,
BUFFPEK *buffpek,int *maxbuffer,
BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MI_SORT_PARAM *info,uint keys,
static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file,
uchar * *sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int,
static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *);
static int flush_ft_buf(MI_SORT_PARAM *info);
static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count);
uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file,
uchar* key, uint sort_length,
uint count);
ha_keys count);
static inline int
my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
ulonglong sortbuff_size)
{
int error,maxbuffer,skr;
uint sort_length, keys;
int error;
uint sort_length, maxbuffer;
ulonglong memavl, old_memavl;
DYNAMIC_ARRAY buffpek;
ha_rows records;
ha_rows records, keys;
uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_create_index_by_sort");
DBUG_PRINT("enter",("sort_length: %d", info->key_length));
DBUG_PRINT("enter",("sort_length: %u", info->key_length));
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{
@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
sort_length= info->key_length;
LINT_INIT(keys);
if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER)
{
if ((records < UINT_MAX32) &&
((my_off_t) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl))
keys= (uint)records+1;
/* Check if we can fit all keys into memory */
if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= memavl))
keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do
{
skr=maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
keys < maxbuffer)
{
mi_check_print_error(info->sort_info->param,
"myisam_sort_buffer_size is too small");
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM;
goto err;
}
}
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0))))
{
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0)))
min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= 0;
@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
}
if (memavl < MIN_SORT_BUFFER)
{
mi_check_print_error(info->sort_info->param,"MyISAM sort buffer too small"); /* purecov: tested */
my_errno= ENOMEM; /* purecov: tested */
goto err; /* purecov: tested */
/* purecov: begin inspected */
mi_check_print_error(info->sort_info->param,
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys);
my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
== HA_POS_ERROR)
goto err; /* purecov: tested */
if (maxbuffer == 0)
{
if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records);
if (write_index(info,sort_keys, (uint) records))
my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */
}
else
@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
if (maxbuffer >= MERGEBUFF2)
{
if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */
my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */
@ -259,13 +286,13 @@ err:
/* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MI_SORT_PARAM *info, uint keys,
static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile,
uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions)
{
int error;
uint idx;
ha_rows idx;
DBUG_ENTER("find_all_keys");
idx=error=0;
@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg)
{
MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg;
int error;
ulonglong memavl, old_memavl;
uint keys, sort_length;
uint idx, maxbuffer;
ulonglong memavl, old_memavl, sortbuff_size;
ha_keys keys, idx;
uint sort_length;
uint maxbuffer;
uchar **sort_keys=0;
LINT_INIT(keys);
@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg)
bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
sort_keys= (uchar **) NULL;
memavl= max(sort_param->sortbuff_size, MIN_SORT_BUFFER);
idx= (uint)sort_param->sort_info->max_records;
sortbuff_size= sort_param->sortbuff_size;
memavl= max(sortbuff_size, MIN_SORT_BUFFER);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length;
maxbuffer= 1;
if ((memavl - sizeof(BUFFPEK)) / (sort_length +
sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER)
{
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <=
(my_off_t) memavl)
keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else
{
uint skr;
uint maxbuffer_org;
do
{
skr= maxbuffer;
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
{
mi_check_print_error(sort_param->sort_info->param,
"myisam_sort_buffer_size is too small");
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err;
}
}
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys= (uchar**)
my_malloc(keys*(sort_length+sizeof(char*))+
@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0))))
{
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0)))
maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */
@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg)
}
if (memavl < MIN_SORT_BUFFER)
{
/* purecov: begin inspected */
mi_check_print_error(sort_param->sort_info->param,
"MyISAM sort buffer too small");
goto err; /* purecov: tested */
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %d keys\n",
sort_param->key + 1, keys);
my_fprintf(stdout,
"Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys;
idx= error= 0;
@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg)
&sort_param->tempfile))
goto err;
sort_keys[0]= (uchar*) (sort_keys+keys);
memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length);
memcpy(sort_keys[0], sort_keys[idx - 1],
(size_t) sort_param->key_length);
idx= 1;
}
sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg)
else
sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok;
err:
@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
{
if (param->testflag & T_VERBOSE)
{
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout);
}
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2)
{
if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile))
&maxbuffer, &sinfo->tempfile))
{
got_error=1;
continue;
@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */
static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{
uchar **end;
uint sort_length=info->key_length;
DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{
if (my_b_write(tempfile,(uchar*) *sort_keys,(uint) sort_length))
if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */
}
DBUG_RETURN(0);
@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MI_SORT_PARAM *info,
register uchar **sort_keys,
uint count, BUFFPEK *buffpek,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
{
uchar **end;
int err;
DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile)
/* Write index */
static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
register uint count)
register ha_keys count)
{
DBUG_ENTER("write_index");
@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
/* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file)
uint *maxbuffer, IO_CACHE *t_file)
{
register int i;
register uint i;
IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff");
@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1))
@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file))
break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1;
*maxbuffer= (uint) (lastbuff-buffpek)-1;
}
cleanup:
close_cached_file(to_file); /* This holds old result */
@ -818,35 +871,36 @@ cleanup:
-1 Error
*/
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
uint length;
register ha_keys count;
my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length*count),
(length= sort_length * count),
buffpek->file_pos, MYF_RW))
return((uint) -1); /* purecov: inspected */
return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */
buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
register ha_keys count;
uint16 length_of_key = 0;
uint idx;
uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
buffp = buffpek->base;
@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
{
if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key,
sizeof(length_of_key), buffpek->file_pos, MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, (uchar*) buffp,
length_of_key, buffpek->file_pos, MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=length_of_key;
buffp = buffp + sort_length;
}
@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */
static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
uint idx;
ha_keys idx;
uchar *bufs = key;
for (idx=1;idx<=count;idx++)
@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info,
static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
return my_b_write(to_file, key, (size_t) sort_length*count);
return my_b_write(to_file, key, ((size_t) sort_length) * count);
}
/*
Merge buffers to one buffer
If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/
static int
merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb)
{
int error;
uint sort_length,maxcount;
int error= 1;
uint sort_length;
ha_keys maxcount;
ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos);
my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos;
BUFFPEK *buffpek,**refpek;
QUEUE queue;
DBUG_ENTER("merge_buffers");
count=error=0;
maxcount=keys/((uint) (Tb-Fb) +1);
maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos);
if (to_file)
@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
count+= buffpek->count;
buffpek->base= (uchar*) strpos;
buffpek->max_keys=maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
sort_length));
if (error == -1)
buffpek->max_keys= maxcount;
strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length));
if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek);
}
@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
/* It's enough to check for killedptr before a slow operation */
if (killed_ptr(info->sort_info->param))
{
error=1;
goto err;
}
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
{
uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys;
@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
break; /* One buffer have been removed */
}
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
}
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */
}
}
@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
}
}
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 &&
error != 0);
while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
if (read_length == 0)
error= 0;
lastbuff->count=count;
if (to_file)
@ -1045,8 +1104,8 @@ err:
/* Do a merge to output-file (save only positions) */
static int
merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{
DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,
@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
DBUG_RETURN(0);
} /* merge_index */
static int
flush_ft_buf(MI_SORT_PARAM *info)
{
@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info)
}
return err;
}