From 95b3813c1ae973c674e256ac59cb4488dd22d0c1 Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Mon, 28 May 2018 12:49:09 +0530 Subject: [PATCH 001/127] Raise version number after cloning 5.5.61 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 23e938e60e1..4acb69e9e07 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=61 +MYSQL_VERSION_PATCH=62 MYSQL_VERSION_EXTRA= From 7b2f4b82ea4dd8a63c2806c452b319f26493206c Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Wed, 23 May 2018 18:20:45 +0200 Subject: [PATCH 002/127] Fix build break with modern compilers: client/mysql.cc: In function void build_completion_hash(bool, bool): client/mysql.cc:2674:37: error: invalid conversion from char to char* [-fpermissive] field_names[i][num_fields*2]= '\0'; ^~~~ --- client/mysql.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 715d74f18b2..9c54f564192 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2671,7 +2671,7 @@ You can turn off this feature to get a quicker startup with -A\n\n"); mysql_free_result(fields); break; } - field_names[i][num_fields*2]= '\0'; + field_names[i][num_fields*2]= NULL; j=0; while ((sql_field=mysql_fetch_field(fields))) { From cad692f919493e2176d52984415561496e13833d Mon Sep 17 00:00:00 2001 From: Ivo Roylev Date: Thu, 14 Jun 2018 17:27:54 +0300 Subject: [PATCH 003/127] Bug#27980823: HEAP OVERFLOW VULNERABILITIES IN MYSQL CLIENT LIBRARY --- sql-common/client.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql-common/client.c b/sql-common/client.c index 3247fd8e339..7938403db59 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1505,7 +1505,8 @@ unpack_fields(MYSQL *mysql, MYSQL_DATA *data,MEM_ROOT *alloc,uint fields, { uchar *pos; /* fields count may be wrong */ - DBUG_ASSERT((uint) (field - result) < fields); + if (field < result || (uint) (field - result) >= fields) + DBUG_RETURN(NULL); cli_fetch_lengths(&lengths[0], row->data, default_value ? 8 : 7); field->catalog= strmake_root(alloc,(char*) row->data[0], lengths[0]); field->db= strmake_root(alloc,(char*) row->data[1], lengths[1]); @@ -1612,6 +1613,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields, if ((pkt_len= cli_safe_read(mysql)) == packet_error) DBUG_RETURN(0); + if (pkt_len == 0) DBUG_RETURN(0); if (!(result=(MYSQL_DATA*) my_malloc(sizeof(MYSQL_DATA), MYF(MY_WME | MY_ZEROFILL)))) { From e1fdeb2468f3ee16e34e0c3af856edde128836e5 Mon Sep 17 00:00:00 2001 From: Terje Rosten Date: Wed, 20 Jun 2018 12:38:32 +0200 Subject: [PATCH 004/127] Bug#27919254 MYSQL USER ESCALATES ITS PRIVILEGE BY PLACING ARBITRARY PIDS INTO ITS PID FILES Shutdown server as mysql user to avoid accidentally sending signal to wrong process. --- packaging/rpm-oel/mysql.init | 2 +- packaging/rpm-sles/mysql.init | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/rpm-oel/mysql.init b/packaging/rpm-oel/mysql.init index e46edd14187..4200e99dd95 100644 --- a/packaging/rpm-oel/mysql.init +++ b/packaging/rpm-oel/mysql.init @@ -159,7 +159,7 @@ stop(){ fi MYSQLPID=`cat "$mypidfile"` if [ -n "$MYSQLPID" ]; then - /bin/kill "$MYSQLPID" >/dev/null 2>&1 + /bin/su - mysql -s /bin/bash -c "/bin/kill $MYSQLPID" >/dev/null 2>&1 ret=$? if [ $ret -eq 0 ]; then TIMEOUT="$STOPTIMEOUT" diff --git a/packaging/rpm-sles/mysql.init b/packaging/rpm-sles/mysql.init index 4c72d976526..c89d56f5c1a 100644 --- a/packaging/rpm-sles/mysql.init +++ b/packaging/rpm-sles/mysql.init @@ -163,7 +163,7 @@ stop () { # We use a signal to avoid having to know the root password # Send single kill command and then wait - if kill $pid >/dev/null 2>&1; then + if su - mysql -s /bin/bash -c "kill $pid" >/dev/null 2>&1; then timer=$STOPTIMEOUT while [ $timer -gt 0 ]; do kill -0 $pid >/dev/null 2>&1 || break @@ -196,7 +196,7 @@ reload () { ret=0 if chk_running && mysqladmin --no-defaults --socket="$socket" ping >/dev/null 2>&1 ; then pid=$(cat "$pidfile") - kill -HUP $pid >/dev/null 2>&1 + su - mysql -s /bin/bash -c "kill -HUP $pid" >/dev/null 2>&1 echo -n "Reloading service MySQL:" rc_reset else From 22e99fcb34712e710c6fe086d44e3643479f9e76 Mon Sep 17 00:00:00 2001 From: Arun Kuruvila Date: Fri, 29 Jun 2018 12:09:18 +0530 Subject: [PATCH 005/127] Bug#27799513: POTENTIAL DOUBLE FREE OR CORRUPTION OF HEAP INFO (HP_INFO) Description:- Server crashes due to memory overflow. Analysis:- Bytes for storing key length is wrongly set for HEAP tables. Fix:- Bytes used to store the key length is properly set inside "heap_create()". --- storage/heap/hp_create.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/storage/heap/hp_create.c b/storage/heap/hp_create.c index 93928cd479e..d32a69dd630 100644 --- a/storage/heap/hp_create.c +++ b/storage/heap/hp_create.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -92,7 +92,14 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, /* fall_through */ case HA_KEYTYPE_VARTEXT1: keyinfo->flag|= HA_VAR_LENGTH_KEY; - length+= 2; + /* + For BTREE algorithm, key length, greater than or equal + to 255, is packed on 3 bytes. + */ + if (keyinfo->algorithm == HA_KEY_ALG_BTREE) + length+= size_to_store_key_length(keyinfo->seg[j].length); + else + length+= 2; /* Save number of bytes used to store length */ keyinfo->seg[j].bit_start= 1; break; @@ -101,7 +108,14 @@ int heap_create(const char *name, HP_CREATE_INFO *create_info, /* fall_through */ case HA_KEYTYPE_VARTEXT2: keyinfo->flag|= HA_VAR_LENGTH_KEY; - length+= 2; + /* + For BTREE algorithm, key length, greater than or equal + to 255, is packed on 3 bytes. + */ + if (keyinfo->algorithm == HA_KEY_ALG_BTREE) + length+= size_to_store_key_length(keyinfo->seg[j].length); + else + length+= 2; /* Save number of bytes used to store length */ keyinfo->seg[j].bit_start= 2; /* From 28b052190e377bdfe176e1dc71d268b94c832263 Mon Sep 17 00:00:00 2001 From: Anushree Prakash B Date: Tue, 3 Jul 2018 15:20:03 +0530 Subject: [PATCH 006/127] Bug#28093271 - MYSQL OVERLOADS -b SHORT OPTION: --BINARY-AS-HEX, --NO-BEEP DESCRIPTION: ============ mysql uses -b as the short-option form for two different long options i.e. no-beep and binary-as-hex. This can result in unintended results if the short form -b is used instead of the specific long option name. FIX: ==== -b will now be used for one long option only i.e --no-beep. The option binary-as-hex will not have any short option and should be provided as a complete name. --- client/mysql.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysql.cc b/client/mysql.cc index 9c54f564192..bc32a58f37e 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1392,7 +1392,7 @@ static struct my_option my_long_options[] = {"batch", 'B', "Don't use history file. Disable interactive behavior. (Enables --silent.)", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"binary-as-hex", 'b', "Print binary data as hex", &opt_binhex, &opt_binhex, + {"binary-as-hex", 0, "Print binary data as hex", &opt_binhex, &opt_binhex, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"character-sets-dir", OPT_CHARSETS_DIR, "Directory for character set files.", &charsets_dir, From c54271723c6f2f4e5b77cd8faf91e89412730209 Mon Sep 17 00:00:00 2001 From: Aditya A Date: Mon, 9 Jul 2018 17:25:49 +0530 Subject: [PATCH 007/127] Bug #26275510 BUNDLED ZLIB DOESN'T INCLUDE FIXES FOR SOME VULNERABILITIES Upgrading the zlib lib to 1.2.11 --- CMakeLists.txt | 8 +- client/CMakeLists.txt | 3 +- client/mysql.cc | 2 +- cmake/plugin.cmake | 4 +- cmake/zlib.cmake | 17 +- extra/CMakeLists.txt | 3 +- libmysql/CMakeLists.txt | 4 +- libmysqld/CMakeLists.txt | 3 +- mysys/CMakeLists.txt | 4 +- sql/CMakeLists.txt | 3 +- zlib/CMakeLists.txt | 157 +++- zlib/ChangeLog | 666 +++++++++++++- zlib/FAQ | 267 +++--- zlib/INDEX | 41 +- zlib/README | 94 +- zlib/README.MySQL | 16 - zlib/adler32.c | 113 ++- zlib/algorithm.txt | 209 ----- zlib/amiga/Makefile.pup | 69 ++ zlib/amiga/Makefile.sas | 68 ++ zlib/compress.c | 45 +- zlib/crc32.c | 149 ++-- zlib/crc32.h | 2 +- zlib/deflate.c | 1321 ++++++++++++++++++---------- zlib/deflate.h | 78 +- zlib/gzclose.c | 25 + zlib/gzguts.h | 218 +++++ zlib/gzio.c | 1031 ---------------------- zlib/gzlib.c | 637 ++++++++++++++ zlib/gzread.c | 654 ++++++++++++++ zlib/gzwrite.c | 665 ++++++++++++++ zlib/infback.c | 107 ++- zlib/inffast.c | 159 ++-- zlib/inffast.h | 4 +- zlib/inffixed.h | 6 +- zlib/inflate.c | 585 +++++++----- zlib/inflate.h | 38 +- zlib/inftrees.c | 109 +-- zlib/inftrees.h | 27 +- zlib/make_vms.com | 867 ++++++++++++++++++ zlib/msdos/Makefile.bor | 115 +++ zlib/msdos/Makefile.dj2 | 104 +++ zlib/msdos/Makefile.emx | 69 ++ zlib/msdos/Makefile.msc | 112 +++ zlib/msdos/Makefile.tc | 100 +++ zlib/nintendods/README | 5 + zlib/old/Makefile.emx | 69 ++ zlib/old/Makefile.riscos | 151 ++++ zlib/old/README | 3 + zlib/old/descrip.mms | 48 + zlib/old/os2/Makefile.os2 | 136 +++ zlib/old/os2/zlib.def | 51 ++ zlib/old/visual-basic.txt | 160 ++++ zlib/os400/README400 | 48 + zlib/os400/bndsrc | 119 +++ zlib/os400/make.sh | 366 ++++++++ zlib/os400/zlib.inc | 527 +++++++++++ zlib/qnx/package.qpg | 141 +++ zlib/test/example.c | 602 +++++++++++++ zlib/test/infcover.c | 671 ++++++++++++++ zlib/test/minigzip.c | 651 ++++++++++++++ zlib/treebuild.xml | 116 +++ zlib/trees.c | 246 +++--- zlib/trees.h | 4 +- zlib/uncompr.c | 116 ++- zlib/watcom/watcom_f.mak | 43 + zlib/watcom/watcom_l.mak | 43 + zlib/win32/DLL_FAQ.txt | 397 +++++++++ zlib/win32/Makefile.bor | 110 +++ zlib/win32/Makefile.gcc | 182 ++++ zlib/win32/Makefile.msc | 163 ++++ zlib/win32/README-WIN32.txt | 103 +++ zlib/win32/VisualC.txt | 3 + zlib/win32/zlib.def | 94 ++ zlib/win32/zlib1.rc | 40 + zlib/zconf.h.cmakein | 536 +++++++++++ zlib/{zconf.h => zconf.h.in} | 330 +++++-- zlib/zlib.3 | 120 ++- zlib/zlib.h | 1613 +++++++++++++++++++++++----------- zlib/zlib.pc.cmakein | 13 + zlib/zlib.pc.in | 13 + zlib/zlib2ansi | 152 ++++ zlib/zutil.c | 101 ++- zlib/zutil.h | 192 ++-- 84 files changed, 13977 insertions(+), 3479 deletions(-) delete mode 100644 zlib/README.MySQL delete mode 100644 zlib/algorithm.txt create mode 100644 zlib/amiga/Makefile.pup create mode 100644 zlib/amiga/Makefile.sas create mode 100644 zlib/gzclose.c create mode 100644 zlib/gzguts.h delete mode 100644 zlib/gzio.c create mode 100644 zlib/gzlib.c create mode 100644 zlib/gzread.c create mode 100644 zlib/gzwrite.c create mode 100644 zlib/make_vms.com create mode 100644 zlib/msdos/Makefile.bor create mode 100644 zlib/msdos/Makefile.dj2 create mode 100644 zlib/msdos/Makefile.emx create mode 100644 zlib/msdos/Makefile.msc create mode 100644 zlib/msdos/Makefile.tc create mode 100644 zlib/nintendods/README create mode 100644 zlib/old/Makefile.emx create mode 100644 zlib/old/Makefile.riscos create mode 100644 zlib/old/README create mode 100644 zlib/old/descrip.mms create mode 100644 zlib/old/os2/Makefile.os2 create mode 100644 zlib/old/os2/zlib.def create mode 100644 zlib/old/visual-basic.txt create mode 100644 zlib/os400/README400 create mode 100644 zlib/os400/bndsrc create mode 100644 zlib/os400/make.sh create mode 100644 zlib/os400/zlib.inc create mode 100644 zlib/qnx/package.qpg create mode 100644 zlib/test/example.c create mode 100644 zlib/test/infcover.c create mode 100644 zlib/test/minigzip.c create mode 100644 zlib/treebuild.xml create mode 100644 zlib/watcom/watcom_f.mak create mode 100644 zlib/watcom/watcom_l.mak create mode 100644 zlib/win32/DLL_FAQ.txt create mode 100644 zlib/win32/Makefile.bor create mode 100644 zlib/win32/Makefile.gcc create mode 100644 zlib/win32/Makefile.msc create mode 100644 zlib/win32/README-WIN32.txt create mode 100644 zlib/win32/VisualC.txt create mode 100644 zlib/win32/zlib.def create mode 100644 zlib/win32/zlib1.rc create mode 100644 zlib/zconf.h.cmakein rename zlib/{zconf.h => zconf.h.in} (50%) create mode 100644 zlib/zlib.pc.cmakein create mode 100644 zlib/zlib.pc.in create mode 100755 zlib/zlib2ansi diff --git a/CMakeLists.txt b/CMakeLists.txt index 154273576d7..89f71fa8580 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -382,6 +382,12 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include) # Add bundled or system zlib. MYSQL_CHECK_ZLIB_WITH_COMPRESS() +IF(BUILD_BUNDLED_ZLIB) + INCLUDE_DIRECTORIES(SYSTEM + ${CMAKE_CURRENT_SOURCE_DIR}/zlib + ${CMAKE_CURRENT_BINARY_DIR}/zlib + ) +ENDIF() # Optionally add bundled yassl/taocrypt or system openssl. MYSQL_CHECK_SSL() # Add readline or libedit. diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 21b1e084409..ea72e9ce1e8 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,6 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/libmysql ${CMAKE_SOURCE_DIR}/regex diff --git a/client/mysql.cc b/client/mysql.cc index bc32a58f37e..8510361a54e 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -2671,7 +2671,7 @@ You can turn off this feature to get a quicker startup with -A\n\n"); mysql_free_result(fields); break; } - field_names[i][num_fields*2]= NULL; + field_names[i][num_fields*2]= '\0'; j=0; while ((sql_field=mysql_fetch_field(fields))) { diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index e946f0ff863..77001edfd34 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ MACRO(MYSQL_ADD_PLUGIN) ${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex ${SSL_INCLUDE_DIRS} - ${ZLIB_INCLUDE_DIR}) + ) LIST(GET ARG_DEFAULT_ARGS 0 plugin) SET(SOURCES ${ARG_DEFAULT_ARGS}) diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake index c70d0e29ecf..495571e4dc7 100644 --- a/cmake/zlib.cmake +++ b/cmake/zlib.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,15 +14,11 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA MACRO (MYSQL_USE_BUNDLED_ZLIB) - SET(ZLIB_LIBRARY zlib) - SET(ZLIB_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/zlib) + SET(BUILD_BUNDLED_ZLIB 1) + SET(ZLIB_LIBRARY zlib CACHE INTERNAL "Bundled zlib library") SET(ZLIB_FOUND TRUE) SET(WITH_ZLIB "bundled" CACHE STRING "Use bundled zlib") ADD_SUBDIRECTORY(zlib) - GET_TARGET_PROPERTY(src zlib SOURCES) - FOREACH(file ${src}) - SET(ZLIB_SOURCES ${ZLIB_SOURCES} ${CMAKE_SOURCE_DIR}/zlib/${file}) - ENDFOREACH() ENDMACRO() # MYSQL_CHECK_ZLIB_WITH_COMPRESS @@ -37,15 +33,10 @@ ENDMACRO() MACRO (MYSQL_CHECK_ZLIB_WITH_COMPRESS) - IF(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR - CMAKE_SYSTEM_NAME STREQUAL "AIX" OR - CMAKE_SYSTEM_NAME STREQUAL "Windows") - # Use bundled zlib on some platforms by default (system one is too - # old or not existent) + # For NDBCLUSTER: Use bundled zlib by default IF (NOT WITH_ZLIB) SET(WITH_ZLIB "bundled" CACHE STRING "By default use bundled zlib on this platform") ENDIF() - ENDIF() IF(WITH_ZLIB STREQUAL "bundled") MYSQL_USE_BUNDLED_ZLIB() diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index a569641a198..bea704875a6 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,6 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include -${ZLIB_INCLUDE_DIR} # Following is for perror, in case NDB is compiled in. ${CMAKE_SOURCE_DIR}/storage/ndb/include ${CMAKE_SOURCE_DIR}/storage/ndb/include/util diff --git a/libmysql/CMakeLists.txt b/libmysql/CMakeLists.txt index 374b38131d2..9650c7edff5 100644 --- a/libmysql/CMakeLists.txt +++ b/libmysql/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,7 +21,7 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/strings ${SSL_INCLUDE_DIRS} ${SSL_INTERNAL_INCLUDE_DIRS} - ${ZLIB_INCLUDE_DIR}) + ) ADD_DEFINITIONS(${SSL_DEFINES}) SET(CLIENT_API_FUNCTIONS diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index abbf887e1e6..b1ed62ce141 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,7 +23,6 @@ ${CMAKE_SOURCE_DIR}/libmysqld ${CMAKE_SOURCE_DIR}/sql ${CMAKE_BINARY_DIR}/sql ${CMAKE_SOURCE_DIR}/regex -${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${SSL_INTERNAL_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/sql/backup diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 6969acd8d04..e0745957c20 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys) +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys) SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c errors.c hash.c list.c md5.c mf_cache.c mf_dirname.c mf_fn_ext.c diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index aa7e0312e05..044cc8fac78 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,6 @@ INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex -${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${CMAKE_BINARY_DIR}/sql ) diff --git a/zlib/CMakeLists.txt b/zlib/CMakeLists.txt index 7668ce723b8..0c224e7ce22 100644 --- a/zlib/CMakeLists.txt +++ b/zlib/CMakeLists.txt @@ -1,29 +1,148 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. -# + +# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. +# # This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# +# it under the terms of the GNU General Public License, version 2.0, +# as published by the Free Software Foundation. +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# +# GNU General Public License, version 2.0, for more details. +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -INCLUDE_DIRECTORIES( -${CMAKE_SOURCE_DIR}/include -${CMAKE_SOURCE_DIR}/zlib +cmake_minimum_required(VERSION 2.4.4) +set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) + +project(zlib C) + +set(VERSION "1.2.11") + +include(CheckTypeSize) +include(CheckFunctionExists) +include(CheckIncludeFile) +include(CheckCSourceCompiles) + +check_include_file(sys/types.h HAVE_SYS_TYPES_H) +check_include_file(stdint.h HAVE_STDINT_H) +check_include_file(stddef.h HAVE_STDDEF_H) + +# +# Check to see if we have large file support +# +set(CMAKE_REQUIRED_DEFINITIONS -D_LARGEFILE64_SOURCE=1) +# We add these other definitions here because CheckTypeSize.cmake +# in CMake 2.4.x does not automatically do so and we want +# compatibility with CMake 2.4.x. +if(HAVE_SYS_TYPES_H) + list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_SYS_TYPES_H) +endif() +if(HAVE_STDINT_H) + list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_STDINT_H) +endif() +if(HAVE_STDDEF_H) + list(APPEND CMAKE_REQUIRED_DEFINITIONS -DHAVE_STDDEF_H) +endif() +check_type_size(off64_t OFF64_T) +if(HAVE_OFF64_T) + add_definitions(-D_LARGEFILE64_SOURCE=1) +endif() +set(CMAKE_REQUIRED_DEFINITIONS) # clear variable + +# +# Check for fseeko +# +check_function_exists(fseeko HAVE_FSEEKO) +if(NOT HAVE_FSEEKO) + add_definitions(-DNO_FSEEKO) +endif() + +# +# Check for unistd.h +# +check_include_file(unistd.h Z_HAVE_UNISTD_H) + + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/zconf.h.cmakein + ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY + ) +include_directories( + SYSTEM ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} + ) + +#============================================================================ +# zlib +#============================================================================ + +set(ZLIB_PUBLIC_HDRS + ${CMAKE_CURRENT_BINARY_DIR}/zconf.h + zlib.h +) +set(ZLIB_PRIVATE_HDRS + crc32.h + deflate.h + gzguts.h + inffast.h + inffixed.h + inflate.h + inftrees.h + trees.h + zutil.h +) +set(ZLIB_SRCS + adler32.c + compress.c + crc32.c + deflate.c + gzclose.c + gzlib.c + gzread.c + gzwrite.c + inflate.c + infback.c + inftrees.c + inffast.c + trees.c + uncompr.c + zutil.c ) -SET(ZLIB_SOURCES adler32.c compress.c crc32.c crc32.h deflate.c deflate.h gzio.c infback.c inffast.c inffast.h - inffixed.h inflate.c inflate.h inftrees.c inftrees.h trees.c trees.h uncompr.c zconf.h zlib.h - zutil.c zutil.h) -ADD_CONVENIENCE_LIBRARY(zlib ${ZLIB_SOURCES}) -RESTRICT_SYMBOL_EXPORTS(zlib) -IF(MSVC) - INSTALL_DEBUG_TARGET(zlib DESTINATION ${INSTALL_LIBDIR}/debug) -ENDIF() +if(NOT MINGW) + set(ZLIB_DLL_SRCS + win32/zlib1.rc # If present will override custom build rule below. + ) +endif() +# parse the full version number from zlib.h and include in ZLIB_FULL_VERSION +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/zlib.h _zlib_h_contents) +string(REGEX REPLACE ".*#define[ \t]+ZLIB_VERSION[ \t]+\"([-0-9A-Za-z.]+)\".*" + "\\1" ZLIB_FULL_VERSION ${_zlib_h_contents}) + +ADD_CONVENIENCE_LIBRARY(zlib STATIC + ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) + +if(NOT CYGWIN) + # This property causes shared libraries on Linux to have the full version + # encoded into their final filename. We disable this on Cygwin because + # it causes cygz-${ZLIB_FULL_VERSION}.dll to be created when cygz.dll + # seems to be the default. + # + # This has no effect with MSVC, on that platform the version info for + # the DLL comes from the resource file win32/zlib1.rc + set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION}) +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "SunOS") + # On unix-like platforms the library is almost always called libz + set_target_properties(zlib PROPERTIES OUTPUT_NAME z) +elseif(UNIX) + # On unix-like platforms the library is almost always called libz + set_target_properties(zlib PROPERTIES OUTPUT_NAME z) + if(NOT APPLE) + set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") + endif() +endif() diff --git a/zlib/ChangeLog b/zlib/ChangeLog index 7f6869d3235..30199a65a03 100644 --- a/zlib/ChangeLog +++ b/zlib/ChangeLog @@ -1,6 +1,666 @@ ChangeLog file for zlib +Changes in 1.2.11 (15 Jan 2017) +- Fix deflate stored bug when pulling last block from window +- Permit immediate deflateParams changes before any deflate input + +Changes in 1.2.10 (2 Jan 2017) +- Avoid warnings on snprintf() return value +- Fix bug in deflate_stored() for zero-length input +- Fix bug in gzwrite.c that produced corrupt gzip files +- Remove files to be installed before copying them in Makefile.in +- Add warnings when compiling with assembler code + +Changes in 1.2.9 (31 Dec 2016) +- Fix contrib/minizip to permit unzipping with desktop API [Zouzou] +- Improve contrib/blast to return unused bytes +- Assure that gzoffset() is correct when appending +- Improve compress() and uncompress() to support large lengths +- Fix bug in test/example.c where error code not saved +- Remedy Coverity warning [Randers-Pehrson] +- Improve speed of gzprintf() in transparent mode +- Fix inflateInit2() bug when windowBits is 16 or 32 +- Change DEBUG macro to ZLIB_DEBUG +- Avoid uninitialized access by gzclose_w() +- Allow building zlib outside of the source directory +- Fix bug that accepted invalid zlib header when windowBits is zero +- Fix gzseek() problem on MinGW due to buggy _lseeki64 there +- Loop on write() calls in gzwrite.c in case of non-blocking I/O +- Add --warn (-w) option to ./configure for more compiler warnings +- Reject a window size of 256 bytes if not using the zlib wrapper +- Fix bug when level 0 used with Z_HUFFMAN or Z_RLE +- Add --debug (-d) option to ./configure to define ZLIB_DEBUG +- Fix bugs in creating a very large gzip header +- Add uncompress2() function, which returns the input size used +- Assure that deflateParams() will not switch functions mid-block +- Dramatically speed up deflation for level 0 (storing) +- Add gzfread(), duplicating the interface of fread() +- Add gzfwrite(), duplicating the interface of fwrite() +- Add deflateGetDictionary() function +- Use snprintf() for later versions of Microsoft C +- Fix *Init macros to use z_ prefix when requested +- Replace as400 with os400 for OS/400 support [Monnerat] +- Add crc32_z() and adler32_z() functions with size_t lengths +- Update Visual Studio project files [AraHaan] + +Changes in 1.2.8 (28 Apr 2013) +- Update contrib/minizip/iowin32.c for Windows RT [Vollant] +- Do not force Z_CONST for C++ +- Clean up contrib/vstudio [Roß] +- Correct spelling error in zlib.h +- Fix mixed line endings in contrib/vstudio + +Changes in 1.2.7.3 (13 Apr 2013) +- Fix version numbers and DLL names in contrib/vstudio/*/zlib.rc + +Changes in 1.2.7.2 (13 Apr 2013) +- Change check for a four-byte type back to hexadecimal +- Fix typo in win32/Makefile.msc +- Add casts in gzwrite.c for pointer differences + +Changes in 1.2.7.1 (24 Mar 2013) +- Replace use of unsafe string functions with snprintf if available +- Avoid including stddef.h on Windows for Z_SOLO compile [Niessink] +- Fix gzgetc undefine when Z_PREFIX set [Turk] +- Eliminate use of mktemp in Makefile (not always available) +- Fix bug in 'F' mode for gzopen() +- Add inflateGetDictionary() function +- Correct comment in deflate.h +- Use _snprintf for snprintf in Microsoft C +- On Darwin, only use /usr/bin/libtool if libtool is not Apple +- Delete "--version" file if created by "ar --version" [Richard G.] +- Fix configure check for veracity of compiler error return codes +- Fix CMake compilation of static lib for MSVC2010 x64 +- Remove unused variable in infback9.c +- Fix argument checks in gzlog_compress() and gzlog_write() +- Clean up the usage of z_const and respect const usage within zlib +- Clean up examples/gzlog.[ch] comparisons of different types +- Avoid shift equal to bits in type (caused endless loop) +- Fix uninitialized value bug in gzputc() introduced by const patches +- Fix memory allocation error in examples/zran.c [Nor] +- Fix bug where gzopen(), gzclose() would write an empty file +- Fix bug in gzclose() when gzwrite() runs out of memory +- Check for input buffer malloc failure in examples/gzappend.c +- Add note to contrib/blast to use binary mode in stdio +- Fix comparisons of differently signed integers in contrib/blast +- Check for invalid code length codes in contrib/puff +- Fix serious but very rare decompression bug in inftrees.c +- Update inflateBack() comments, since inflate() can be faster +- Use underscored I/O function names for WINAPI_FAMILY +- Add _tr_flush_bits to the external symbols prefixed by --zprefix +- Add contrib/vstudio/vc10 pre-build step for static only +- Quote --version-script argument in CMakeLists.txt +- Don't specify --version-script on Apple platforms in CMakeLists.txt +- Fix casting error in contrib/testzlib/testzlib.c +- Fix types in contrib/minizip to match result of get_crc_table() +- Simplify contrib/vstudio/vc10 with 'd' suffix +- Add TOP support to win32/Makefile.msc +- Suport i686 and amd64 assembler builds in CMakeLists.txt +- Fix typos in the use of _LARGEFILE64_SOURCE in zconf.h +- Add vc11 and vc12 build files to contrib/vstudio +- Add gzvprintf() as an undocumented function in zlib +- Fix configure for Sun shell +- Remove runtime check in configure for four-byte integer type +- Add casts and consts to ease user conversion to C++ +- Add man pages for minizip and miniunzip +- In Makefile uninstall, don't rm if preceding cd fails +- Do not return Z_BUF_ERROR if deflateParam() has nothing to write + +Changes in 1.2.7 (2 May 2012) +- Replace use of memmove() with a simple copy for portability +- Test for existence of strerror +- Restore gzgetc_ for backward compatibility with 1.2.6 +- Fix build with non-GNU make on Solaris +- Require gcc 4.0 or later on Mac OS X to use the hidden attribute +- Include unistd.h for Watcom C +- Use __WATCOMC__ instead of __WATCOM__ +- Do not use the visibility attribute if NO_VIZ defined +- Improve the detection of no hidden visibility attribute +- Avoid using __int64 for gcc or solo compilation +- Cast to char * in gzprintf to avoid warnings [Zinser] +- Fix make_vms.com for VAX [Zinser] +- Don't use library or built-in byte swaps +- Simplify test and use of gcc hidden attribute +- Fix bug in gzclose_w() when gzwrite() fails to allocate memory +- Add "x" (O_EXCL) and "e" (O_CLOEXEC) modes support to gzopen() +- Fix bug in test/minigzip.c for configure --solo +- Fix contrib/vstudio project link errors [Mohanathas] +- Add ability to choose the builder in make_vms.com [Schweda] +- Add DESTDIR support to mingw32 win32/Makefile.gcc +- Fix comments in win32/Makefile.gcc for proper usage +- Allow overriding the default install locations for cmake +- Generate and install the pkg-config file with cmake +- Build both a static and a shared version of zlib with cmake +- Include version symbols for cmake builds +- If using cmake with MSVC, add the source directory to the includes +- Remove unneeded EXTRA_CFLAGS from win32/Makefile.gcc [Truta] +- Move obsolete emx makefile to old [Truta] +- Allow the use of -Wundef when compiling or using zlib +- Avoid the use of the -u option with mktemp +- Improve inflate() documentation on the use of Z_FINISH +- Recognize clang as gcc +- Add gzopen_w() in Windows for wide character path names +- Rename zconf.h in CMakeLists.txt to move it out of the way +- Add source directory in CMakeLists.txt for building examples +- Look in build directory for zlib.pc in CMakeLists.txt +- Remove gzflags from zlibvc.def in vc9 and vc10 +- Fix contrib/minizip compilation in the MinGW environment +- Update ./configure for Solaris, support --64 [Mooney] +- Remove -R. from Solaris shared build (possible security issue) +- Avoid race condition for parallel make (-j) running example +- Fix type mismatch between get_crc_table() and crc_table +- Fix parsing of version with "-" in CMakeLists.txt [Snider, Ziegler] +- Fix the path to zlib.map in CMakeLists.txt +- Force the native libtool in Mac OS X to avoid GNU libtool [Beebe] +- Add instructions to win32/Makefile.gcc for shared install [Torri] + +Changes in 1.2.6.1 (12 Feb 2012) +- Avoid the use of the Objective-C reserved name "id" +- Include io.h in gzguts.h for Microsoft compilers +- Fix problem with ./configure --prefix and gzgetc macro +- Include gz_header definition when compiling zlib solo +- Put gzflags() functionality back in zutil.c +- Avoid library header include in crc32.c for Z_SOLO +- Use name in GCC_CLASSIC as C compiler for coverage testing, if set +- Minor cleanup in contrib/minizip/zip.c [Vollant] +- Update make_vms.com [Zinser] +- Remove unnecessary gzgetc_ function +- Use optimized byte swap operations for Microsoft and GNU [Snyder] +- Fix minor typo in zlib.h comments [Rzesniowiecki] + +Changes in 1.2.6 (29 Jan 2012) +- Update the Pascal interface in contrib/pascal +- Fix function numbers for gzgetc_ in zlibvc.def files +- Fix configure.ac for contrib/minizip [Schiffer] +- Fix large-entry detection in minizip on 64-bit systems [Schiffer] +- Have ./configure use the compiler return code for error indication +- Fix CMakeLists.txt for cross compilation [McClure] +- Fix contrib/minizip/zip.c for 64-bit architectures [Dalsnes] +- Fix compilation of contrib/minizip on FreeBSD [Marquez] +- Correct suggested usages in win32/Makefile.msc [Shachar, Horvath] +- Include io.h for Turbo C / Borland C on all platforms [Truta] +- Make version explicit in contrib/minizip/configure.ac [Bosmans] +- Avoid warning for no encryption in contrib/minizip/zip.c [Vollant] +- Minor cleanup up contrib/minizip/unzip.c [Vollant] +- Fix bug when compiling minizip with C++ [Vollant] +- Protect for long name and extra fields in contrib/minizip [Vollant] +- Avoid some warnings in contrib/minizip [Vollant] +- Add -I../.. -L../.. to CFLAGS for minizip and miniunzip +- Add missing libs to minizip linker command +- Add support for VPATH builds in contrib/minizip +- Add an --enable-demos option to contrib/minizip/configure +- Add the generation of configure.log by ./configure +- Exit when required parameters not provided to win32/Makefile.gcc +- Have gzputc return the character written instead of the argument +- Use the -m option on ldconfig for BSD systems [Tobias] +- Correct in zlib.map when deflateResetKeep was added + +Changes in 1.2.5.3 (15 Jan 2012) +- Restore gzgetc function for binary compatibility +- Do not use _lseeki64 under Borland C++ [Truta] +- Update win32/Makefile.msc to build test/*.c [Truta] +- Remove old/visualc6 given CMakefile and other alternatives +- Update AS400 build files and documentation [Monnerat] +- Update win32/Makefile.gcc to build test/*.c [Truta] +- Permit stronger flushes after Z_BLOCK flushes +- Avoid extraneous empty blocks when doing empty flushes +- Permit Z_NULL arguments to deflatePending +- Allow deflatePrime() to insert bits in the middle of a stream +- Remove second empty static block for Z_PARTIAL_FLUSH +- Write out all of the available bits when using Z_BLOCK +- Insert the first two strings in the hash table after a flush + +Changes in 1.2.5.2 (17 Dec 2011) +- fix ld error: unable to find version dependency 'ZLIB_1.2.5' +- use relative symlinks for shared libs +- Avoid searching past window for Z_RLE strategy +- Assure that high-water mark initialization is always applied in deflate +- Add assertions to fill_window() in deflate.c to match comments +- Update python link in README +- Correct spelling error in gzread.c +- Fix bug in gzgets() for a concatenated empty gzip stream +- Correct error in comment for gz_make() +- Change gzread() and related to ignore junk after gzip streams +- Allow gzread() and related to continue after gzclearerr() +- Allow gzrewind() and gzseek() after a premature end-of-file +- Simplify gzseek() now that raw after gzip is ignored +- Change gzgetc() to a macro for speed (~40% speedup in testing) +- Fix gzclose() to return the actual error last encountered +- Always add large file support for windows +- Include zconf.h for windows large file support +- Include zconf.h.cmakein for windows large file support +- Update zconf.h.cmakein on make distclean +- Merge vestigial vsnprintf determination from zutil.h to gzguts.h +- Clarify how gzopen() appends in zlib.h comments +- Correct documentation of gzdirect() since junk at end now ignored +- Add a transparent write mode to gzopen() when 'T' is in the mode +- Update python link in zlib man page +- Get inffixed.h and MAKEFIXED result to match +- Add a ./config --solo option to make zlib subset with no library use +- Add undocumented inflateResetKeep() function for CAB file decoding +- Add --cover option to ./configure for gcc coverage testing +- Add #define ZLIB_CONST option to use const in the z_stream interface +- Add comment to gzdopen() in zlib.h to use dup() when using fileno() +- Note behavior of uncompress() to provide as much data as it can +- Add files in contrib/minizip to aid in building libminizip +- Split off AR options in Makefile.in and configure +- Change ON macro to Z_ARG to avoid application conflicts +- Facilitate compilation with Borland C++ for pragmas and vsnprintf +- Include io.h for Turbo C / Borland C++ +- Move example.c and minigzip.c to test/ +- Simplify incomplete code table filling in inflate_table() +- Remove code from inflate.c and infback.c that is impossible to execute +- Test the inflate code with full coverage +- Allow deflateSetDictionary, inflateSetDictionary at any time (in raw) +- Add deflateResetKeep and fix inflateResetKeep to retain dictionary +- Fix gzwrite.c to accommodate reduced memory zlib compilation +- Have inflate() with Z_FINISH avoid the allocation of a window +- Do not set strm->adler when doing raw inflate +- Fix gzeof() to behave just like feof() when read is not past end of file +- Fix bug in gzread.c when end-of-file is reached +- Avoid use of Z_BUF_ERROR in gz* functions except for premature EOF +- Document gzread() capability to read concurrently written files +- Remove hard-coding of resource compiler in CMakeLists.txt [Blammo] + +Changes in 1.2.5.1 (10 Sep 2011) +- Update FAQ entry on shared builds (#13) +- Avoid symbolic argument to chmod in Makefile.in +- Fix bug and add consts in contrib/puff [Oberhumer] +- Update contrib/puff/zeros.raw test file to have all block types +- Add full coverage test for puff in contrib/puff/Makefile +- Fix static-only-build install in Makefile.in +- Fix bug in unzGetCurrentFileInfo() in contrib/minizip [Kuno] +- Add libz.a dependency to shared in Makefile.in for parallel builds +- Spell out "number" (instead of "nb") in zlib.h for total_in, total_out +- Replace $(...) with `...` in configure for non-bash sh [Bowler] +- Add darwin* to Darwin* and solaris* to SunOS\ 5* in configure [Groffen] +- Add solaris* to Linux* in configure to allow gcc use [Groffen] +- Add *bsd* to Linux* case in configure [Bar-Lev] +- Add inffast.obj to dependencies in win32/Makefile.msc +- Correct spelling error in deflate.h [Kohler] +- Change libzdll.a again to libz.dll.a (!) in win32/Makefile.gcc +- Add test to configure for GNU C looking for gcc in output of $cc -v +- Add zlib.pc generation to win32/Makefile.gcc [Weigelt] +- Fix bug in zlib.h for _FILE_OFFSET_BITS set and _LARGEFILE64_SOURCE not +- Add comment in zlib.h that adler32_combine with len2 < 0 makes no sense +- Make NO_DIVIDE option in adler32.c much faster (thanks to John Reiser) +- Make stronger test in zconf.h to include unistd.h for LFS +- Apply Darwin patches for 64-bit file offsets to contrib/minizip [Slack] +- Fix zlib.h LFS support when Z_PREFIX used +- Add updated as400 support (removed from old) [Monnerat] +- Avoid deflate sensitivity to volatile input data +- Avoid division in adler32_combine for NO_DIVIDE +- Clarify the use of Z_FINISH with deflateBound() amount of space +- Set binary for output file in puff.c +- Use u4 type for crc_table to avoid conversion warnings +- Apply casts in zlib.h to avoid conversion warnings +- Add OF to prototypes for adler32_combine_ and crc32_combine_ [Miller] +- Improve inflateSync() documentation to note indeterminancy +- Add deflatePending() function to return the amount of pending output +- Correct the spelling of "specification" in FAQ [Randers-Pehrson] +- Add a check in configure for stdarg.h, use for gzprintf() +- Check that pointers fit in ints when gzprint() compiled old style +- Add dummy name before $(SHAREDLIBV) in Makefile [Bar-Lev, Bowler] +- Delete line in configure that adds -L. libz.a to LDFLAGS [Weigelt] +- Add debug records in assmebler code [Londer] +- Update RFC references to use http://tools.ietf.org/html/... [Li] +- Add --archs option, use of libtool to configure for Mac OS X [Borstel] + +Changes in 1.2.5 (19 Apr 2010) +- Disable visibility attribute in win32/Makefile.gcc [Bar-Lev] +- Default to libdir as sharedlibdir in configure [Nieder] +- Update copyright dates on modified source files +- Update trees.c to be able to generate modified trees.h +- Exit configure for MinGW, suggesting win32/Makefile.gcc +- Check for NULL path in gz_open [Homurlu] + +Changes in 1.2.4.5 (18 Apr 2010) +- Set sharedlibdir in configure [Torok] +- Set LDFLAGS in Makefile.in [Bar-Lev] +- Avoid mkdir objs race condition in Makefile.in [Bowler] +- Add ZLIB_INTERNAL in front of internal inter-module functions and arrays +- Define ZLIB_INTERNAL to hide internal functions and arrays for GNU C +- Don't use hidden attribute when it is a warning generator (e.g. Solaris) + +Changes in 1.2.4.4 (18 Apr 2010) +- Fix CROSS_PREFIX executable testing, CHOST extract, mingw* [Torok] +- Undefine _LARGEFILE64_SOURCE in zconf.h if it is zero, but not if empty +- Try to use bash or ksh regardless of functionality of /bin/sh +- Fix configure incompatibility with NetBSD sh +- Remove attempt to run under bash or ksh since have better NetBSD fix +- Fix win32/Makefile.gcc for MinGW [Bar-Lev] +- Add diagnostic messages when using CROSS_PREFIX in configure +- Added --sharedlibdir option to configure [Weigelt] +- Use hidden visibility attribute when available [Frysinger] + +Changes in 1.2.4.3 (10 Apr 2010) +- Only use CROSS_PREFIX in configure for ar and ranlib if they exist +- Use CROSS_PREFIX for nm [Bar-Lev] +- Assume _LARGEFILE64_SOURCE defined is equivalent to true +- Avoid use of undefined symbols in #if with && and || +- Make *64 prototypes in gzguts.h consistent with functions +- Add -shared load option for MinGW in configure [Bowler] +- Move z_off64_t to public interface, use instead of off64_t +- Remove ! from shell test in configure (not portable to Solaris) +- Change +0 macro tests to -0 for possibly increased portability + +Changes in 1.2.4.2 (9 Apr 2010) +- Add consistent carriage returns to readme.txt's in masmx86 and masmx64 +- Really provide prototypes for *64 functions when building without LFS +- Only define unlink() in minigzip.c if unistd.h not included +- Update README to point to contrib/vstudio project files +- Move projects/vc6 to old/ and remove projects/ +- Include stdlib.h in minigzip.c for setmode() definition under WinCE +- Clean up assembler builds in win32/Makefile.msc [Rowe] +- Include sys/types.h for Microsoft for off_t definition +- Fix memory leak on error in gz_open() +- Symbolize nm as $NM in configure [Weigelt] +- Use TEST_LDSHARED instead of LDSHARED to link test programs [Weigelt] +- Add +0 to _FILE_OFFSET_BITS and _LFS64_LARGEFILE in case not defined +- Fix bug in gzeof() to take into account unused input data +- Avoid initialization of structures with variables in puff.c +- Updated win32/README-WIN32.txt [Rowe] + +Changes in 1.2.4.1 (28 Mar 2010) +- Remove the use of [a-z] constructs for sed in configure [gentoo 310225] +- Remove $(SHAREDLIB) from LIBS in Makefile.in [Creech] +- Restore "for debugging" comment on sprintf() in gzlib.c +- Remove fdopen for MVS from gzguts.h +- Put new README-WIN32.txt in win32 [Rowe] +- Add check for shell to configure and invoke another shell if needed +- Fix big fat stinking bug in gzseek() on uncompressed files +- Remove vestigial F_OPEN64 define in zutil.h +- Set and check the value of _LARGEFILE_SOURCE and _LARGEFILE64_SOURCE +- Avoid errors on non-LFS systems when applications define LFS macros +- Set EXE to ".exe" in configure for MINGW [Kahle] +- Match crc32() in crc32.c exactly to the prototype in zlib.h [Sherrill] +- Add prefix for cross-compilation in win32/makefile.gcc [Bar-Lev] +- Add DLL install in win32/makefile.gcc [Bar-Lev] +- Allow Linux* or linux* from uname in configure [Bar-Lev] +- Allow ldconfig to be redefined in configure and Makefile.in [Bar-Lev] +- Add cross-compilation prefixes to configure [Bar-Lev] +- Match type exactly in gz_load() invocation in gzread.c +- Match type exactly of zcalloc() in zutil.c to zlib.h alloc_func +- Provide prototypes for *64 functions when building zlib without LFS +- Don't use -lc when linking shared library on MinGW +- Remove errno.h check in configure and vestigial errno code in zutil.h + +Changes in 1.2.4 (14 Mar 2010) +- Fix VER3 extraction in configure for no fourth subversion +- Update zlib.3, add docs to Makefile.in to make .pdf out of it +- Add zlib.3.pdf to distribution +- Don't set error code in gzerror() if passed pointer is NULL +- Apply destination directory fixes to CMakeLists.txt [Lowman] +- Move #cmakedefine's to a new zconf.in.cmakein +- Restore zconf.h for builds that don't use configure or cmake +- Add distclean to dummy Makefile for convenience +- Update and improve INDEX, README, and FAQ +- Update CMakeLists.txt for the return of zconf.h [Lowman] +- Update contrib/vstudio/vc9 and vc10 [Vollant] +- Change libz.dll.a back to libzdll.a in win32/Makefile.gcc +- Apply license and readme changes to contrib/asm686 [Raiter] +- Check file name lengths and add -c option in minigzip.c [Li] +- Update contrib/amd64 and contrib/masmx86/ [Vollant] +- Avoid use of "eof" parameter in trees.c to not shadow library variable +- Update make_vms.com for removal of zlibdefs.h [Zinser] +- Update assembler code and vstudio projects in contrib [Vollant] +- Remove outdated assembler code contrib/masm686 and contrib/asm586 +- Remove old vc7 and vc8 from contrib/vstudio +- Update win32/Makefile.msc, add ZLIB_VER_SUBREVISION [Rowe] +- Fix memory leaks in gzclose_r() and gzclose_w(), file leak in gz_open() +- Add contrib/gcc_gvmat64 for longest_match and inflate_fast [Vollant] +- Remove *64 functions from win32/zlib.def (they're not 64-bit yet) +- Fix bug in void-returning vsprintf() case in gzwrite.c +- Fix name change from inflate.h in contrib/inflate86/inffas86.c +- Check if temporary file exists before removing in make_vms.com [Zinser] +- Fix make install and uninstall for --static option +- Fix usage of _MSC_VER in gzguts.h and zutil.h [Truta] +- Update readme.txt in contrib/masmx64 and masmx86 to assemble + +Changes in 1.2.3.9 (21 Feb 2010) +- Expunge gzio.c +- Move as400 build information to old +- Fix updates in contrib/minizip and contrib/vstudio +- Add const to vsnprintf test in configure to avoid warnings [Weigelt] +- Delete zconf.h (made by configure) [Weigelt] +- Change zconf.in.h to zconf.h.in per convention [Weigelt] +- Check for NULL buf in gzgets() +- Return empty string for gzgets() with len == 1 (like fgets()) +- Fix description of gzgets() in zlib.h for end-of-file, NULL return +- Update minizip to 1.1 [Vollant] +- Avoid MSVC loss of data warnings in gzread.c, gzwrite.c +- Note in zlib.h that gzerror() should be used to distinguish from EOF +- Remove use of snprintf() from gzlib.c +- Fix bug in gzseek() +- Update contrib/vstudio, adding vc9 and vc10 [Kuno, Vollant] +- Fix zconf.h generation in CMakeLists.txt [Lowman] +- Improve comments in zconf.h where modified by configure + +Changes in 1.2.3.8 (13 Feb 2010) +- Clean up text files (tabs, trailing whitespace, etc.) [Oberhumer] +- Use z_off64_t in gz_zero() and gz_skip() to match state->skip +- Avoid comparison problem when sizeof(int) == sizeof(z_off64_t) +- Revert to Makefile.in from 1.2.3.6 (live with the clutter) +- Fix missing error return in gzflush(), add zlib.h note +- Add *64 functions to zlib.map [Levin] +- Fix signed/unsigned comparison in gz_comp() +- Use SFLAGS when testing shared linking in configure +- Add --64 option to ./configure to use -m64 with gcc +- Fix ./configure --help to correctly name options +- Have make fail if a test fails [Levin] +- Avoid buffer overrun in contrib/masmx64/gvmat64.asm [Simpson] +- Remove assembler object files from contrib + +Changes in 1.2.3.7 (24 Jan 2010) +- Always gzopen() with O_LARGEFILE if available +- Fix gzdirect() to work immediately after gzopen() or gzdopen() +- Make gzdirect() more precise when the state changes while reading +- Improve zlib.h documentation in many places +- Catch memory allocation failure in gz_open() +- Complete close operation if seek forward in gzclose_w() fails +- Return Z_ERRNO from gzclose_r() if close() fails +- Return Z_STREAM_ERROR instead of EOF for gzclose() being passed NULL +- Return zero for gzwrite() errors to match zlib.h description +- Return -1 on gzputs() error to match zlib.h description +- Add zconf.in.h to allow recovery from configure modification [Weigelt] +- Fix static library permissions in Makefile.in [Weigelt] +- Avoid warnings in configure tests that hide functionality [Weigelt] +- Add *BSD and DragonFly to Linux case in configure [gentoo 123571] +- Change libzdll.a to libz.dll.a in win32/Makefile.gcc [gentoo 288212] +- Avoid access of uninitialized data for first inflateReset2 call [Gomes] +- Keep object files in subdirectories to reduce the clutter somewhat +- Remove default Makefile and zlibdefs.h, add dummy Makefile +- Add new external functions to Z_PREFIX, remove duplicates, z_z_ -> z_ +- Remove zlibdefs.h completely -- modify zconf.h instead + +Changes in 1.2.3.6 (17 Jan 2010) +- Avoid void * arithmetic in gzread.c and gzwrite.c +- Make compilers happier with const char * for gz_error message +- Avoid unused parameter warning in inflate.c +- Avoid signed-unsigned comparison warning in inflate.c +- Indent #pragma's for traditional C +- Fix usage of strwinerror() in glib.c, change to gz_strwinerror() +- Correct email address in configure for system options +- Update make_vms.com and add make_vms.com to contrib/minizip [Zinser] +- Update zlib.map [Brown] +- Fix Makefile.in for Solaris 10 make of example64 and minizip64 [Torok] +- Apply various fixes to CMakeLists.txt [Lowman] +- Add checks on len in gzread() and gzwrite() +- Add error message for no more room for gzungetc() +- Remove zlib version check in gzwrite() +- Defer compression of gzprintf() result until need to +- Use snprintf() in gzdopen() if available +- Remove USE_MMAP configuration determination (only used by minigzip) +- Remove examples/pigz.c (available separately) +- Update examples/gun.c to 1.6 + +Changes in 1.2.3.5 (8 Jan 2010) +- Add space after #if in zutil.h for some compilers +- Fix relatively harmless bug in deflate_fast() [Exarevsky] +- Fix same problem in deflate_slow() +- Add $(SHAREDLIBV) to LIBS in Makefile.in [Brown] +- Add deflate_rle() for faster Z_RLE strategy run-length encoding +- Add deflate_huff() for faster Z_HUFFMAN_ONLY encoding +- Change name of "write" variable in inffast.c to avoid library collisions +- Fix premature EOF from gzread() in gzio.c [Brown] +- Use zlib header window size if windowBits is 0 in inflateInit2() +- Remove compressBound() call in deflate.c to avoid linking compress.o +- Replace use of errno in gz* with functions, support WinCE [Alves] +- Provide alternative to perror() in minigzip.c for WinCE [Alves] +- Don't use _vsnprintf on later versions of MSVC [Lowman] +- Add CMake build script and input file [Lowman] +- Update contrib/minizip to 1.1 [Svensson, Vollant] +- Moved nintendods directory from contrib to . +- Replace gzio.c with a new set of routines with the same functionality +- Add gzbuffer(), gzoffset(), gzclose_r(), gzclose_w() as part of above +- Update contrib/minizip to 1.1b +- Change gzeof() to return 0 on error instead of -1 to agree with zlib.h + +Changes in 1.2.3.4 (21 Dec 2009) +- Use old school .SUFFIXES in Makefile.in for FreeBSD compatibility +- Update comments in configure and Makefile.in for default --shared +- Fix test -z's in configure [Marquess] +- Build examplesh and minigzipsh when not testing +- Change NULL's to Z_NULL's in deflate.c and in comments in zlib.h +- Import LDFLAGS from the environment in configure +- Fix configure to populate SFLAGS with discovered CFLAGS options +- Adapt make_vms.com to the new Makefile.in [Zinser] +- Add zlib2ansi script for C++ compilation [Marquess] +- Add _FILE_OFFSET_BITS=64 test to make test (when applicable) +- Add AMD64 assembler code for longest match to contrib [Teterin] +- Include options from $SFLAGS when doing $LDSHARED +- Simplify 64-bit file support by introducing z_off64_t type +- Make shared object files in objs directory to work around old Sun cc +- Use only three-part version number for Darwin shared compiles +- Add rc option to ar in Makefile.in for when ./configure not run +- Add -WI,-rpath,. to LDFLAGS for OSF 1 V4* +- Set LD_LIBRARYN32_PATH for SGI IRIX shared compile +- Protect against _FILE_OFFSET_BITS being defined when compiling zlib +- Rename Makefile.in targets allstatic to static and allshared to shared +- Fix static and shared Makefile.in targets to be independent +- Correct error return bug in gz_open() by setting state [Brown] +- Put spaces before ;;'s in configure for better sh compatibility +- Add pigz.c (parallel implementation of gzip) to examples/ +- Correct constant in crc32.c to UL [Leventhal] +- Reject negative lengths in crc32_combine() +- Add inflateReset2() function to work like inflateEnd()/inflateInit2() +- Include sys/types.h for _LARGEFILE64_SOURCE [Brown] +- Correct typo in doc/algorithm.txt [Janik] +- Fix bug in adler32_combine() [Zhu] +- Catch missing-end-of-block-code error in all inflates and in puff + Assures that random input to inflate eventually results in an error +- Added enough.c (calculation of ENOUGH for inftrees.h) to examples/ +- Update ENOUGH and its usage to reflect discovered bounds +- Fix gzerror() error report on empty input file [Brown] +- Add ush casts in trees.c to avoid pedantic runtime errors +- Fix typo in zlib.h uncompress() description [Reiss] +- Correct inflate() comments with regard to automatic header detection +- Remove deprecation comment on Z_PARTIAL_FLUSH (it stays) +- Put new version of gzlog (2.0) in examples with interruption recovery +- Add puff compile option to permit invalid distance-too-far streams +- Add puff TEST command options, ability to read piped input +- Prototype the *64 functions in zlib.h when _FILE_OFFSET_BITS == 64, but + _LARGEFILE64_SOURCE not defined +- Fix Z_FULL_FLUSH to truly erase the past by resetting s->strstart +- Fix deflateSetDictionary() to use all 32K for output consistency +- Remove extraneous #define MIN_LOOKAHEAD in deflate.c (in deflate.h) +- Clear bytes after deflate lookahead to avoid use of uninitialized data +- Change a limit in inftrees.c to be more transparent to Coverity Prevent +- Update win32/zlib.def with exported symbols from zlib.h +- Correct spelling errors in zlib.h [Willem, Sobrado] +- Allow Z_BLOCK for deflate() to force a new block +- Allow negative bits in inflatePrime() to delete existing bit buffer +- Add Z_TREES flush option to inflate() to return at end of trees +- Add inflateMark() to return current state information for random access +- Add Makefile for NintendoDS to contrib [Costa] +- Add -w in configure compile tests to avoid spurious warnings [Beucler] +- Fix typos in zlib.h comments for deflateSetDictionary() +- Fix EOF detection in transparent gzread() [Maier] + +Changes in 1.2.3.3 (2 October 2006) +- Make --shared the default for configure, add a --static option +- Add compile option to permit invalid distance-too-far streams +- Add inflateUndermine() function which is required to enable above +- Remove use of "this" variable name for C++ compatibility [Marquess] +- Add testing of shared library in make test, if shared library built +- Use ftello() and fseeko() if available instead of ftell() and fseek() +- Provide two versions of all functions that use the z_off_t type for + binary compatibility -- a normal version and a 64-bit offset version, + per the Large File Support Extension when _LARGEFILE64_SOURCE is + defined; use the 64-bit versions by default when _FILE_OFFSET_BITS + is defined to be 64 +- Add a --uname= option to configure to perhaps help with cross-compiling + +Changes in 1.2.3.2 (3 September 2006) +- Turn off silly Borland warnings [Hay] +- Use off64_t and define _LARGEFILE64_SOURCE when present +- Fix missing dependency on inffixed.h in Makefile.in +- Rig configure --shared to build both shared and static [Teredesai, Truta] +- Remove zconf.in.h and instead create a new zlibdefs.h file +- Fix contrib/minizip/unzip.c non-encrypted after encrypted [Vollant] +- Add treebuild.xml (see http://treebuild.metux.de/) [Weigelt] + +Changes in 1.2.3.1 (16 August 2006) +- Add watcom directory with OpenWatcom make files [Daniel] +- Remove #undef of FAR in zconf.in.h for MVS [Fedtke] +- Update make_vms.com [Zinser] +- Use -fPIC for shared build in configure [Teredesai, Nicholson] +- Use only major version number for libz.so on IRIX and OSF1 [Reinholdtsen] +- Use fdopen() (not _fdopen()) for Interix in zutil.h [Bäck] +- Add some FAQ entries about the contrib directory +- Update the MVS question in the FAQ +- Avoid extraneous reads after EOF in gzio.c [Brown] +- Correct spelling of "successfully" in gzio.c [Randers-Pehrson] +- Add comments to zlib.h about gzerror() usage [Brown] +- Set extra flags in gzip header in gzopen() like deflate() does +- Make configure options more compatible with double-dash conventions + [Weigelt] +- Clean up compilation under Solaris SunStudio cc [Rowe, Reinholdtsen] +- Fix uninstall target in Makefile.in [Truta] +- Add pkgconfig support [Weigelt] +- Use $(DESTDIR) macro in Makefile.in [Reinholdtsen, Weigelt] +- Replace set_data_type() with a more accurate detect_data_type() in + trees.c, according to the txtvsbin.txt document [Truta] +- Swap the order of #include and #include "zlib.h" in + gzio.c, example.c and minigzip.c [Truta] +- Shut up annoying VS2005 warnings about standard C deprecation [Rowe, + Truta] (where?) +- Fix target "clean" from win32/Makefile.bor [Truta] +- Create .pdb and .manifest files in win32/makefile.msc [Ziegler, Rowe] +- Update zlib www home address in win32/DLL_FAQ.txt [Truta] +- Update contrib/masmx86/inffas32.asm for VS2005 [Vollant, Van Wassenhove] +- Enable browse info in the "Debug" and "ASM Debug" configurations in + the Visual C++ 6 project, and set (non-ASM) "Debug" as default [Truta] +- Add pkgconfig support [Weigelt] +- Add ZLIB_VER_MAJOR, ZLIB_VER_MINOR and ZLIB_VER_REVISION in zlib.h, + for use in win32/zlib1.rc [Polushin, Rowe, Truta] +- Add a document that explains the new text detection scheme to + doc/txtvsbin.txt [Truta] +- Add rfc1950.txt, rfc1951.txt and rfc1952.txt to doc/ [Truta] +- Move algorithm.txt into doc/ [Truta] +- Synchronize FAQ with website +- Fix compressBound(), was low for some pathological cases [Fearnley] +- Take into account wrapper variations in deflateBound() +- Set examples/zpipe.c input and output to binary mode for Windows +- Update examples/zlib_how.html with new zpipe.c (also web site) +- Fix some warnings in examples/gzlog.c and examples/zran.c (it seems + that gcc became pickier in 4.0) +- Add zlib.map for Linux: "All symbols from zlib-1.1.4 remain + un-versioned, the patch adds versioning only for symbols introduced in + zlib-1.2.0 or later. It also declares as local those symbols which are + not designed to be exported." [Levin] +- Update Z_PREFIX list in zconf.in.h, add --zprefix option to configure +- Do not initialize global static by default in trees.c, add a response + NO_INIT_GLOBAL_POINTERS to initialize them if needed [Marquess] +- Don't use strerror() in gzio.c under WinCE [Yakimov] +- Don't use errno.h in zutil.h under WinCE [Yakimov] +- Move arguments for AR to its usage to allow replacing ar [Marot] +- Add HAVE_VISIBILITY_PRAGMA in zconf.in.h for Mozilla [Randers-Pehrson] +- Improve inflateInit() and inflateInit2() documentation +- Fix structure size comment in inflate.h +- Change configure help option from --h* to --help [Santos] + Changes in 1.2.3 (18 July 2005) - Apply security vulnerability fixes to contrib/infback9 as well - Clean up some text files (carriage returns, trailing space) @@ -13,7 +673,7 @@ Changes in 1.2.2.4 (11 July 2005) compile - Fix some spelling errors in comments [Betts] - Correct inflateInit2() error return documentation in zlib.h -- Added zran.c example of compressed data random access to examples +- Add zran.c example of compressed data random access to examples directory, shows use of inflatePrime() - Fix cast for assignments to strm->state in inflate.c and infback.c - Fix zlibCompileFlags() in zutil.c to use 1L for long shifts [Oberhumer] @@ -561,7 +1221,7 @@ Changes in 1.0.6 (19 Jan 1998) 386 asm code replacing longest_match(). contrib/iostream/ by Kevin Ruland A C++ I/O streams interface to the zlib gz* functions - contrib/iostream2/ by Tyge Lvset + contrib/iostream2/ by Tyge Løvset Another C++ I/O streams interface contrib/untgz/ by "Pedro A. Aranda Guti\irrez" A very simple tar.gz file extractor using zlib @@ -650,7 +1310,7 @@ Changes in 1.0.1 (20 May 96) [1.0 skipped to avoid confusion] - fix array overlay in deflate.c which sometimes caused bad compressed data - fix inflate bug with empty stored block - fix MSDOS medium model which was broken in 0.99 -- fix deflateParams() which could generated bad compressed data. +- fix deflateParams() which could generate bad compressed data. - Bytef is define'd instead of typedef'ed (work around Borland bug) - added an INDEX file - new makefiles for DJGPP (Makefile.dj2), 32-bit Borland (Makefile.b32), diff --git a/zlib/FAQ b/zlib/FAQ index 441d910daa1..99b7cf92e45 100644 --- a/zlib/FAQ +++ b/zlib/FAQ @@ -3,8 +3,8 @@ If your question is not there, please check the zlib home page -http://www.zlib.org which may have more recent information. -The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html +http://zlib.net/ which may have more recent information. +The lastest zlib FAQ is at http://zlib.net/zlib_faq.html 1. Is zlib Y2K-compliant? @@ -13,54 +13,51 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 2. Where can I get a Windows DLL version? - The zlib sources can be compiled without change to produce a DLL. - See the file win32/DLL_FAQ.txt in the zlib distribution. - Pointers to the precompiled DLL are found in the zlib web site at - http://www.zlib.org. + The zlib sources can be compiled without change to produce a DLL. See the + file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the + precompiled DLL are found in the zlib web site at http://zlib.net/ . 3. Where can I get a Visual Basic interface to zlib? See - * http://www.dogma.net/markn/articles/zlibtool/zlibtool.htm - * contrib/visual-basic.txt in the zlib distribution + * http://marknelson.us/1997/01/01/zlib-engine/ * win32/DLL_FAQ.txt in the zlib distribution 4. compress() returns Z_BUF_ERROR. - Make sure that before the call of compress, the length of the compressed - buffer is equal to the total size of the compressed buffer and not - zero. For Visual Basic, check that this parameter is passed by reference + Make sure that before the call of compress(), the length of the compressed + buffer is equal to the available size of the compressed buffer and not + zero. For Visual Basic, check that this parameter is passed by reference ("as any"), not by value ("as long"). 5. deflate() or inflate() returns Z_BUF_ERROR. - Before making the call, make sure that avail_in and avail_out are not - zero. When setting the parameter flush equal to Z_FINISH, also make sure - that avail_out is big enough to allow processing all pending input. - Note that a Z_BUF_ERROR is not fatal--another call to deflate() or - inflate() can be made with more input or output space. A Z_BUF_ERROR - may in fact be unavoidable depending on how the functions are used, since - it is not possible to tell whether or not there is more output pending - when strm.avail_out returns with zero. + Before making the call, make sure that avail_in and avail_out are not zero. + When setting the parameter flush equal to Z_FINISH, also make sure that + avail_out is big enough to allow processing all pending input. Note that a + Z_BUF_ERROR is not fatal--another call to deflate() or inflate() can be + made with more input or output space. A Z_BUF_ERROR may in fact be + unavoidable depending on how the functions are used, since it is not + possible to tell whether or not there is more output pending when + strm.avail_out returns with zero. See http://zlib.net/zlib_how.html for a + heavily annotated example. 6. Where's the zlib documentation (man pages, etc.)? - It's in zlib.h for the moment, and Francis S. Lin has converted it to a - web page zlib.html. Volunteers to transform this to Unix-style man pages, - please contact us (zlib@gzip.org). Examples of zlib usage are in the files - example.c and minigzip.c. + It's in zlib.h . Examples of zlib usage are in the files test/example.c + and test/minigzip.c, with more in examples/ . 7. Why don't you use GNU autoconf or libtool or ...? - Because we would like to keep zlib as a very small and simple - package. zlib is rather portable and doesn't need much configuration. + Because we would like to keep zlib as a very small and simple package. + zlib is rather portable and doesn't need much configuration. 8. I found a bug in zlib. - Most of the time, such problems are due to an incorrect usage of - zlib. Please try to reproduce the problem with a small program and send - the corresponding source to us at zlib@gzip.org . Do not send - multi-megabyte data files without prior agreement. + Most of the time, such problems are due to an incorrect usage of zlib. + Please try to reproduce the problem with a small program and send the + corresponding source to us at zlib@gzip.org . Do not send multi-megabyte + data files without prior agreement. 9. Why do I get "undefined reference to gzputc"? @@ -82,13 +79,15 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 12. Can zlib handle .Z files? - No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt + No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt the code of uncompress on your own. 13. How can I make a Unix shared library? - make clean - ./configure -s + By default a shared (and a static) library is built for Unix. So: + + make distclean + ./configure make 14. How do I install a shared zlib library on Unix? @@ -99,8 +98,10 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html However, many flavors of Unix come with a shared zlib already installed. Before going to the trouble of compiling a shared version of zlib and - trying to install it, you may want to check if it's already there! If you - can #include , it's there. The -lz option will probably link to it. + trying to install it, you may want to check if it's already there! If you + can #include , it's there. The -lz option will probably link to + it. You can check the version at the top of zlib.h or with the + ZLIB_VERSION symbol defined in zlib.h . 15. I have a question about OttoPDF. @@ -109,8 +110,8 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 16. Can zlib decode Flate data in an Adobe PDF file? - Yes. See http://www.fastio.com/ (ClibPDF), or http://www.pdflib.com/ . - To modify PDF forms, see http://sourceforge.net/projects/acroformtool/ . + Yes. See http://www.pdflib.com/ . To modify PDF forms, see + http://sourceforge.net/projects/acroformtool/ . 17. Why am I getting this "register_frame_info not found" error on Solaris? @@ -121,67 +122,67 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html symbol __register_frame_info: referenced symbol not found The symbol __register_frame_info is not part of zlib, it is generated by - the C compiler (cc or gcc). You must recompile applications using zlib - which have this problem. This problem is specific to Solaris. See + the C compiler (cc or gcc). You must recompile applications using zlib + which have this problem. This problem is specific to Solaris. See http://www.sunfreeware.com for Solaris versions of zlib and applications using zlib. 18. Why does gzip give an error on a file I make with compress/deflate? The compress and deflate functions produce data in the zlib format, which - is different and incompatible with the gzip format. The gz* functions in - zlib on the other hand use the gzip format. Both the zlib and gzip - formats use the same compressed data format internally, but have different - headers and trailers around the compressed data. + is different and incompatible with the gzip format. The gz* functions in + zlib on the other hand use the gzip format. Both the zlib and gzip formats + use the same compressed data format internally, but have different headers + and trailers around the compressed data. 19. Ok, so why are there two different formats? - The gzip format was designed to retain the directory information about - a single file, such as the name and last modification date. The zlib - format on the other hand was designed for in-memory and communication - channel applications, and has a much more compact header and trailer and - uses a faster integrity check than gzip. + The gzip format was designed to retain the directory information about a + single file, such as the name and last modification date. The zlib format + on the other hand was designed for in-memory and communication channel + applications, and has a much more compact header and trailer and uses a + faster integrity check than gzip. 20. Well that's nice, but how do I make a gzip file in memory? You can request that deflate write the gzip format instead of the zlib - format using deflateInit2(). You can also request that inflate decode - the gzip format using inflateInit2(). Read zlib.h for more details. + format using deflateInit2(). You can also request that inflate decode the + gzip format using inflateInit2(). Read zlib.h for more details. 21. Is zlib thread-safe? - Yes. However any library routines that zlib uses and any application- - provided memory allocation routines must also be thread-safe. zlib's gz* + Yes. However any library routines that zlib uses and any application- + provided memory allocation routines must also be thread-safe. zlib's gz* functions use stdio library routines, and most of zlib's functions use the - library memory allocation routines by default. zlib's Init functions allow - for the application to provide custom memory allocation routines. + library memory allocation routines by default. zlib's *Init* functions + allow for the application to provide custom memory allocation routines. Of course, you should only operate on any given zlib or gzip stream from a single thread at a time. 22. Can I use zlib in my commercial application? - Yes. Please read the license in zlib.h. + Yes. Please read the license in zlib.h. 23. Is zlib under the GNU license? - No. Please read the license in zlib.h. + No. Please read the license in zlib.h. 24. The license says that altered source versions must be "plainly marked". So what exactly do I need to do to meet that requirement? - You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In + You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In particular, the final version number needs to be changed to "f", and an - identification string should be appended to ZLIB_VERSION. Version numbers + identification string should be appended to ZLIB_VERSION. Version numbers x.x.x.f are reserved for modifications to zlib by others than the zlib - maintainers. For example, if the version of the base zlib you are altering + maintainers. For example, if the version of the base zlib you are altering is "1.2.3.4", then in zlib.h you should change ZLIB_VERNUM to 0x123f, and - ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also + ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also update the version strings in deflate.c and inftrees.c. For altered source distributions, you should also note the origin and nature of the changes in zlib.h, as well as in ChangeLog and README, along - with the dates of the alterations. The origin should include at least your + with the dates of the alterations. The origin should include at least your name (or your company's name), and an email address to contact for help or issues with the library. @@ -197,105 +198,112 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 26. Will zlib work on a 64-bit machine? - It should. It has been tested on 64-bit machines, and has no dependence - on any data types being limited to 32-bits in length. If you have any + Yes. It has been tested on 64-bit machines, and has no dependence on any + data types being limited to 32-bits in length. If you have any difficulties, please provide a complete problem report to zlib@gzip.org 27. Will zlib decompress data from the PKWare Data Compression Library? - No. The PKWare DCL uses a completely different compressed data format - than does PKZIP and zlib. However, you can look in zlib's contrib/blast + No. The PKWare DCL uses a completely different compressed data format than + does PKZIP and zlib. However, you can look in zlib's contrib/blast directory for a possible solution to your problem. 28. Can I access data randomly in a compressed stream? - No, not without some preparation. If when compressing you periodically - use Z_FULL_FLUSH, carefully write all the pending data at those points, - and keep an index of those locations, then you can start decompression - at those points. You have to be careful to not use Z_FULL_FLUSH too - often, since it can significantly degrade compression. + No, not without some preparation. If when compressing you periodically use + Z_FULL_FLUSH, carefully write all the pending data at those points, and + keep an index of those locations, then you can start decompression at those + points. You have to be careful to not use Z_FULL_FLUSH too often, since it + can significantly degrade compression. Alternatively, you can scan a + deflate stream once to generate an index, and then use that index for + random access. See examples/zran.c . 29. Does zlib work on MVS, OS/390, CICS, etc.? - We don't know for sure. We have heard occasional reports of success on - these systems. If you do use it on one of these, please provide us with - a report, instructions, and patches that we can reference when we get - these questions. Thanks. + It has in the past, but we have not heard of any recent evidence. There + were working ports of zlib 1.1.4 to MVS, but those links no longer work. + If you know of recent, successful applications of zlib on these operating + systems, please let us know. Thanks. -30. Is there some simpler, easier to read version of inflate I can look at - to understand the deflate format? +30. Is there some simpler, easier to read version of inflate I can look at to + understand the deflate format? - First off, you should read RFC 1951. Second, yes. Look in zlib's + First off, you should read RFC 1951. Second, yes. Look in zlib's contrib/puff directory. 31. Does zlib infringe on any patents? - As far as we know, no. In fact, that was originally the whole point behind - zlib. Look here for some more information: + As far as we know, no. In fact, that was originally the whole point behind + zlib. Look here for some more information: http://www.gzip.org/#faq11 32. Can zlib work with greater than 4 GB of data? - Yes. inflate() and deflate() will process any amount of data correctly. + Yes. inflate() and deflate() will process any amount of data correctly. Each call of inflate() or deflate() is limited to input and output chunks of the maximum value that can be stored in the compiler's "unsigned int" - type, but there is no limit to the number of chunks. Note however that the - strm.total_in and strm_total_out counters may be limited to 4 GB. These + type, but there is no limit to the number of chunks. Note however that the + strm.total_in and strm_total_out counters may be limited to 4 GB. These counters are provided as a convenience and are not used internally by - inflate() or deflate(). The application can easily set up its own counters + inflate() or deflate(). The application can easily set up its own counters updated after each call of inflate() or deflate() to count beyond 4 GB. compress() and uncompress() may be limited to 4 GB, since they operate in a - single call. gzseek() and gztell() may be limited to 4 GB depending on how - zlib is compiled. See the zlibCompileFlags() function in zlib.h. + single call. gzseek() and gztell() may be limited to 4 GB depending on how + zlib is compiled. See the zlibCompileFlags() function in zlib.h. - The word "may" appears several times above since there is a 4 GB limit - only if the compiler's "long" type is 32 bits. If the compiler's "long" - type is 64 bits, then the limit is 16 exabytes. + The word "may" appears several times above since there is a 4 GB limit only + if the compiler's "long" type is 32 bits. If the compiler's "long" type is + 64 bits, then the limit is 16 exabytes. 33. Does zlib have any security vulnerabilities? - The only one that we are aware of is potentially in gzprintf(). If zlib - is compiled to use sprintf() or vsprintf(), then there is no protection - against a buffer overflow of a 4K string space, other than the caller of - gzprintf() assuring that the output will not exceed 4K. On the other - hand, if zlib is compiled to use snprintf() or vsnprintf(), which should - normally be the case, then there is no vulnerability. The ./configure - script will display warnings if an insecure variation of sprintf() will - be used by gzprintf(). Also the zlibCompileFlags() function will return - information on what variant of sprintf() is used by gzprintf(). + The only one that we are aware of is potentially in gzprintf(). If zlib is + compiled to use sprintf() or vsprintf(), then there is no protection + against a buffer overflow of an 8K string space (or other value as set by + gzbuffer()), other than the caller of gzprintf() assuring that the output + will not exceed 8K. On the other hand, if zlib is compiled to use + snprintf() or vsnprintf(), which should normally be the case, then there is + no vulnerability. The ./configure script will display warnings if an + insecure variation of sprintf() will be used by gzprintf(). Also the + zlibCompileFlags() function will return information on what variant of + sprintf() is used by gzprintf(). If you don't have snprintf() or vsnprintf() and would like one, you can find a portable implementation here: http://www.ijs.si/software/snprintf/ - Note that you should be using the most recent version of zlib. Versions - 1.1.3 and before were subject to a double-free vulnerability. + Note that you should be using the most recent version of zlib. Versions + 1.1.3 and before were subject to a double-free vulnerability, and versions + 1.2.1 and 1.2.2 were subject to an access exception when decompressing + invalid compressed data. 34. Is there a Java version of zlib? Probably what you want is to use zlib in Java. zlib is already included as part of the Java SDK in the java.util.zip package. If you really want a version of zlib written in the Java language, look on the zlib home - page for links: http://www.zlib.org/ + page for links: http://zlib.net/ . 35. I get this or that compiler or source-code scanner warning when I crank it up to maximally-pedantic. Can't you guys write proper code? Many years ago, we gave up attempting to avoid warnings on every compiler - in the universe. It just got to be a waste of time, and some compilers - were downright silly. So now, we simply make sure that the code always - works. + in the universe. It just got to be a waste of time, and some compilers + were downright silly as well as contradicted each other. So now, we simply + make sure that the code always works. 36. Valgrind (or some similar memory access checker) says that deflate is performing a conditional jump that depends on an uninitialized value. Isn't that a bug? - No. That is intentional for performance reasons, and the output of - deflate is not affected. This only started showing up recently since - zlib 1.2.x uses malloc() by default for allocations, whereas earlier - versions used calloc(), which zeros out the allocated memory. + No. That is intentional for performance reasons, and the output of deflate + is not affected. This only started showing up recently since zlib 1.2.x + uses malloc() by default for allocations, whereas earlier versions used + calloc(), which zeros out the allocated memory. Even though the code was + correct, versions 1.2.4 and later was changed to not stimulate these + checkers. 37. Will zlib read the (insert any ancient or arcane format here) compressed data format? @@ -305,20 +313,21 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 38. How can I encrypt/decrypt zip files with zlib? - zlib doesn't support encryption. The original PKZIP encryption is very weak - and can be broken with freely available programs. To get strong encryption, - use GnuPG, http://www.gnupg.org/ , which already includes zlib compression. - For PKZIP compatible "encryption", look at http://www.info-zip.org/ + zlib doesn't support encryption. The original PKZIP encryption is very + weak and can be broken with freely available programs. To get strong + encryption, use GnuPG, http://www.gnupg.org/ , which already includes zlib + compression. For PKZIP compatible "encryption", look at + http://www.info-zip.org/ 39. What's the difference between the "gzip" and "deflate" HTTP 1.1 encodings? - "gzip" is the gzip format, and "deflate" is the zlib format. They should - probably have called the second one "zlib" instead to avoid confusion - with the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 + "gzip" is the gzip format, and "deflate" is the zlib format. They should + probably have called the second one "zlib" instead to avoid confusion with + the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 correctly points to the zlib specification in RFC 1950 for the "deflate" transfer encoding, there have been reports of servers and browsers that incorrectly produce or expect raw deflate data per the deflate - specficiation in RFC 1951, most notably Microsoft. So even though the + specification in RFC 1951, most notably Microsoft. So even though the "deflate" transfer encoding using the zlib format would be the more efficient approach (and in fact exactly what the zlib format was designed for), using the "gzip" transfer encoding is probably more reliable due to @@ -328,12 +337,32 @@ The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html 40. Does zlib support the new "Deflate64" format introduced by PKWare? - No. PKWare has apparently decided to keep that format proprietary, since - they have not documented it as they have previous compression formats. - In any case, the compression improvements are so modest compared to other - more modern approaches, that it's not worth the effort to implement. + No. PKWare has apparently decided to keep that format proprietary, since + they have not documented it as they have previous compression formats. In + any case, the compression improvements are so modest compared to other more + modern approaches, that it's not worth the effort to implement. -41. Can you please sign these lengthy legal documents and fax them back to us +41. I'm having a problem with the zip functions in zlib, can you help? + + There are no zip functions in zlib. You are probably using minizip by + Giles Vollant, which is found in the contrib directory of zlib. It is not + part of zlib. In fact none of the stuff in contrib is part of zlib. The + files in there are not supported by the zlib authors. You need to contact + the authors of the respective contribution for help. + +42. The match.asm code in contrib is under the GNU General Public License. + Since it's part of zlib, doesn't that mean that all of zlib falls under the + GNU GPL? + + No. The files in contrib are not part of zlib. They were contributed by + other authors and are provided as a convenience to the user within the zlib + distribution. Each item in contrib has its own license. + +43. Is zlib subject to export controls? What is its ECCN? + + zlib is not subject to export controls, and so is classified as EAR99. + +44. Can you please sign these lengthy legal documents and fax them back to us so that we can use your software in our product? No. Go away. Shoo. diff --git a/zlib/INDEX b/zlib/INDEX index 0587e5902bd..2ba06412048 100644 --- a/zlib/INDEX +++ b/zlib/INDEX @@ -1,23 +1,37 @@ +CMakeLists.txt cmake build file ChangeLog history of changes FAQ Frequently Asked Questions about zlib INDEX this file -Makefile makefile for Unix (generated by configure) -Makefile.in makefile for Unix (template for configure) +Makefile dummy Makefile that tells you to ./configure +Makefile.in template for Unix Makefile README guess what -algorithm.txt description of the (de)compression algorithm configure configure script for Unix -zconf.in.h template for zconf.h (used by configure) +make_vms.com makefile for VMS +test/example.c zlib usages examples for build testing +test/minigzip.c minimal gzip-like functionality for build testing +test/infcover.c inf*.c code coverage for build coverage testing +treebuild.xml XML description of source file dependencies +zconf.h.cmakein zconf.h template for cmake +zconf.h.in zconf.h template for configure +zlib.3 Man page for zlib +zlib.3.pdf Man page in PDF format +zlib.map Linux symbol information +zlib.pc.in Template for pkg-config descriptor +zlib.pc.cmakein zlib.pc template for cmake +zlib2ansi perl script to convert source files for C++ compilation amiga/ makefiles for Amiga SAS C -as400/ makefiles for IBM AS/400 +as400/ makefiles for AS/400 +doc/ documentation for formats and algorithms msdos/ makefiles for MSDOS +nintendods/ makefile for Nintendo DS old/ makefiles for various architectures and zlib documentation files that have not yet been updated for zlib 1.2.x -projects/ projects for various Integrated Development Environments qnx/ makefiles for QNX +watcom/ makefiles for OpenWatcom win32/ makefiles for Windows - zlib public header files (must be kept): + zlib public header files (required for library use): zconf.h zlib.h @@ -28,7 +42,11 @@ crc32.c crc32.h deflate.c deflate.h -gzio.c +gzclose.c +gzguts.h +gzlib.c +gzread.c +gzwrite.c infback.c inffast.c inffast.h @@ -43,9 +61,8 @@ uncompr.c zutil.c zutil.h - source files for sample programs: -example.c -minigzip.c + source files for sample programs +See examples/README.examples - unsupported contribution by third parties + unsupported contributions by third parties See contrib/README.contrib diff --git a/zlib/README b/zlib/README index 758cc50020d..51106de4753 100644 --- a/zlib/README +++ b/zlib/README @@ -1,56 +1,52 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.2.3 is a general purpose data compression library. All the code is +zlib 1.2.11 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files -http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) -and rfc1952.txt (gzip format). These documents are also available in other -formats from ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html +http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and +rfc1952 (gzip format). All functions of the compression library are documented in the file zlib.h -(volunteer to write man pages welcome, contact zlib@gzip.org). A usage example -of the library is given in the file example.c which also tests that the library -is working correctly. Another example is given in the file minigzip.c. The -compression library itself is composed of all source files except example.c and -minigzip.c. +(volunteer to write man pages welcome, contact zlib@gzip.org). A usage example +of the library is given in the file test/example.c which also tests that +the library is working correctly. Another example is given in the file +test/minigzip.c. The compression library itself is composed of all source +files in the root directory. To compile all files and run the test program, follow the instructions given at -the top of Makefile. In short "make test; make install" should work for most -machines. For Unix: "./configure; make test; make install". For MSDOS, use one -of the special makefiles such as Makefile.msc. For VMS, use make_vms.com. +the top of Makefile.in. In short "./configure; make test", and if that goes +well, "make install" should work for most flavors of Unix. For Windows, use +one of the special makefiles in win32/ or contrib/vstudio/ . For VMS, use +make_vms.com. Questions about zlib should be sent to , or to Gilles Vollant - for the Windows DLL version. The zlib home page is -http://www.zlib.org or http://www.gzip.org/zlib/ Before reporting a problem, -please check this site to verify that you have the latest version of zlib; -otherwise get the latest version and check whether the problem still exists or -not. + for the Windows DLL version. The zlib home page is +http://zlib.net/ . Before reporting a problem, please check this site to +verify that you have the latest version of zlib; otherwise get the latest +version and check whether the problem still exists or not. -PLEASE read the zlib FAQ http://www.gzip.org/zlib/zlib_faq.html before asking -for help. +PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help. -Mark Nelson wrote an article about zlib for the Jan. 1997 -issue of Dr. Dobb's Journal; a copy of the article is available in -http://dogma.net/markn/articles/zlibtool/zlibtool.htm +Mark Nelson wrote an article about zlib for the Jan. 1997 +issue of Dr. Dobb's Journal; a copy of the article is available at +http://marknelson.us/1997/01/01/zlib-engine/ . -The changes made in version 1.2.3 are documented in the file ChangeLog. +The changes made in version 1.2.11 are documented in the file ChangeLog. -Unsupported third party contributions are provided in directory "contrib". +Unsupported third party contributions are provided in directory contrib/ . -A Java implementation of zlib is available in the Java Development Kit -http://java.sun.com/j2se/1.4.2/docs/api/java/util/zip/package-summary.html -See the zlib home page http://www.zlib.org for details. +zlib is available in Java using the java.util.zip package, documented at +http://java.sun.com/developer/technicalArticles/Programming/compression/ . -A Perl interface to zlib written by Paul Marquess is in the -CPAN (Comprehensive Perl Archive Network) sites -http://www.cpan.org/modules/by-module/Compress/ +A Perl interface to zlib written by Paul Marquess is available +at CPAN (Comprehensive Perl Archive Network) sites, including +http://search.cpan.org/~pmqs/IO-Compress-Zlib/ . A Python interface to zlib written by A.M. Kuchling is available in Python 1.5 and later versions, see -http://www.python.org/doc/lib/module-zlib.html +http://docs.python.org/library/zlib.html . -A zlib binding for TCL written by Andreas Kupries is -availlable at http://www.oche.de/~akupries/soft/trf/trf_zip.html +zlib is built into tcl: http://wiki.tcl.tk/4610 . An experimental package to read and write files in .zip format, written on top of zlib by Gilles Vollant , is available in the @@ -74,25 +70,21 @@ Notes for some targets: - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with other compilers. Use "make test" to check your compiler. -- gzdopen is not supported on RISCOS, BEOS and by some Mac compilers. +- gzdopen is not supported on RISCOS or BEOS. - For PalmOs, see http://palmzlib.sourceforge.net/ -- When building a shared, i.e. dynamic library on Mac OS X, the library must be - installed before testing (do "make install" before "make test"), since the - library location is specified in the library. - Acknowledgments: - The deflate format used by zlib was defined by Phil Katz. The deflate - and zlib specifications were written by L. Peter Deutsch. Thanks to all the - people who reported problems and suggested various improvements in zlib; - they are too numerous to cite here. + The deflate format used by zlib was defined by Phil Katz. The deflate and + zlib specifications were written by L. Peter Deutsch. Thanks to all the + people who reported problems and suggested various improvements in zlib; they + are too numerous to cite here. Copyright notice: - (C) 1995-2004 Jean-loup Gailly and Mark Adler + (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -113,13 +105,11 @@ Copyright notice: Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu -If you use the zlib library in a product, we would appreciate *not* -receiving lengthy legal documents to sign. The sources are provided -for free but without warranty of any kind. The library has been -entirely written by Jean-loup Gailly and Mark Adler; it does not -include third-party code. +If you use the zlib library in a product, we would appreciate *not* receiving +lengthy legal documents to sign. The sources are provided for free but without +warranty of any kind. The library has been entirely written by Jean-loup +Gailly and Mark Adler; it does not include third-party code. -If you redistribute modified sources, we would appreciate that you include -in the file ChangeLog history information documenting your changes. Please -read the FAQ for more information on the distribution of modified source -versions. +If you redistribute modified sources, we would appreciate that you include in +the file ChangeLog history information documenting your changes. Please read +the FAQ for more information on the distribution of modified source versions. diff --git a/zlib/README.MySQL b/zlib/README.MySQL deleted file mode 100644 index c17d3eeb6f9..00000000000 --- a/zlib/README.MySQL +++ /dev/null @@ -1,16 +0,0 @@ -This an incomplete version of the zlib library -- it excludes some of the -platform-specific project files, contributed code, and examples from the -original zlib distribution. You can find the original distribution at - - http://www.gzip.org/zlib/ - or - http://www.zlib.net/ - -Revision history: - -20.01.2006. The following files were changed as part of #15787 fix: - makefile.am - gzio.c - zconf.h - README.mysql - diff --git a/zlib/adler32.c b/zlib/adler32.c index 007ba26277c..e82e8eedd13 100644 --- a/zlib/adler32.c +++ b/zlib/adler32.c @@ -1,14 +1,15 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2004 Mark Adler + * Copyright (C) 1995-2011, 2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#define ZLIB_INTERNAL -#include "zlib.h" +#include "zutil.h" -#define BASE 65521UL /* largest prime smaller than 65536 */ +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); + +#define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -18,46 +19,51 @@ #define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); #define DO16(buf) DO8(buf,0); DO8(buf,8); -/* use NO_DIVIDE if your processor does not do division in hardware */ +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ #ifdef NO_DIVIDE -# define MOD(a) \ +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ do { \ - if (a >= (BASE << 16)) a -= (BASE << 16); \ - if (a >= (BASE << 15)) a -= (BASE << 15); \ - if (a >= (BASE << 14)) a -= (BASE << 14); \ - if (a >= (BASE << 13)) a -= (BASE << 13); \ - if (a >= (BASE << 12)) a -= (BASE << 12); \ - if (a >= (BASE << 11)) a -= (BASE << 11); \ - if (a >= (BASE << 10)) a -= (BASE << 10); \ - if (a >= (BASE << 9)) a -= (BASE << 9); \ - if (a >= (BASE << 8)) a -= (BASE << 8); \ - if (a >= (BASE << 7)) a -= (BASE << 7); \ - if (a >= (BASE << 6)) a -= (BASE << 6); \ - if (a >= (BASE << 5)) a -= (BASE << 5); \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ + do { \ + CHOP(a); \ if (a >= BASE) a -= BASE; \ } while (0) -# define MOD4(a) \ +# define MOD(a) \ do { \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ + CHOP(a); \ + MOD28(a); \ + } while (0) +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ if (a >= BASE) a -= BASE; \ } while (0) #else # define MOD(a) a %= BASE -# define MOD4(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE #endif /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) +uLong ZEXPORT adler32_z(adler, buf, len) uLong adler; const Bytef *buf; - uInt len; + z_size_t len; { unsigned long sum2; unsigned n; @@ -89,7 +95,7 @@ uLong ZEXPORT adler32(adler, buf, len) } if (adler >= BASE) adler -= BASE; - MOD4(sum2); /* only added so many BASE's */ + MOD28(sum2); /* only added so many BASE's */ return adler | (sum2 << 16); } @@ -125,25 +131,56 @@ uLong ZEXPORT adler32(adler, buf, len) } /* ========================================================================= */ -uLong ZEXPORT adler32_combine(adler1, adler2, len2) +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ +local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; uLong adler2; - z_off_t len2; + z_off64_t len2; { unsigned long sum1; unsigned long sum2; unsigned rem; + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + /* the derivation of this formula is left as an exercise for the reader */ - rem = (unsigned)(len2 % BASE); + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; sum1 = adler1 & 0xffff; sum2 = rem * sum1; MOD(sum2); sum1 += (adler2 & 0xffff) + BASE - 1; sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; - if (sum1 > BASE) sum1 -= BASE; - if (sum1 > BASE) sum1 -= BASE; - if (sum2 > (BASE << 1)) sum2 -= (BASE << 1); - if (sum2 > BASE) sum2 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); + if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } + +/* ========================================================================= */ +uLong ZEXPORT adler32_combine(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} + +uLong ZEXPORT adler32_combine64(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} diff --git a/zlib/algorithm.txt b/zlib/algorithm.txt deleted file mode 100644 index b022dde312a..00000000000 --- a/zlib/algorithm.txt +++ /dev/null @@ -1,209 +0,0 @@ -1. Compression algorithm (deflate) - -The deflation algorithm used by gzip (also zip and zlib) is a variation of -LZ77 (Lempel-Ziv 1977, see reference below). It finds duplicated strings in -the input data. The second occurrence of a string is replaced by a -pointer to the previous string, in the form of a pair (distance, -length). Distances are limited to 32K bytes, and lengths are limited -to 258 bytes. When a string does not occur anywhere in the previous -32K bytes, it is emitted as a sequence of literal bytes. (In this -description, `string' must be taken as an arbitrary sequence of bytes, -and is not restricted to printable characters.) - -Literals or match lengths are compressed with one Huffman tree, and -match distances are compressed with another tree. The trees are stored -in a compact form at the start of each block. The blocks can have any -size (except that the compressed data for one block must fit in -available memory). A block is terminated when deflate() determines that -it would be useful to start another block with fresh trees. (This is -somewhat similar to the behavior of LZW-based _compress_.) - -Duplicated strings are found using a hash table. All input strings of -length 3 are inserted in the hash table. A hash index is computed for -the next 3 bytes. If the hash chain for this index is not empty, all -strings in the chain are compared with the current input string, and -the longest match is selected. - -The hash chains are searched starting with the most recent strings, to -favor small distances and thus take advantage of the Huffman encoding. -The hash chains are singly linked. There are no deletions from the -hash chains, the algorithm simply discards matches that are too old. - -To avoid a worst-case situation, very long hash chains are arbitrarily -truncated at a certain length, determined by a runtime option (level -parameter of deflateInit). So deflate() does not always find the longest -possible match but generally finds a match which is long enough. - -deflate() also defers the selection of matches with a lazy evaluation -mechanism. After a match of length N has been found, deflate() searches for -a longer match at the next input byte. If a longer match is found, the -previous match is truncated to a length of one (thus producing a single -literal byte) and the process of lazy evaluation begins again. Otherwise, -the original match is kept, and the next match search is attempted only N -steps later. - -The lazy match evaluation is also subject to a runtime parameter. If -the current match is long enough, deflate() reduces the search for a longer -match, thus speeding up the whole process. If compression ratio is more -important than speed, deflate() attempts a complete second search even if -the first match is already long enough. - -The lazy match evaluation is not performed for the fastest compression -modes (level parameter 1 to 3). For these fast modes, new strings -are inserted in the hash table only when no match was found, or -when the match is not too long. This degrades the compression ratio -but saves time since there are both fewer insertions and fewer searches. - - -2. Decompression algorithm (inflate) - -2.1 Introduction - -The key question is how to represent a Huffman code (or any prefix code) so -that you can decode fast. The most important characteristic is that shorter -codes are much more common than longer codes, so pay attention to decoding the -short codes fast, and let the long codes take longer to decode. - -inflate() sets up a first level table that covers some number of bits of -input less than the length of longest code. It gets that many bits from the -stream, and looks it up in the table. The table will tell if the next -code is that many bits or less and how many, and if it is, it will tell -the value, else it will point to the next level table for which inflate() -grabs more bits and tries to decode a longer code. - -How many bits to make the first lookup is a tradeoff between the time it -takes to decode and the time it takes to build the table. If building the -table took no time (and if you had infinite memory), then there would only -be a first level table to cover all the way to the longest code. However, -building the table ends up taking a lot longer for more bits since short -codes are replicated many times in such a table. What inflate() does is -simply to make the number of bits in the first table a variable, and then -to set that variable for the maximum speed. - -For inflate, which has 286 possible codes for the literal/length tree, the size -of the first table is nine bits. Also the distance trees have 30 possible -values, and the size of the first table is six bits. Note that for each of -those cases, the table ended up one bit longer than the ``average'' code -length, i.e. the code length of an approximately flat code which would be a -little more than eight bits for 286 symbols and a little less than five bits -for 30 symbols. - - -2.2 More details on the inflate table lookup - -Ok, you want to know what this cleverly obfuscated inflate tree actually -looks like. You are correct that it's not a Huffman tree. It is simply a -lookup table for the first, let's say, nine bits of a Huffman symbol. The -symbol could be as short as one bit or as long as 15 bits. If a particular -symbol is shorter than nine bits, then that symbol's translation is duplicated -in all those entries that start with that symbol's bits. For example, if the -symbol is four bits, then it's duplicated 32 times in a nine-bit table. If a -symbol is nine bits long, it appears in the table once. - -If the symbol is longer than nine bits, then that entry in the table points -to another similar table for the remaining bits. Again, there are duplicated -entries as needed. The idea is that most of the time the symbol will be short -and there will only be one table look up. (That's whole idea behind data -compression in the first place.) For the less frequent long symbols, there -will be two lookups. If you had a compression method with really long -symbols, you could have as many levels of lookups as is efficient. For -inflate, two is enough. - -So a table entry either points to another table (in which case nine bits in -the above example are gobbled), or it contains the translation for the symbol -and the number of bits to gobble. Then you start again with the next -ungobbled bit. - -You may wonder: why not just have one lookup table for how ever many bits the -longest symbol is? The reason is that if you do that, you end up spending -more time filling in duplicate symbol entries than you do actually decoding. -At least for deflate's output that generates new trees every several 10's of -kbytes. You can imagine that filling in a 2^15 entry table for a 15-bit code -would take too long if you're only decoding several thousand symbols. At the -other extreme, you could make a new table for every bit in the code. In fact, -that's essentially a Huffman tree. But then you spend two much time -traversing the tree while decoding, even for short symbols. - -So the number of bits for the first lookup table is a trade of the time to -fill out the table vs. the time spent looking at the second level and above of -the table. - -Here is an example, scaled down: - -The code being decoded, with 10 symbols, from 1 to 6 bits long: - -A: 0 -B: 10 -C: 1100 -D: 11010 -E: 11011 -F: 11100 -G: 11101 -H: 11110 -I: 111110 -J: 111111 - -Let's make the first table three bits long (eight entries): - -000: A,1 -001: A,1 -010: A,1 -011: A,1 -100: B,2 -101: B,2 -110: -> table X (gobble 3 bits) -111: -> table Y (gobble 3 bits) - -Each entry is what the bits decode as and how many bits that is, i.e. how -many bits to gobble. Or the entry points to another table, with the number of -bits to gobble implicit in the size of the table. - -Table X is two bits long since the longest code starting with 110 is five bits -long: - -00: C,1 -01: C,1 -10: D,2 -11: E,2 - -Table Y is three bits long since the longest code starting with 111 is six -bits long: - -000: F,2 -001: F,2 -010: G,2 -011: G,2 -100: H,2 -101: H,2 -110: I,3 -111: J,3 - -So what we have here are three tables with a total of 20 entries that had to -be constructed. That's compared to 64 entries for a single table. Or -compared to 16 entries for a Huffman tree (six two entry tables and one four -entry table). Assuming that the code ideally represents the probability of -the symbols, it takes on the average 1.25 lookups per symbol. That's compared -to one lookup for the single table, or 1.66 lookups per symbol for the -Huffman tree. - -There, I think that gives you a picture of what's going on. For inflate, the -meaning of a particular symbol is often more than just a letter. It can be a -byte (a "literal"), or it can be either a length or a distance which -indicates a base value and a number of bits to fetch after the code that is -added to the base value. Or it might be the special end-of-block code. The -data structures created in inftrees.c try to encode all that information -compactly in the tables. - - -Jean-loup Gailly Mark Adler -jloup@gzip.org madler@alumni.caltech.edu - - -References: - -[LZ77] Ziv J., Lempel A., ``A Universal Algorithm for Sequential Data -Compression,'' IEEE Transactions on Information Theory, Vol. 23, No. 3, -pp. 337-343. - -``DEFLATE Compressed Data Format Specification'' available in -http://www.ietf.org/rfc/rfc1951.txt diff --git a/zlib/amiga/Makefile.pup b/zlib/amiga/Makefile.pup new file mode 100644 index 00000000000..8940c120fbb --- /dev/null +++ b/zlib/amiga/Makefile.pup @@ -0,0 +1,69 @@ +# Amiga powerUP (TM) Makefile +# makefile for libpng and SAS C V6.58/7.00 PPC compiler +# Copyright (C) 1998 by Andreas R. Kleinert + +LIBNAME = libzip.a + +CC = scppc +CFLAGS = NOSTKCHK NOSINT OPTIMIZE OPTGO OPTPEEP OPTINLOCAL OPTINL \ + OPTLOOP OPTRDEP=8 OPTDEP=8 OPTCOMP=8 NOVER +AR = ppc-amigaos-ar cr +RANLIB = ppc-amigaos-ranlib +LD = ppc-amigaos-ld -r +LDFLAGS = -o +LDLIBS = LIB:scppc.a LIB:end.o +RM = delete quiet + +OBJS = adler32.o compress.o crc32.o gzclose.o gzlib.o gzread.o gzwrite.o \ + uncompr.o deflate.o trees.o zutil.o inflate.o infback.o inftrees.o inffast.o + +TEST_OBJS = example.o minigzip.o + +all: example minigzip + +check: test +test: all + example + echo hello world | minigzip | minigzip -d + +$(LIBNAME): $(OBJS) + $(AR) $@ $(OBJS) + -$(RANLIB) $@ + +example: example.o $(LIBNAME) + $(LD) $(LDFLAGS) $@ LIB:c_ppc.o $@.o $(LIBNAME) $(LDLIBS) + +minigzip: minigzip.o $(LIBNAME) + $(LD) $(LDFLAGS) $@ LIB:c_ppc.o $@.o $(LIBNAME) $(LDLIBS) + +mostlyclean: clean +clean: + $(RM) *.o example minigzip $(LIBNAME) foo.gz + +zip: + zip -ul9 zlib README ChangeLog Makefile Make????.??? Makefile.?? \ + descrip.mms *.[ch] + +tgz: + cd ..; tar cfz zlib/zlib.tgz zlib/README zlib/ChangeLog zlib/Makefile \ + zlib/Make????.??? zlib/Makefile.?? zlib/descrip.mms zlib/*.[ch] + +# DO NOT DELETE THIS LINE -- make depend depends on it. + +adler32.o: zlib.h zconf.h +compress.o: zlib.h zconf.h +crc32.o: crc32.h zlib.h zconf.h +deflate.o: deflate.h zutil.h zlib.h zconf.h +example.o: zlib.h zconf.h +gzclose.o: zlib.h zconf.h gzguts.h +gzlib.o: zlib.h zconf.h gzguts.h +gzread.o: zlib.h zconf.h gzguts.h +gzwrite.o: zlib.h zconf.h gzguts.h +inffast.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +inflate.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +infback.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +inftrees.o: zutil.h zlib.h zconf.h inftrees.h +minigzip.o: zlib.h zconf.h +trees.o: deflate.h zutil.h zlib.h zconf.h trees.h +uncompr.o: zlib.h zconf.h +zutil.o: zutil.h zlib.h zconf.h diff --git a/zlib/amiga/Makefile.sas b/zlib/amiga/Makefile.sas new file mode 100644 index 00000000000..749e2915271 --- /dev/null +++ b/zlib/amiga/Makefile.sas @@ -0,0 +1,68 @@ +# SMakefile for zlib +# Modified from the standard UNIX Makefile Copyright Jean-loup Gailly +# Osma Ahvenlampi +# Amiga, SAS/C 6.56 & Smake + +CC=sc +CFLAGS=OPT +#CFLAGS=OPT CPU=68030 +#CFLAGS=DEBUG=LINE +LDFLAGS=LIB z.lib + +SCOPTIONS=OPTSCHED OPTINLINE OPTALIAS OPTTIME OPTINLOCAL STRMERGE \ + NOICONS PARMS=BOTH NOSTACKCHECK UTILLIB NOVERSION ERRORREXX \ + DEF=POSTINC + +OBJS = adler32.o compress.o crc32.o gzclose.o gzlib.o gzread.o gzwrite.o \ + uncompr.o deflate.o trees.o zutil.o inflate.o infback.o inftrees.o inffast.o + +TEST_OBJS = example.o minigzip.o + +all: SCOPTIONS example minigzip + +check: test +test: all + example + echo hello world | minigzip | minigzip -d + +install: z.lib + copy clone zlib.h zconf.h INCLUDE: + copy clone z.lib LIB: + +z.lib: $(OBJS) + oml z.lib r $(OBJS) + +example: example.o z.lib + $(CC) $(CFLAGS) LINK TO $@ example.o $(LDFLAGS) + +minigzip: minigzip.o z.lib + $(CC) $(CFLAGS) LINK TO $@ minigzip.o $(LDFLAGS) + +mostlyclean: clean +clean: + -delete force quiet example minigzip *.o z.lib foo.gz *.lnk SCOPTIONS + +SCOPTIONS: Makefile.sas + copy to $@ 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; -#endif - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + left = *destLen; + *destLen = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; @@ -46,15 +41,26 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) err = deflateInit(&stream, level); if (err != Z_OK) return err; - err = deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err == Z_OK ? Z_BUF_ERROR : err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; - err = deflateEnd(&stream); - return err; + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); + + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; } /* =========================================================================== @@ -75,5 +81,6 @@ int ZEXPORT compress (dest, destLen, source, sourceLen) uLong ZEXPORT compressBound (sourceLen) uLong sourceLen; { - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + 11; + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; } diff --git a/zlib/crc32.c b/zlib/crc32.c index f658a9ef55e..9580440c0e6 100644 --- a/zlib/crc32.c +++ b/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster @@ -17,6 +17,8 @@ of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should first call get_crc_table() to initialize the tables before allowing more than one thread to use crc32(). + + DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. */ #ifdef MAKECRCH @@ -28,37 +30,15 @@ #include "zutil.h" /* for STDC and FAR definitions */ -#define local static - -/* Find a four-byte integer type for crc32_little() and crc32_big(). */ -#ifndef NOBYFOUR -# ifdef STDC /* need ANSI C limits.h to determine sizes */ -# include -# define BYFOUR -# if (UINT_MAX == 0xffffffffUL) - typedef unsigned int u4; -# else -# if (ULONG_MAX == 0xffffffffUL) - typedef unsigned long u4; -# else -# if (USHRT_MAX == 0xffffffffUL) - typedef unsigned short u4; -# else -# undef BYFOUR /* can't find a four-byte integer type! */ -# endif -# endif -# endif -# endif /* STDC */ -#endif /* !NOBYFOUR */ - /* Definitions for doing the crc four data bytes at a time. */ +#if !defined(NOBYFOUR) && defined(Z_U4) +# define BYFOUR +#endif #ifdef BYFOUR -# define REV(w) (((w)>>24)+(((w)>>8)&0xff00)+ \ - (((w)&0xff00)<<8)+(((w)&0xff)<<24)) local unsigned long crc32_little OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); local unsigned long crc32_big OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); # define TBLS 8 #else # define TBLS 1 @@ -68,14 +48,16 @@ local unsigned long gf2_matrix_times OF((unsigned long *mat, unsigned long vec)); local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); +local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); + #ifdef DYNAMIC_CRC_TABLE local volatile int crc_table_empty = 1; -local unsigned long FAR crc_table[TBLS][256]; +local z_crc_t FAR crc_table[TBLS][256]; local void make_crc_table OF((void)); #ifdef MAKECRCH - local void write_table OF((FILE *, const unsigned long FAR *)); + local void write_table OF((FILE *, const z_crc_t FAR *)); #endif /* MAKECRCH */ /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: @@ -105,9 +87,9 @@ local void make_crc_table OF((void)); */ local void make_crc_table() { - unsigned long c; + z_crc_t c; int n, k; - unsigned long poly; /* polynomial exclusive-or pattern */ + z_crc_t poly; /* polynomial exclusive-or pattern */ /* terms of polynomial defining this crc (except x^32): */ static volatile int first = 1; /* flag to limit concurrent making */ static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; @@ -119,13 +101,13 @@ local void make_crc_table() first = 0; /* make exclusive-or pattern from polynomial (0xedb88320UL) */ - poly = 0UL; - for (n = 0; n < sizeof(p)/sizeof(unsigned char); n++) - poly |= 1UL << (31 - p[n]); + poly = 0; + for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++) + poly |= (z_crc_t)1 << (31 - p[n]); /* generate a crc for every 8-bit value */ for (n = 0; n < 256; n++) { - c = (unsigned long)n; + c = (z_crc_t)n; for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >> 1) : c >> 1; crc_table[0][n] = c; @@ -136,11 +118,11 @@ local void make_crc_table() and then the byte reversal of those as well as the first table */ for (n = 0; n < 256; n++) { c = crc_table[0][n]; - crc_table[4][n] = REV(c); + crc_table[4][n] = ZSWAP32(c); for (k = 1; k < 4; k++) { c = crc_table[0][c & 0xff] ^ (c >> 8); crc_table[k][n] = c; - crc_table[k + 4][n] = REV(c); + crc_table[k + 4][n] = ZSWAP32(c); } } #endif /* BYFOUR */ @@ -162,7 +144,7 @@ local void make_crc_table() if (out == NULL) return; fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by crc32.c\n */\n\n"); - fprintf(out, "local const unsigned long FAR "); + fprintf(out, "local const z_crc_t FAR "); fprintf(out, "crc_table[TBLS][256] =\n{\n {\n"); write_table(out, crc_table[0]); # ifdef BYFOUR @@ -182,12 +164,13 @@ local void make_crc_table() #ifdef MAKECRCH local void write_table(out, table) FILE *out; - const unsigned long FAR *table; + const z_crc_t FAR *table; { int n; for (n = 0; n < 256; n++) - fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", table[n], + fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", + (unsigned long)(table[n]), n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); } #endif /* MAKECRCH */ @@ -202,13 +185,13 @@ local void write_table(out, table) /* ========================================================================= * This function can be used by asm versions of crc32() */ -const unsigned long FAR * ZEXPORT get_crc_table() +const z_crc_t FAR * ZEXPORT get_crc_table() { #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ - return (const unsigned long FAR *)crc_table; + return (const z_crc_t FAR *)crc_table; } /* ========================================================================= */ @@ -216,10 +199,10 @@ const unsigned long FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) +unsigned long ZEXPORT crc32_z(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { if (buf == Z_NULL) return 0UL; @@ -230,7 +213,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) #ifdef BYFOUR if (sizeof(void *) == sizeof(ptrdiff_t)) { - u4 endian; + z_crc_t endian; endian = 1; if (*((unsigned char *)(&endian))) @@ -250,8 +233,29 @@ unsigned long ZEXPORT crc32(crc, buf, len) return crc ^ 0xffffffffUL; } +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ + return crc32_z(crc, buf, len); +} + #ifdef BYFOUR +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ @@ -262,19 +266,19 @@ unsigned long ZEXPORT crc32(crc, buf, len) local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = (u4)crc; + c = (z_crc_t)crc; c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOLIT32; len -= 32; @@ -293,7 +297,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -302,20 +306,19 @@ local unsigned long crc32_little(crc, buf, len) local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = REV((u4)crc); + c = ZSWAP32((z_crc_t)crc); c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; - buf4--; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOBIG32; len -= 32; @@ -324,14 +327,13 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; - return (unsigned long)(REV(c)); + return (unsigned long)(ZSWAP32(c)); } #endif /* BYFOUR */ @@ -367,22 +369,22 @@ local void gf2_matrix_square(square, mat) } /* ========================================================================= */ -uLong ZEXPORT crc32_combine(crc1, crc2, len2) +local uLong crc32_combine_(crc1, crc2, len2) uLong crc1; uLong crc2; - z_off_t len2; + z_off64_t len2; { int n; unsigned long row; unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */ unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */ - /* degenerate case */ - if (len2 == 0) + /* degenerate case (also disallow negative lengths) */ + if (len2 <= 0) return crc1; /* put operator for one zero bit in odd */ - odd[0] = 0xedb88320L; /* CRC-32 polynomial */ + odd[0] = 0xedb88320UL; /* CRC-32 polynomial */ row = 1; for (n = 1; n < GF2_DIM; n++) { odd[n] = row; @@ -421,3 +423,20 @@ uLong ZEXPORT crc32_combine(crc1, crc2, len2) crc1 ^= crc2; return crc1; } + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} + +uLong ZEXPORT crc32_combine64(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off64_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} diff --git a/zlib/crc32.h b/zlib/crc32.h index 8053b6117c0..9e0c7781025 100644 --- a/zlib/crc32.h +++ b/zlib/crc32.h @@ -2,7 +2,7 @@ * Generated automatically by crc32.c */ -local const unsigned long FAR crc_table[TBLS][256] = +local const z_crc_t FAR crc_table[TBLS][256] = { { 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, diff --git a/zlib/deflate.c b/zlib/deflate.c index 29ce1f64a57..1ec761448de 100644 --- a/zlib/deflate.c +++ b/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -37,7 +37,7 @@ * REFERENCES * * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". - * Available in http://www.ietf.org/rfc/rfc1951.txt + * Available in http://tools.ietf.org/html/rfc1951 * * A description of the Rabin and Karp algorithm is given in the book * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. @@ -52,7 +52,7 @@ #include "deflate.h" const char deflate_copyright[] = - " deflate 1.2.3 Copyright 1995-2005 Jean-loup Gailly "; + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -73,27 +73,29 @@ typedef enum { typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); local void fill_window OF((deflate_state *s)); local block_state deflate_stored OF((deflate_state *s, int flush)); local block_state deflate_fast OF((deflate_state *s, int flush)); #ifndef FASTEST local block_state deflate_slow OF((deflate_state *s, int flush)); #endif +local block_state deflate_rle OF((deflate_state *s, int flush)); +local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); -#ifndef FASTEST +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#endif -local uInt longest_match_fast OF((deflate_state *s, IPos cur_match)); -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif @@ -110,11 +112,6 @@ local void check_match OF((deflate_state *s, IPos start, IPos match, #endif /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ -#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) -/* Minimum amount of lookahead, except at the end of the input file. - * See deflate.c for comments about the MIN_MATCH+1. - */ - /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to * exclude worst case performance for pathological files. Better values may be @@ -154,18 +151,14 @@ local const config configuration_table[10] = { * meaning. */ -#define EQUAL 0 -/* result of memcmp for equal strings */ - -#ifndef NO_DUMMY_DECL -struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ -#endif +/* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ +#define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) /* =========================================================================== * Update a hash value with the given input byte - * IN assertion: all calls to to UPDATE_HASH are made with consecutive - * input characters, so that a running hash key can be computed from the - * previous key instead of complete recalculation each time. + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. */ #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) @@ -176,9 +169,9 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. - * IN assertion: all calls to to INSERT_STRING are made with consecutive - * input characters and the first MIN_MATCH bytes of str are valid - * (except for the last MIN_MATCH-1 bytes of the input file). + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). */ #ifdef FASTEST #define INSERT_STRING(s, str, match_head) \ @@ -200,6 +193,37 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ s->head[s->hash_size-1] = NIL; \ zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local void slide_hash(s) + deflate_state *s; +{ + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; + + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif +} + /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; @@ -241,10 +265,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, strm->msg = Z_NULL; if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif #ifdef FASTEST if (level != 0) level = 1; @@ -264,7 +297,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || - strategy < 0 || strategy > Z_FIXED) { + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ @@ -272,14 +305,15 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ s->wrap = wrap; s->gzhead = Z_NULL; - s->w_bits = windowBits; + s->w_bits = (uInt)windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - s->hash_bits = memLevel + 7; + s->hash_bits = (uInt)memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); @@ -288,6 +322,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + s->high_water = 0; /* nothing written to s->window yet */ + s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */ overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2); @@ -297,7 +333,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || s->pending_buf == Z_NULL) { s->status = FINISH_STATE; - strm->msg = (char*)ERR_MSG(Z_MEM_ERROR); + strm->msg = ERR_MSG(Z_MEM_ERROR); deflateEnd (strm); return Z_MEM_ERROR; } @@ -311,6 +347,31 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, return deflateReset(strm); } +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -318,49 +379,97 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) uInt dictLength; { deflate_state *s; - uInt length = dictLength; - uInt n; - IPos hash_head = 0; + uInt str, n; + int wrap; + unsigned avail; + z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL || - strm->state->wrap == 2 || - (strm->state->wrap == 1 && strm->state->status != INIT_STATE)) + if (deflateStateCheck(strm) || dictionary == Z_NULL) + return Z_STREAM_ERROR; + s = strm->state; + wrap = s->wrap; + if (wrap == 2 || (wrap == 1 && s->status != INIT_STATE) || s->lookahead) return Z_STREAM_ERROR; - s = strm->state; - if (s->wrap) + /* when using zlib wrappers, compute Adler-32 for provided dictionary */ + if (wrap == 1) strm->adler = adler32(strm->adler, dictionary, dictLength); + s->wrap = 0; /* avoid computing Adler-32 in read_buf */ - if (length < MIN_MATCH) return Z_OK; - if (length > MAX_DIST(s)) { - length = MAX_DIST(s); - dictionary += dictLength - length; /* use the tail of the dictionary */ + /* if dictionary would fill window, just replace the history */ + if (dictLength >= s->w_size) { + if (wrap == 0) { /* already empty otherwise */ + CLEAR_HASH(s); + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } + dictionary += dictLength - s->w_size; /* use the tail */ + dictLength = s->w_size; } - zmemcpy(s->window, dictionary, length); - s->strstart = length; - s->block_start = (long)length; - /* Insert all strings in the hash table (except for the last two bytes). - * s->lookahead stays null, so s->ins_h will be recomputed at the next - * call of fill_window. - */ - s->ins_h = s->window[0]; - UPDATE_HASH(s, s->ins_h, s->window[1]); - for (n = 0; n <= length - MIN_MATCH; n++) { - INSERT_STRING(s, n, hash_head); + /* insert dictionary into window and hash */ + avail = strm->avail_in; + next = strm->next_in; + strm->avail_in = dictLength; + strm->next_in = (z_const Bytef *)dictionary; + fill_window(s); + while (s->lookahead >= MIN_MATCH) { + str = s->strstart; + n = s->lookahead - (MIN_MATCH-1); + do { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + } while (--n); + s->strstart = str; + s->lookahead = MIN_MATCH-1; + fill_window(s); } - if (hash_head) hash_head = 0; /* to make compiler happy */ + s->strstart += s->lookahead; + s->block_start = (long)s->strstart; + s->insert = s->lookahead; + s->lookahead = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + strm->next_in = next; + strm->avail_in = avail; + s->wrap = wrap; return Z_OK; } /* ========================================================================= */ -int ZEXPORT deflateReset (strm) +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ + deflate_state *s; + uInt len; + + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { + if (deflateStateCheck(strm)) { return Z_STREAM_ERROR; } @@ -375,7 +484,11 @@ int ZEXPORT deflateReset (strm) if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } - s->status = s->wrap ? INIT_STATE : BUSY_STATE; + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : @@ -384,31 +497,70 @@ int ZEXPORT deflateReset (strm) s->last_flush = Z_NO_FLUSH; _tr_init(s); - lm_init(s); return Z_OK; } +/* ========================================================================= */ +int ZEXPORT deflateReset (strm) + z_streamp strm; +{ + int ret; + + ret = deflateResetKeep(strm); + if (ret == Z_OK) + lm_init(strm->state); + return ret; +} + /* ========================================================================= */ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - if (strm->state->wrap != 2) return Z_STREAM_ERROR; + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } +/* ========================================================================= */ +int ZEXPORT deflatePending (strm, pending, bits) + unsigned *pending; + int *bits; + z_streamp strm; +{ + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + if (pending != Z_NULL) + *pending = strm->state->pending; + if (bits != Z_NULL) + *bits = strm->state->bi_valid; + return Z_OK; +} + /* ========================================================================= */ int ZEXPORT deflatePrime (strm, bits, value) z_streamp strm; int bits; int value; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - strm->state->bi_valid = bits; - strm->state->bi_buf = (ush)(value & ((1 << bits) - 1)); + deflate_state *s; + int put; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + s = strm->state; + if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; + do { + put = Buf_size - s->bi_valid; + if (put > bits) + put = bits; + s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); + s->bi_valid += put; + _tr_flush_bits(s); + value >>= put; + bits -= put; + } while (bits); return Z_OK; } @@ -420,9 +572,8 @@ int ZEXPORT deflateParams(strm, level, strategy) { deflate_state *s; compress_func func; - int err = Z_OK; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST @@ -435,11 +586,23 @@ int ZEXPORT deflateParams(strm, level, strategy) } func = configuration_table[s->level].func; - if (func != configuration_table[level].func && strm->total_in != 0) { + if ((strategy != s->strategy || func != configuration_table[level].func) && + s->high_water) { /* Flush the last buffer: */ - err = deflate(strm, Z_PARTIAL_FLUSH); + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_out == 0) + return Z_BUF_ERROR; } if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; @@ -447,7 +610,7 @@ int ZEXPORT deflateParams(strm, level, strategy) s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; - return err; + return Z_OK; } /* ========================================================================= */ @@ -460,12 +623,12 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; - s->good_match = good_length; - s->max_lazy_match = max_lazy; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; s->nice_match = nice_length; - s->max_chain_length = max_chain; + s->max_chain_length = (uInt)max_chain; return Z_OK; } @@ -481,33 +644,68 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) * resulting from using fixed blocks instead of stored blocks, which deflate * can emit on compressed data for some combinations of the parameters. * - * This function could be more sophisticated to provide closer upper bounds - * for every combination of windowBits and memLevel, as well as wrap. - * But even the conservative upper bound of about 14% expansion does not - * seem onerous for output buffer allocation. + * This function could be more sophisticated to provide closer upper bounds for + * every combination of windowBits and memLevel. But even the conservative + * upper bound of about 14% expansion does not seem onerous for output buffer + * allocation. */ uLong ZEXPORT deflateBound(strm, sourceLen) z_streamp strm; uLong sourceLen; { deflate_state *s; - uLong destLen; + uLong complen, wraplen; - /* conservative upper bound */ - destLen = sourceLen + - ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 11; + /* conservative upper bound for compressed data */ + complen = sourceLen + + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; - /* if can't get parameters, return conservative bound */ - if (strm == Z_NULL || strm->state == Z_NULL) - return destLen; + /* if can't get parameters, return conservative bound plus zlib wrapper */ + if (deflateStateCheck(strm)) + return complen + 6; + + /* compute wrapper length */ + s = strm->state; + switch (s->wrap) { + case 0: /* raw deflate */ + wraplen = 0; + break; + case 1: /* zlib wrapper */ + wraplen = 6 + (s->strstart ? 4 : 0); + break; +#ifdef GZIP + case 2: /* gzip wrapper */ + wraplen = 18; + if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; + if (s->gzhead->extra != Z_NULL) + wraplen += 2 + s->gzhead->extra_len; + str = s->gzhead->name; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + str = s->gzhead->comment; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + if (s->gzhead->hcrc) + wraplen += 2; + } + break; +#endif + default: /* for compiler happiness */ + wraplen = 6; + } /* if not default parameters, return conservative bound */ - s = strm->state; if (s->w_bits != 15 || s->hash_bits != 8 + 7) - return destLen; + return complen + wraplen; /* default settings: return tight bound for that case */ - return compressBound(sourceLen); + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13 - 6 + wraplen; } /* ========================================================================= @@ -524,30 +722,43 @@ local void putShortMSB (s, b) } /* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also read_buf()). */ local void flush_pending(strm) z_streamp strm; { - unsigned len = strm->state->pending; + unsigned len; + deflate_state *s = strm->state; + _tr_flush_bits(s); + len = s->pending; if (len > strm->avail_out) len = strm->avail_out; if (len == 0) return; - zmemcpy(strm->next_out, strm->state->pending_out, len); + zmemcpy(strm->next_out, s->pending_out, len); strm->next_out += len; - strm->state->pending_out += len; + s->pending_out += len; strm->total_out += len; - strm->avail_out -= len; - strm->state->pending -= len; - if (strm->state->pending == 0) { - strm->state->pending_out = strm->state->pending_buf; + strm->avail_out -= len; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; } } +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; @@ -556,203 +767,21 @@ int ZEXPORT deflate (strm, flush) int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_FINISH || flush < 0) { + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || - (strm->next_in == Z_NULL && strm->avail_in != 0) || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); - s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; - /* Write the header */ - if (s->status == INIT_STATE) { -#ifdef GZIP - if (s->wrap == 2) { - strm->adler = crc32(0L, Z_NULL, 0); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); - if (s->gzhead == NULL) { - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, OS_CODE); - s->status = BUSY_STATE; - } - else { - put_byte(s, (s->gzhead->text ? 1 : 0) + - (s->gzhead->hcrc ? 2 : 0) + - (s->gzhead->extra == Z_NULL ? 0 : 4) + - (s->gzhead->name == Z_NULL ? 0 : 8) + - (s->gzhead->comment == Z_NULL ? 0 : 16) - ); - put_byte(s, (Byte)(s->gzhead->time & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != NULL) { - put_byte(s, s->gzhead->extra_len & 0xff); - put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); - } - if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); - s->gzindex = 0; - s->status = EXTRA_STATE; - } - } - else -#endif - { - uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; - uInt level_flags; - - if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) - level_flags = 0; - else if (s->level < 6) - level_flags = 1; - else if (s->level == 6) - level_flags = 2; - else - level_flags = 3; - header |= (level_flags << 6); - if (s->strstart != 0) header |= PRESET_DICT; - header += 31 - (header % 31); - - s->status = BUSY_STATE; - putShortMSB(s, header); - - /* Save the adler32 of the preset dictionary: */ - if (s->strstart != 0) { - putShortMSB(s, (uInt)(strm->adler >> 16)); - putShortMSB(s, (uInt)(strm->adler & 0xffff)); - } - strm->adler = adler32(0L, Z_NULL, 0); - } - } -#ifdef GZIP - if (s->status == EXTRA_STATE) { - if (s->gzhead->extra != NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - - while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) - break; - } - put_byte(s, s->gzhead->extra[s->gzindex]); - s->gzindex++; - } - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (s->gzindex == s->gzhead->extra_len) { - s->gzindex = 0; - s->status = NAME_STATE; - } - } - else - s->status = NAME_STATE; - } - if (s->status == NAME_STATE) { - if (s->gzhead->name != NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - int val; - - do { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; - } - } - val = s->gzhead->name[s->gzindex++]; - put_byte(s, val); - } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) { - s->gzindex = 0; - s->status = COMMENT_STATE; - } - } - else - s->status = COMMENT_STATE; - } - if (s->status == COMMENT_STATE) { - if (s->gzhead->comment != NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - int val; - - do { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; - } - } - val = s->gzhead->comment[s->gzindex++]; - put_byte(s, val); - } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) - s->status = HCRC_STATE; - } - else - s->status = HCRC_STATE; - } - if (s->status == HCRC_STATE) { - if (s->gzhead->hcrc) { - if (s->pending + 2 > s->pending_buf_size) - flush_pending(strm); - if (s->pending + 2 <= s->pending_buf_size) { - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - strm->adler = crc32(0L, Z_NULL, 0); - s->status = BUSY_STATE; - } - } - else - s->status = BUSY_STATE; - } -#endif - /* Flush as much pending output as possible */ if (s->pending != 0) { flush_pending(strm); @@ -771,7 +800,7 @@ int ZEXPORT deflate (strm, flush) * flushes. For repeated and useless calls with Z_FINISH, we keep * returning Z_STREAM_END instead of Z_BUF_ERROR. */ - } else if (strm->avail_in == 0 && flush <= old_flush && + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && flush != Z_FINISH) { ERR_RETURN(strm, Z_BUF_ERROR); } @@ -781,13 +810,197 @@ int ZEXPORT deflate (strm, flush) ERR_RETURN(strm, Z_BUF_ERROR); } + /* Write the header */ + if (s->status == INIT_STATE) { + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; + else + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + + putShortMSB(s, header); + + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + strm->adler = crc32(0L, Z_NULL, 0); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); + } + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; + } + } + if (s->status == EXTRA_STATE) { + if (s->gzhead->extra != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + left -= copy; + } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = NAME_STATE; + } + if (s->status == NAME_STATE) { + if (s->gzhead->name != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->name[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = COMMENT_STATE; + } + if (s->status == COMMENT_STATE) { + if (s->gzhead->comment != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->comment[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + } + s->status = HCRC_STATE; + } + if (s->status == HCRC_STATE) { + if (s->gzhead->hcrc) { + if (s->pending + 2 > s->pending_buf_size) { + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); + } + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#endif + /* Start a new block or continue the current one. */ if (strm->avail_in != 0 || s->lookahead != 0 || (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = (*(configuration_table[s->level].func))(s, flush); + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -808,13 +1021,18 @@ int ZEXPORT deflate (strm, flush) if (bstate == block_done) { if (flush == Z_PARTIAL_FLUSH) { _tr_align(s); - } else { /* FULL_FLUSH or SYNC_FLUSH */ + } else if (flush != Z_BLOCK) { /* FULL_FLUSH or SYNC_FLUSH */ _tr_stored_block(s, (char*)0, 0L, 0); /* For a full flush, this empty block will be recognized * as a special marker by inflate_sync(). */ if (flush == Z_FULL_FLUSH) { CLEAR_HASH(s); /* forget history */ + if (s->lookahead == 0) { + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } } } flush_pending(strm); @@ -824,7 +1042,6 @@ int ZEXPORT deflate (strm, flush) } } } - Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; @@ -861,18 +1078,9 @@ int ZEXPORT deflateEnd (strm) { int status; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; status = strm->state->status; - if (status != INIT_STATE && - status != EXTRA_STATE && - status != NAME_STATE && - status != COMMENT_STATE && - status != HCRC_STATE && - status != BUSY_STATE && - status != FINISH_STATE) { - return Z_STREAM_ERROR; - } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); @@ -903,18 +1111,18 @@ int ZEXPORT deflateCopy (dest, source) ushf *overlay; - if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { + if (deflateStateCheck(source) || dest == Z_NULL) { return Z_STREAM_ERROR; } ss = source->state; - zmemcpy(dest, source, sizeof(z_stream)); + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; dest->state = (struct internal_state FAR *) ds; - zmemcpy(ds, ss, sizeof(deflate_state)); + zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); ds->strm = dest; ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); @@ -930,8 +1138,8 @@ int ZEXPORT deflateCopy (dest, source) } /* following zmemcpy do not work for 16-bit MSDOS */ zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); - zmemcpy(ds->prev, ss->prev, ds->w_size * sizeof(Pos)); - zmemcpy(ds->head, ss->head, ds->hash_size * sizeof(Pos)); + zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); + zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); @@ -953,7 +1161,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local int read_buf(strm, buf, size) +local unsigned read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -965,19 +1173,19 @@ local int read_buf(strm, buf, size) strm->avail_in -= len; + zmemcpy(buf, strm->next_in, len); if (strm->state->wrap == 1) { - strm->adler = adler32(strm->adler, strm->next_in, len); + strm->adler = adler32(strm->adler, buf, len); } #ifdef GZIP else if (strm->state->wrap == 2) { - strm->adler = crc32(strm->adler, strm->next_in, len); + strm->adler = crc32(strm->adler, buf, len); } #endif - zmemcpy(buf, strm->next_in, len); strm->next_in += len; strm->total_in += len; - return (int)len; + return len; } /* =========================================================================== @@ -1000,6 +1208,7 @@ local void lm_init (s) s->strstart = 0; s->block_start = 0L; s->lookahead = 0; + s->insert = 0; s->match_length = s->prev_length = MIN_MATCH-1; s->match_available = 0; s->ins_h = 0; @@ -1030,9 +1239,9 @@ local uInt longest_match(s, cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ + register Bytef *match; /* matched string */ register int len; /* length of current match */ - int best_len = s->prev_length; /* best match length so far */ + int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1067,7 +1276,7 @@ local uInt longest_match(s, cur_match) /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ - if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); @@ -1167,12 +1376,13 @@ local uInt longest_match(s, cur_match) return s->lookahead; } #endif /* ASMV */ -#endif /* FASTEST */ + +#else /* FASTEST */ /* --------------------------------------------------------------------------- - * Optimized version for level == 1 or strategy == Z_RLE only + * Optimized version for FASTEST only */ -local uInt longest_match_fast(s, cur_match) +local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ { @@ -1225,7 +1435,13 @@ local uInt longest_match_fast(s, cur_match) return (uInt)len <= s->lookahead ? (uInt)len : s->lookahead; } -#ifdef DEBUG +#endif /* FASTEST */ + +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + /* =========================================================================== * Check that the match at match_start is indeed a match. */ @@ -1251,7 +1467,7 @@ local void check_match(s, start, match, length) } #else # define check_match(s, start, match, length) -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. @@ -1266,11 +1482,12 @@ local void check_match(s, start, match, length) local void fill_window(s) deflate_state *s; { - register unsigned n, m; - register Posf *p; + unsigned n; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + do { more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); @@ -1292,39 +1509,14 @@ local void fill_window(s) */ if (s->strstart >= wsize+MAX_DIST(s)) { - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - /* %%% avoid this when Z_RLE */ - n = s->hash_size; - p = &s->head[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - } while (--n); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - /* If n is not on any hash chain, prev[n] is garbage but - * its value will never be used. - */ - } while (--n); -#endif + slide_hash(s); more += wsize; } - if (s->strm->avail_in == 0) return; + if (s->strm->avail_in == 0) break; /* If there was no sliding: * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && @@ -1343,99 +1535,283 @@ local void fill_window(s) s->lookahead += n; /* Initialize the hash value now that we have some input: */ - if (s->lookahead >= MIN_MATCH) { - s->ins_h = s->window[s->strstart]; - UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]); + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); #if MIN_MATCH != 3 Call UPDATE_HASH() MIN_MATCH-3 more times #endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } } /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, * but this is not important since only literal bytes will be emitted. */ } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); } /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. */ -#define FLUSH_BLOCK_ONLY(s, eof) { \ +#define FLUSH_BLOCK_ONLY(s, last) { \ _tr_flush_block(s, (s->block_start >= 0L ? \ (charf *)&s->window[(unsigned)s->block_start] : \ (charf *)Z_NULL), \ (ulg)((long)s->strstart - s->block_start), \ - (eof)); \ + (last)); \ s->block_start = s->strstart; \ flush_pending(s->strm); \ Tracev((stderr,"[FLUSH]")); \ } /* Same but force premature exit if necessary. */ -#define FLUSH_BLOCK(s, eof) { \ - FLUSH_BLOCK_ONLY(s, eof); \ - if (s->strm->avail_out == 0) return (eof) ? finish_started : need_more; \ +#define FLUSH_BLOCK(s, last) { \ + FLUSH_BLOCK_ONLY(s, last); \ + if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. - * This function does not insert new strings in the dictionary since - * uncompressible data is probably not useful. This function is used - * only for the level=0 compression option. - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. */ local block_state deflate_stored(s, flush) deflate_state *s; int flush; { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. */ - ulg max_block_size = 0xffff; - ulg max_start; + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); - if (max_block_size > s->pending_buf_size - 5) { - max_block_size = s->pending_buf_size - 5; - } - - /* Copy as much as possible from input to output: */ - for (;;) { - /* Fill the window as much as possible: */ - if (s->lookahead <= 1) { - - Assert(s->strstart < s->w_size+MAX_DIST(s) || - s->block_start >= (long)s->w_size, "slide too late"); - - fill_window(s); - if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; - - if (s->lookahead == 0) break; /* flush the current block */ - } - Assert(s->block_start >= 0L, "block gone"); - - s->strstart += s->lookahead; - s->lookahead = 0; - - /* Emit a stored block if pending_buf will be full: */ - max_start = s->block_start + max_block_size; - if (s->strstart == 0 || (ulg)s->strstart >= max_start) { - /* strstart == 0 is possible when wraparound on 16-bit machine */ - s->lookahead = (uInt)(s->strstart - max_start); - s->strstart = (uInt)max_start; - FLUSH_BLOCK(s, 0); - } - /* Flush if we may have to slide, otherwise block_start may become - * negative and the data will be gone: + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. */ - if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { - FLUSH_BLOCK(s, 0); + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; + + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif + + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; } + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; + } + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. + */ + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; + } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; + } + s->block_start = s->strstart; + s->insert += MIN(used, s->w_size - s->insert); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) + return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart - 1; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ + } + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); + } + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; } /* =========================================================================== @@ -1449,7 +1825,7 @@ local block_state deflate_fast(s, flush) deflate_state *s; int flush; { - IPos hash_head = NIL; /* head of the hash chain */ + IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ for (;;) { @@ -1469,6 +1845,7 @@ local block_state deflate_fast(s, flush) /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ + hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } @@ -1481,19 +1858,8 @@ local block_state deflate_fast(s, flush) * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ -#ifdef FASTEST - if ((s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) || - (s->strategy == Z_RLE && s->strstart - hash_head == 1)) { - s->match_length = longest_match_fast (s, hash_head); - } -#else - if (s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) { - s->match_length = longest_match (s, hash_head); - } else if (s->strategy == Z_RLE && s->strstart - hash_head == 1) { - s->match_length = longest_match_fast (s, hash_head); - } -#endif - /* longest_match() or longest_match_fast() sets match_start */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ } if (s->match_length >= MIN_MATCH) { check_match(s, s->strstart, s->match_start, s->match_length); @@ -1541,8 +1907,14 @@ local block_state deflate_fast(s, flush) } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } #ifndef FASTEST @@ -1555,7 +1927,7 @@ local block_state deflate_slow(s, flush) deflate_state *s; int flush; { - IPos hash_head = NIL; /* head of hash chain */ + IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ /* Process the input block. */ @@ -1576,6 +1948,7 @@ local block_state deflate_slow(s, flush) /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ + hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } @@ -1591,12 +1964,8 @@ local block_state deflate_slow(s, flush) * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ - if (s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) { - s->match_length = longest_match (s, hash_head); - } else if (s->strategy == Z_RLE && s->strstart - hash_head == 1) { - s->match_length = longest_match_fast (s, hash_head); - } - /* longest_match() or longest_match_fast() sets match_start */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ if (s->match_length <= 5 && (s->strategy == Z_FILTERED #if TOO_FAR <= 32767 @@ -1669,12 +2038,17 @@ local block_state deflate_slow(s, flush) _tr_tally_lit(s, s->window[s->strstart-1], bflush); s->match_available = 0; } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } #endif /* FASTEST */ -#if 0 /* =========================================================================== * For Z_RLE, simply look for runs of bytes, generate matches only of distance * one. Do not maintain a hash table. (It will be regenerated if this run of @@ -1684,43 +2058,52 @@ local block_state deflate_rle(s, flush) deflate_state *s; int flush; { - int bflush; /* set if current block must be flushed */ - uInt run; /* length of run */ - uInt max; /* maximum length of run */ - uInt prev; /* byte at distance one to match */ - Bytef *scan; /* scan for end of run */ + int bflush; /* set if current block must be flushed */ + uInt prev; /* byte at distance one to match */ + Bytef *scan, *strend; /* scan goes up to strend for length of run */ for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes - * for the longest encodable run. + * for the longest run, plus one for the unrolled loop. */ - if (s->lookahead < MAX_MATCH) { + if (s->lookahead <= MAX_MATCH) { fill_window(s); - if (s->lookahead < MAX_MATCH && flush == Z_NO_FLUSH) { + if (s->lookahead <= MAX_MATCH && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ } /* See how many times the previous byte repeats */ - run = 0; - if (s->strstart > 0) { /* if there is a previous byte, that is */ - max = s->lookahead < MAX_MATCH ? s->lookahead : MAX_MATCH; + s->match_length = 0; + if (s->lookahead >= MIN_MATCH && s->strstart > 0) { scan = s->window + s->strstart - 1; - prev = *scan++; - do { - if (*scan++ != prev) - break; - } while (++run < max); + prev = *scan; + if (prev == *++scan && prev == *++scan && prev == *++scan) { + strend = s->window + s->strstart + MAX_MATCH; + do { + } while (prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + scan < strend); + s->match_length = MAX_MATCH - (uInt)(strend - scan); + if (s->match_length > s->lookahead) + s->match_length = s->lookahead; + } + Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan"); } /* Emit match if have run of MIN_MATCH or longer, else emit literal */ - if (run >= MIN_MATCH) { - check_match(s, s->strstart, s->strstart - 1, run); - _tr_tally_dist(s, 1, run - MIN_MATCH, bflush); - s->lookahead -= run; - s->strstart += run; + if (s->match_length >= MIN_MATCH) { + check_match(s, s->strstart, s->strstart - 1, s->match_length); + + _tr_tally_dist(s, 1, s->match_length - MIN_MATCH, bflush); + + s->lookahead -= s->match_length; + s->strstart += s->match_length; + s->match_length = 0; } else { /* No match, output a literal byte */ Tracevv((stderr,"%c", s->window[s->strstart])); @@ -1730,7 +2113,51 @@ local block_state deflate_rle(s, flush) } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; +} + +/* =========================================================================== + * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. + * (It will be regenerated if this run of deflate switches away from Huffman.) + */ +local block_state deflate_huff(s, flush) + deflate_state *s; + int flush; +{ + int bflush; /* set if current block must be flushed */ + + for (;;) { + /* Make sure that we have a literal to write. */ + if (s->lookahead == 0) { + fill_window(s); + if (s->lookahead == 0) { + if (flush == Z_NO_FLUSH) + return need_more; + break; /* flush the current block */ + } + } + + /* Output a literal byte */ + s->match_length = 0; + Tracevv((stderr,"%c", s->window[s->strstart])); + _tr_tally_lit (s, s->window[s->strstart], bflush); + s->lookahead--; + s->strstart++; + if (bflush) FLUSH_BLOCK(s, 0); + } + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } -#endif diff --git a/zlib/deflate.h b/zlib/deflate.h index 05a5ab3a2c1..23ecdd312bc 100644 --- a/zlib/deflate.h +++ b/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2004 Jean-loup Gailly + * Copyright (C) 1995-2016 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -48,13 +48,19 @@ #define MAX_BITS 15 /* All codes must not exceed MAX_BITS bits */ -#define INIT_STATE 42 -#define EXTRA_STATE 69 -#define NAME_STATE 73 -#define COMMENT_STATE 91 -#define HCRC_STATE 103 -#define BUSY_STATE 113 -#define FINISH_STATE 666 +#define Buf_size 16 +/* size of bit buffer in bi_buf */ + +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ /* Stream status */ @@ -80,7 +86,7 @@ typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; @@ -97,11 +103,11 @@ typedef struct internal_state { Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ - uInt pending; /* nb of bytes in the pending buffer */ + ulg pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ - uInt gzindex; /* where in extra, name, or comment */ - Byte method; /* STORED (for zip only) or DEFLATED */ + ulg gzindex; /* where in extra, name, or comment */ + Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ /* used by deflate.c: */ @@ -188,7 +194,7 @@ typedef struct internal_state { int nice_match; /* Stop searching when current match exceeds this */ /* used by trees.c: */ - /* Didn't use ct_data typedef below to supress compiler warning */ + /* Didn't use ct_data typedef below to suppress compiler warning */ struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ @@ -244,9 +250,9 @@ typedef struct internal_state { ulg opt_len; /* bit length of current block with optimal trees */ ulg static_len; /* bit length of current block with static trees */ uInt matches; /* number of string matches in current block */ - int last_eob_len; /* bit length of EOB code for last block */ + uInt insert; /* bytes at end of window left to insert */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif @@ -260,12 +266,19 @@ typedef struct internal_state { * are always zero. */ + ulg high_water; + /* High water mark offset in window for initialized bytes -- bytes above + * this are set to zero in order to avoid memory check warnings when + * longest match routines access bytes past the input. This is then + * updated to the new high water mark. + */ + } FAR deflate_state; /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) @@ -278,14 +291,19 @@ typedef struct internal_state { * distances are limited to MAX_DIST instead of WSIZE. */ +#define WIN_INIT MAX_MATCH +/* Number of bytes after end of data in window to initialize in order to avoid + memory checker errors from longest match routines */ + /* in trees.c */ -void _tr_init OF((deflate_state *s)); -int _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); -void _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); -void _tr_align OF((deflate_state *s)); -void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); +void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); +int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); +void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) @@ -294,15 +312,15 @@ void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, * used. */ -#ifndef DEBUG +#ifndef ZLIB_DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) - extern uch _length_code[]; - extern uch _dist_code[]; + extern uch ZLIB_INTERNAL _length_code[]; + extern uch ZLIB_INTERNAL _dist_code[]; #else - extern const uch _length_code[]; - extern const uch _dist_code[]; + extern const uch ZLIB_INTERNAL _length_code[]; + extern const uch ZLIB_INTERNAL _dist_code[]; #endif # define _tr_tally_lit(s, c, flush) \ @@ -313,8 +331,8 @@ void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, flush = (s->last_lit == s->lit_bufsize-1); \ } # define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (length); \ - ush dist = (distance); \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ s->d_buf[s->last_lit] = dist; \ s->l_buf[s->last_lit++] = len; \ dist--; \ diff --git a/zlib/gzclose.c b/zlib/gzclose.c new file mode 100644 index 00000000000..caeb99a3177 --- /dev/null +++ b/zlib/gzclose.c @@ -0,0 +1,25 @@ +/* gzclose.c -- zlib gzclose() function + * Copyright (C) 2004, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* gzclose() is in a separate file so that it is linked in only if it is used. + That way the other gzclose functions can be used instead to avoid linking in + unneeded compression or decompression routines. */ +int ZEXPORT gzclose(file) + gzFile file; +{ +#ifndef NO_GZCOMPRESS + gz_statep state; + + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + return state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); +#else + return gzclose_r(file); +#endif +} diff --git a/zlib/gzguts.h b/zlib/gzguts.h new file mode 100644 index 00000000000..990a4d25149 --- /dev/null +++ b/zlib/gzguts.h @@ -0,0 +1,218 @@ +/* gzguts.h -- zlib internal header definitions for gz* operations + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifdef _LARGEFILE64_SOURCE +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE 1 +# endif +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif +#endif + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include +#include "zlib.h" +#ifdef STDC +# include +# include +# include +#endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif +#include + +#ifdef _WIN32 +# include +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +# define WIDECHAR +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + +#ifdef NO_DEFLATE /* for compatibility with old definition */ +# define NO_GZCOMPRESS +#endif + +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +/* gz* functions always use library allocation functions */ +#ifndef STDC + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); +#endif + +/* get errno and strerror definition */ +#if defined UNDER_CE +# include +# define zstrerror() gz_strwinerror((DWORD)GetLastError()) +#else +# ifndef NO_STRERROR +# include +# define zstrerror() strerror(errno) +# else +# define zstrerror() "stdio error (consult errno)" +# endif +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); +#endif + +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ +#define GZBUFSIZE 8192 + +/* gzip modes, also provide a little integrity check on the passed structure */ +#define GZ_NONE 0 +#define GZ_READ 7247 +#define GZ_WRITE 31153 +#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ + +/* values for gz_state how */ +#define LOOK 0 /* look for a gzip header */ +#define COPY 1 /* copy input directly */ +#define GZIP 2 /* decompress a gzip stream */ + +/* internal gzip file state data structure */ +typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ + /* used for both reading and writing */ + int mode; /* see gzip modes above */ + int fd; /* file descriptor */ + char *path; /* path or fd for error messages */ + unsigned size; /* buffer size, zero if not allocated yet */ + unsigned want; /* requested buffer size, default is GZBUFSIZE */ + unsigned char *in; /* input buffer (double-sized when writing) */ + unsigned char *out; /* output buffer (double-sized when reading) */ + int direct; /* 0 if processing gzip, 1 if transparent */ + /* just for reading */ + int how; /* 0: get header, 1: copy, 2: decompress */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ + /* just for writing */ + int level; /* compression level */ + int strategy; /* compression strategy */ + /* seek request */ + z_off64_t skip; /* amount to skip (already rewound if backwards) */ + int seek; /* true if seek request pending */ + /* error information */ + int err; /* error code */ + char *msg; /* error message */ + /* zlib inflate or deflate stream */ + z_stream strm; /* stream structure in-place (not a pointer) */ +} gz_state; +typedef gz_state FAR *gz_statep; + +/* shared functions */ +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +#if defined UNDER_CE +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +#endif + +/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t + value -- needed when comparing unsigned to z_off64_t, which is signed + (possible z_off64_t types off_t, off64_t, and long are all signed) */ +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/zlib/gzio.c b/zlib/gzio.c deleted file mode 100644 index ed4e77ca7e9..00000000000 --- a/zlib/gzio.c +++ /dev/null @@ -1,1031 +0,0 @@ -/* gzio.c -- IO on .gz files - * Copyright (C) 1995-2005 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - * - * Compile this file with -DNO_GZCOMPRESS to avoid the compression code. - */ - -/* @(#) $Id$ */ - -/* Need to be included "early" to control other headers */ -#ifdef HAVE_CONFIG_H -#include -#endif - -#include - -#include "zutil.h" - -#ifdef NO_DEFLATE /* for compatibility with old definition */ -# define NO_GZCOMPRESS -#endif - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - -#ifndef Z_BUFSIZE -# ifdef MAXSEG_64K -# define Z_BUFSIZE 4096 /* minimize memory usage for 16-bit DOS */ -# else -# define Z_BUFSIZE 16384 -# endif -#endif -#ifndef Z_PRINTF_BUFSIZE -# define Z_PRINTF_BUFSIZE 4096 -#endif - -#ifdef __MVS__ -# pragma map (fdopen , "\174\174FDOPEN") - FILE *fdopen(int, const char *); -#endif - -#ifndef STDC -extern voidp malloc OF((uInt size)); -extern void free OF((voidpf ptr)); -#endif - -#define ALLOC(size) malloc(size) -#define TRYFREE(p) {if (p) free(p);} - -static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ - -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -typedef struct gz_stream { - z_stream stream; - int z_err; /* error code for last stream operation */ - int z_eof; /* set if end of input file */ - FILE *file; /* .gz file */ - Byte *inbuf; /* input buffer */ - Byte *outbuf; /* output buffer */ - uLong crc; /* crc32 of uncompressed data */ - char *msg; /* error message */ - char *path; /* path name for debugging only */ - int transparent; /* 1 if input file is not a .gz file */ - char mode; /* 'w' or 'r' */ - z_off_t start; /* start of compressed data in file (header skipped) */ - z_off_t in; /* bytes into deflate or inflate */ - z_off_t out; /* bytes out of deflate or inflate */ - int back; /* one character push-back */ - int last; /* true if push-back is last character */ -} gz_stream; - - -local gzFile gz_open OF((const char *path, const char *mode, int fd)); -local int do_flush OF((gzFile file, int flush)); -local int get_byte OF((gz_stream *s)); -local void check_header OF((gz_stream *s)); -local int destroy OF((gz_stream *s)); -local void putLong OF((FILE *file, uLong x)); -local uLong getLong OF((gz_stream *s)); - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb"). The file is given either by file descriptor - or path name (if fd == -1). - gz_open returns NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). -*/ -local gzFile gz_open (path, mode, fd) - const char *path; - const char *mode; - int fd; -{ - int err; - int level = Z_DEFAULT_COMPRESSION; /* compression level */ - int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - char *p = (char*)mode; - gz_stream *s; - char fmode[80]; /* copy of mode, without the compression level */ - char *m = fmode; - - if (!path || !mode) return Z_NULL; - - s = (gz_stream *)ALLOC(sizeof(gz_stream)); - if (!s) return Z_NULL; - - s->stream.zalloc = (alloc_func)0; - s->stream.zfree = (free_func)0; - s->stream.opaque = (voidpf)0; - s->stream.next_in = s->inbuf = Z_NULL; - s->stream.next_out = s->outbuf = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; - s->file = NULL; - s->z_err = Z_OK; - s->z_eof = 0; - s->in = 0; - s->out = 0; - s->back = EOF; - s->crc = crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->transparent = 0; - - s->path = (char*)ALLOC(strlen(path)+1); - if (s->path == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - strcpy(s->path, path); /* do this early for debugging */ - - s->mode = '\0'; - do { - if (*p == 'r') s->mode = 'r'; - if (*p == 'w' || *p == 'a') s->mode = 'w'; - if (*p >= '0' && *p <= '9') { - level = *p - '0'; - } else if (*p == 'f') { - strategy = Z_FILTERED; - } else if (*p == 'h') { - strategy = Z_HUFFMAN_ONLY; - } else if (*p == 'R') { - strategy = Z_RLE; - } else { - *m++ = *p; /* copy the mode */ - } - } while (*p++ && m != fmode + sizeof(fmode)); - if (s->mode == '\0') return destroy(s), (gzFile)Z_NULL; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateInit2(&(s->stream), level, - Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, strategy); - /* windowBits is passed < 0 to suppress zlib header */ - - s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); -#endif - if (err != Z_OK || s->outbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } else { - s->stream.next_in = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); - - err = inflateInit2(&(s->stream), -MAX_WBITS); - /* windowBits is passed < 0 to tell that there is no zlib header. - * Note that in this case inflate *requires* an extra "dummy" byte - * after the compressed stream in order to complete decompression and - * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are - * present after the compressed stream. - */ - if (err != Z_OK || s->inbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } - s->stream.avail_out = Z_BUFSIZE; - - errno = 0; - s->file = fd < 0 ? F_OPEN(path, fmode) : (FILE*)fdopen(fd, fmode); - - if (s->file == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - if (s->mode == 'w') { - /* Write a very simple .gz header: - */ - fprintf(s->file, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1], - Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/, OS_CODE); - s->start = 10L; - /* We use 10L instead of ftell(s->file) to because ftell causes an - * fflush on some systems. This version of the library doesn't use - * start anyway in write mode, so this initialization is not - * necessary. - */ - } else { - check_header(s); /* skip the .gz header */ - s->start = ftell(s->file) - s->stream.avail_in; - } - - return (gzFile)s; -} - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. -*/ -gzFile ZEXPORT gzopen (path, mode) - const char *path; - const char *mode; -{ - return gz_open (path, mode, -1); -} - -/* =========================================================================== - Associate a gzFile with the file descriptor fd. fd is not dup'ed here - to mimic the behavio(u)r of fdopen. -*/ -gzFile ZEXPORT gzdopen (fd, mode) - int fd; - const char *mode; -{ - char name[46]; /* allow for up to 128-bit integers */ - - if (fd < 0) return (gzFile)Z_NULL; - sprintf(name, "", fd); /* for debugging */ - - return gz_open (name, mode, fd); -} - -/* =========================================================================== - * Update the compression level and strategy - */ -int ZEXPORT gzsetparams (file, level, strategy) - gzFile file; - int level; - int strategy; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - /* Make room to allow flushing */ - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - } - s->stream.avail_out = Z_BUFSIZE; - } - - return deflateParams (&(s->stream), level, strategy); -} - -/* =========================================================================== - Read a byte from a gz_stream; update next_in and avail_in. Return EOF - for end of file. - IN assertion: the stream s has been sucessfully opened for reading. -*/ -local int get_byte(s) - gz_stream *s; -{ - if (s->z_eof) return EOF; - if (s->stream.avail_in == 0) { - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) s->z_err = Z_ERRNO; - return EOF; - } - s->stream.next_in = s->inbuf; - } - s->stream.avail_in--; - return *(s->stream.next_in)++; -} - -/* =========================================================================== - Check the gzip header of a gz_stream opened for reading. Set the stream - mode to transparent if the gzip magic header is not present; set s->err - to Z_DATA_ERROR if the magic header is present but the rest of the header - is incorrect. - IN assertion: the stream s has already been created sucessfully; - s->stream.avail_in is zero for the first time, but may be non-zero - for concatenated .gz files. -*/ -local void check_header(s) - gz_stream *s; -{ - int method; /* method byte */ - int flags; /* flags byte */ - uInt len; - int c; - - /* Assure two bytes in the buffer so we can peek ahead -- handle case - where first byte of header is at the end of the buffer after the last - gzip segment */ - len = s->stream.avail_in; - if (len < 2) { - if (len) s->inbuf[0] = s->stream.next_in[0]; - errno = 0; - len = (uInt)fread(s->inbuf + len, 1, Z_BUFSIZE >> len, s->file); - if (len == 0 && ferror(s->file)) s->z_err = Z_ERRNO; - s->stream.avail_in += len; - s->stream.next_in = s->inbuf; - if (s->stream.avail_in < 2) { - s->transparent = s->stream.avail_in; - return; - } - } - - /* Peek ahead to check the gzip magic header */ - if (s->stream.next_in[0] != gz_magic[0] || - s->stream.next_in[1] != gz_magic[1]) { - s->transparent = 1; - return; - } - s->stream.avail_in -= 2; - s->stream.next_in += 2; - - /* Check the rest of the gzip header */ - method = get_byte(s); - flags = get_byte(s); - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - s->z_err = Z_DATA_ERROR; - return; - } - - /* Discard time, xflags and OS code: */ - for (len = 0; len < 6; len++) (void)get_byte(s); - - if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ - len = (uInt)get_byte(s); - len += ((uInt)get_byte(s))<<8; - /* len is garbage if EOF but the loop below will quit anyway */ - while (len-- != 0 && get_byte(s) != EOF) ; - } - if ((flags & ORIG_NAME) != 0) { /* skip the original file name */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & COMMENT) != 0) { /* skip the .gz file comment */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ - for (len = 0; len < 2; len++) (void)get_byte(s); - } - s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK; -} - - /* =========================================================================== - * Cleanup then free the given gz_stream. Return a zlib error code. - Try freeing in the reverse order of allocations. - */ -local int destroy (s) - gz_stream *s; -{ - int err = Z_OK; - - if (!s) return Z_STREAM_ERROR; - - TRYFREE(s->msg); - - if (s->stream.state != NULL) { - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateEnd(&(s->stream)); -#endif - } else if (s->mode == 'r') { - err = inflateEnd(&(s->stream)); - } - } - if (s->file != NULL && fclose(s->file)) { -#ifdef ESPIPE - if (errno != ESPIPE) /* fclose is broken for pipes in HP/UX */ -#endif - err = Z_ERRNO; - } - if (s->z_err < 0) err = s->z_err; - - TRYFREE(s->inbuf); - TRYFREE(s->outbuf); - TRYFREE(s->path); - TRYFREE(s); - return err; -} - -/* =========================================================================== - Reads the given number of uncompressed bytes from the compressed file. - gzread returns the number of bytes actually read (0 for end of file). -*/ -int ZEXPORT gzread (file, buf, len) - gzFile file; - voidp buf; - unsigned len; -{ - gz_stream *s = (gz_stream*)file; - Bytef *start = (Bytef*)buf; /* starting point for crc computation */ - Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */ - - if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR; - - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1; - if (s->z_err == Z_STREAM_END) return 0; /* EOF */ - - next_out = (Byte*)buf; - s->stream.next_out = (Bytef*)buf; - s->stream.avail_out = len; - - if (s->stream.avail_out && s->back != EOF) { - *next_out++ = s->back; - s->stream.next_out++; - s->stream.avail_out--; - s->back = EOF; - s->out++; - start++; - if (s->last) { - s->z_err = Z_STREAM_END; - return 1; - } - } - - while (s->stream.avail_out != 0) { - - if (s->transparent) { - /* Copy first the lookahead bytes: */ - uInt n = s->stream.avail_in; - if (n > s->stream.avail_out) n = s->stream.avail_out; - if (n > 0) { - zmemcpy(s->stream.next_out, s->stream.next_in, n); - next_out += n; - s->stream.next_out = next_out; - s->stream.next_in += n; - s->stream.avail_out -= n; - s->stream.avail_in -= n; - } - if (s->stream.avail_out > 0) { - s->stream.avail_out -= - (uInt)fread(next_out, 1, s->stream.avail_out, s->file); - } - len -= s->stream.avail_out; - s->in += len; - s->out += len; - if (len == 0) s->z_eof = 1; - return (int)len; - } - if (s->stream.avail_in == 0 && !s->z_eof) { - - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) { - s->z_err = Z_ERRNO; - break; - } - } - s->stream.next_in = s->inbuf; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = inflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - - if (s->z_err == Z_STREAM_END) { - /* Check CRC and original size */ - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - start = s->stream.next_out; - - if (getLong(s) != s->crc) { - s->z_err = Z_DATA_ERROR; - } else { - (void)getLong(s); - /* The uncompressed length returned by above getlong() may be - * different from s->out in case of concatenated .gz files. - * Check for such files: - */ - check_header(s); - if (s->z_err == Z_OK) { - inflateReset(&(s->stream)); - s->crc = crc32(0L, Z_NULL, 0); - } - } - } - if (s->z_err != Z_OK || s->z_eof) break; - } - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - - if (len == s->stream.avail_out && - (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)) - return -1; - return (int)(len - s->stream.avail_out); -} - - -/* =========================================================================== - Reads one byte from the compressed file. gzgetc returns this byte - or -1 in case of end of file or error. -*/ -int ZEXPORT gzgetc(file) - gzFile file; -{ - unsigned char c; - - return gzread(file, &c, 1) == 1 ? c : -1; -} - - -/* =========================================================================== - Push one byte back onto the stream. -*/ -int ZEXPORT gzungetc(c, file) - int c; - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r' || c == EOF || s->back != EOF) return EOF; - s->back = c; - s->out--; - s->last = (s->z_err == Z_STREAM_END); - if (s->last) s->z_err = Z_OK; - s->z_eof = 0; - return c; -} - - -/* =========================================================================== - Reads bytes from the compressed file until len-1 characters are - read, or a newline character is read and transferred to buf, or an - end-of-file condition is encountered. The string is then terminated - with a null character. - gzgets returns buf, or Z_NULL in case of error. - - The current implementation is not optimized at all. -*/ -char * ZEXPORT gzgets(file, buf, len) - gzFile file; - char *buf; - int len; -{ - char *b = buf; - if (buf == Z_NULL || len <= 0) return Z_NULL; - - while (--len > 0 && gzread(file, buf, 1) == 1 && *buf++ != '\n') ; - *buf = '\0'; - return b == buf && len > 0 ? Z_NULL : b; -} - - -#ifndef NO_GZCOMPRESS -/* =========================================================================== - Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of bytes actually written (0 in case of error). -*/ -int ZEXPORT gzwrite (file, buf, len) - gzFile file; - voidpc buf; - unsigned len; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.next_in = (Bytef*)buf; - s->stream.avail_in = len; - - while (s->stream.avail_in != 0) { - - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - break; - } - s->stream.avail_out = Z_BUFSIZE; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - if (s->z_err != Z_OK) break; - } - s->crc = crc32(s->crc, (const Bytef *)buf, len); - - return (int)(len - s->stream.avail_in); -} - - -/* =========================================================================== - Converts, formats, and writes the args to the compressed file under - control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written (0 in case of error). -*/ -#ifdef STDC -#include - -int ZEXPORTVA gzprintf (gzFile file, const char *format, /* args */ ...) -{ - char buf[Z_PRINTF_BUFSIZE]; - va_list va; - int len; - - buf[sizeof(buf) - 1] = 0; - va_start(va, format); -#ifdef NO_vsnprintf -# ifdef HAS_vsprintf_void - (void)vsprintf(buf, format, va); - va_end(va); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = vsprintf(buf, format, va); - va_end(va); -# endif -#else -# ifdef HAS_vsnprintf_void - (void)vsnprintf(buf, sizeof(buf), format, va); - va_end(va); - len = strlen(buf); -# else - len = vsnprintf(buf, sizeof(buf), format, va); - va_end(va); -# endif -#endif - if (len <= 0 || len >= (int)sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, (unsigned)len); -} -#else /* not ANSI C */ - -int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) - gzFile file; - const char *format; - int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; -{ - char buf[Z_PRINTF_BUFSIZE]; - int len; - - buf[sizeof(buf) - 1] = 0; -#ifdef NO_snprintf -# ifdef HAS_sprintf_void - sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#else -# ifdef HAS_snprintf_void - snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen(buf); -# else - len = snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#endif - if (len <= 0 || len >= sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, len); -} -#endif - -/* =========================================================================== - Writes c, converted to an unsigned char, into the compressed file. - gzputc returns the value that was written, or -1 in case of error. -*/ -int ZEXPORT gzputc(file, c) - gzFile file; - int c; -{ - unsigned char cc = (unsigned char) c; /* required for big endian systems */ - - return gzwrite(file, &cc, 1) == 1 ? (int)cc : -1; -} - - -/* =========================================================================== - Writes the given null-terminated string to the compressed file, excluding - the terminating null character. - gzputs returns the number of characters written, or -1 in case of error. -*/ -int ZEXPORT gzputs(file, s) - gzFile file; - const char *s; -{ - return gzwrite(file, (char*)s, (unsigned)strlen(s)); -} - - -/* =========================================================================== - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. -*/ -local int do_flush (file, flush) - gzFile file; - int flush; -{ - uInt len; - int done = 0; - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.avail_in = 0; /* should be zero already anyway */ - - for (;;) { - len = Z_BUFSIZE - s->stream.avail_out; - - if (len != 0) { - if ((uInt)fwrite(s->outbuf, 1, len, s->file) != len) { - s->z_err = Z_ERRNO; - return Z_ERRNO; - } - s->stream.next_out = s->outbuf; - s->stream.avail_out = Z_BUFSIZE; - } - if (done) break; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), flush); - s->out -= s->stream.avail_out; - - /* Ignore the second of two consecutive flushes: */ - if (len == 0 && s->z_err == Z_BUF_ERROR) s->z_err = Z_OK; - - /* deflate has finished flushing only when it hasn't used up - * all the available space in the output buffer: - */ - done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END); - - if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break; - } - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} - -int ZEXPORT gzflush (file, flush) - gzFile file; - int flush; -{ - gz_stream *s = (gz_stream*)file; - int err = do_flush (file, flush); - - if (err) return err; - fflush(s->file); - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} -#endif /* NO_GZCOMPRESS */ - -/* =========================================================================== - Sets the starting position for the next gzread or gzwrite on the given - compressed file. The offset represents a number of bytes in the - gzseek returns the resulting offset location as measured in bytes from - the beginning of the uncompressed stream, or -1 in case of error. - SEEK_END is not implemented, returns error. - In this version of the library, gzseek can be extremely slow. -*/ -z_off_t ZEXPORT gzseek (file, offset, whence) - gzFile file; - z_off_t offset; - int whence; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || whence == SEEK_END || - s->z_err == Z_ERRNO || s->z_err == Z_DATA_ERROR) { - return -1L; - } - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return -1L; -#else - if (whence == SEEK_SET) { - offset -= s->in; - } - if (offset < 0) return -1L; - - /* At this point, offset is the number of zero bytes to write. */ - if (s->inbuf == Z_NULL) { - s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); /* for seeking */ - if (s->inbuf == Z_NULL) return -1L; - zmemzero(s->inbuf, Z_BUFSIZE); - } - while (offset > 0) { - uInt size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (uInt)offset; - - size = gzwrite(file, s->inbuf, size); - if (size == 0) return -1L; - - offset -= size; - } - return s->in; -#endif - } - /* Rest of function is for reading only */ - - /* compute absolute position */ - if (whence == SEEK_CUR) { - offset += s->out; - } - if (offset < 0) return -1L; - - if (s->transparent) { - /* map to fseek */ - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - if (fseek(s->file, offset, SEEK_SET) < 0) return -1L; - - s->in = s->out = offset; - return offset; - } - - /* For a negative seek, rewind and use positive seek */ - if (offset >= s->out) { - offset -= s->out; - } else if (gzrewind(file) < 0) { - return -1L; - } - /* offset is now the number of bytes to skip. */ - - if (offset != 0 && s->outbuf == Z_NULL) { - s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); - if (s->outbuf == Z_NULL) return -1L; - } - if (offset && s->back != EOF) { - s->back = EOF; - s->out++; - offset--; - if (s->last) s->z_err = Z_STREAM_END; - } - while (offset > 0) { - int size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (int)offset; - - size = gzread(file, s->outbuf, (uInt)size); - if (size <= 0) return -1L; - offset -= size; - } - return s->out; -} - -/* =========================================================================== - Rewinds input file. -*/ -int ZEXPORT gzrewind (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return -1; - - s->z_err = Z_OK; - s->z_eof = 0; - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - s->crc = crc32(0L, Z_NULL, 0); - if (!s->transparent) (void)inflateReset(&s->stream); - s->in = 0; - s->out = 0; - return fseek(s->file, s->start, SEEK_SET); -} - -/* =========================================================================== - Returns the starting position for the next gzread or gzwrite on the - given compressed file. This position represents a number of bytes in the - uncompressed data stream. -*/ -z_off_t ZEXPORT gztell (file) - gzFile file; -{ - return gzseek(file, 0L, SEEK_CUR); -} - -/* =========================================================================== - Returns 1 when EOF has previously been detected reading the given - input stream, otherwise zero. -*/ -int ZEXPORT gzeof (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - /* With concatenated compressed files that can have embedded - * crc trailers, z_eof is no longer the only/best indicator of EOF - * on a gz_stream. Handle end-of-stream error explicitly here. - */ - if (s == NULL || s->mode != 'r') return 0; - if (s->z_eof) return 1; - return s->z_err == Z_STREAM_END; -} - -/* =========================================================================== - Returns 1 if reading and doing so transparently, otherwise zero. -*/ -int ZEXPORT gzdirect (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return 0; - return s->transparent; -} - -/* =========================================================================== - Outputs a long in LSB order to the given file -*/ -local void putLong (file, x) - FILE *file; - uLong x; -{ - int n; - for (n = 0; n < 4; n++) { - fputc((int)(x & 0xff), file); - x >>= 8; - } -} - -/* =========================================================================== - Reads a long in LSB order from the given gz_stream. Sets z_err in case - of error. -*/ -local uLong getLong (s) - gz_stream *s; -{ - uLong x = (uLong)get_byte(s); - int c; - - x += ((uLong)get_byte(s))<<8; - x += ((uLong)get_byte(s))<<16; - c = get_byte(s); - if (c == EOF) s->z_err = Z_DATA_ERROR; - x += ((uLong)c)<<24; - return x; -} - -/* =========================================================================== - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. -*/ -int ZEXPORT gzclose (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return Z_STREAM_ERROR; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return Z_STREAM_ERROR; -#else - if (do_flush (file, Z_FINISH) != Z_OK) - return destroy((gz_stream*)file); - - putLong (s->file, s->crc); - putLong (s->file, (uLong)(s->in & 0xffffffff)); -#endif - } - return destroy((gz_stream*)file); -} - -#ifdef STDC -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - -/* =========================================================================== - Returns the error message for the last error which occurred on the - given compressed file. errnum is set to zlib error number. If an - error occurred in the file system and not in the compression library, - errnum is set to Z_ERRNO and the application may consult errno - to get the exact error code. -*/ -const char * ZEXPORT gzerror (file, errnum) - gzFile file; - int *errnum; -{ - char *m; - gz_stream *s = (gz_stream*)file; - - if (s == NULL) { - *errnum = Z_STREAM_ERROR; - return (const char*)ERR_MSG(Z_STREAM_ERROR); - } - *errnum = s->z_err; - if (*errnum == Z_OK) return (const char*)""; - - m = (char*)(*errnum == Z_ERRNO ? zstrerror(errno) : s->stream.msg); - - if (m == NULL || *m == '\0') m = (char*)ERR_MSG(s->z_err); - - TRYFREE(s->msg); - s->msg = (char*)ALLOC(strlen(s->path) + strlen(m) + 3); - if (s->msg == Z_NULL) return (const char*)ERR_MSG(Z_MEM_ERROR); - strcpy(s->msg, s->path); - strcat(s->msg, ": "); - strcat(s->msg, m); - return (const char*)s->msg; -} - -/* =========================================================================== - Clear the error and end-of-file flags, and do the same for the real file. -*/ -void ZEXPORT gzclearerr (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return; - if (s->z_err != Z_STREAM_END) s->z_err = Z_OK; - s->z_eof = 0; - clearerr(s->file); -} diff --git a/zlib/gzlib.c b/zlib/gzlib.c new file mode 100644 index 00000000000..4105e6aff92 --- /dev/null +++ b/zlib/gzlib.c @@ -0,0 +1,637 @@ +/* gzlib.c -- zlib functions common to reading and writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) +# define LSEEK _lseeki64 +#else +#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +# define LSEEK lseek64 +#else +# define LSEEK lseek +#endif +#endif + +/* Local functions */ +local void gz_reset OF((gz_statep)); +local gzFile gz_open OF((const void *, int, const char *)); + +#if defined UNDER_CE + +/* Map the Windows error number in ERROR to a locale-dependent error message + string and return a pointer to it. Typically, the values for ERROR come + from GetLastError. + + The string pointed to shall not be modified by the application, but may be + overwritten by a subsequent call to gz_strwinerror + + The gz_strwinerror function does not change the current setting of + GetLastError. */ +char ZLIB_INTERNAL *gz_strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +#endif /* UNDER_CE */ + +/* Reset gzip file state */ +local void gz_reset(state) + gz_statep state; +{ + state->x.have = 0; /* no output data available */ + if (state->mode == GZ_READ) { /* for reading ... */ + state->eof = 0; /* not at end of file */ + state->past = 0; /* have not read past end yet */ + state->how = LOOK; /* look for gzip header */ + } + state->seek = 0; /* no seek request pending */ + gz_error(state, Z_OK, NULL); /* clear error */ + state->x.pos = 0; /* no uncompressed data yet */ + state->strm.avail_in = 0; /* no input data yet */ +} + +/* Open a gzip file either by name or file descriptor. */ +local gzFile gz_open(path, fd, mode) + const void *path; + int fd; + const char *mode; +{ + gz_statep state; + z_size_t len; + int oflag; +#ifdef O_CLOEXEC + int cloexec = 0; +#endif +#ifdef O_EXCL + int exclusive = 0; +#endif + + /* check input */ + if (path == NULL) + return NULL; + + /* allocate gzFile structure to return */ + state = (gz_statep)malloc(sizeof(gz_state)); + if (state == NULL) + return NULL; + state->size = 0; /* no buffers allocated yet */ + state->want = GZBUFSIZE; /* requested buffer size */ + state->msg = NULL; /* no error message yet */ + + /* interpret mode */ + state->mode = GZ_NONE; + state->level = Z_DEFAULT_COMPRESSION; + state->strategy = Z_DEFAULT_STRATEGY; + state->direct = 0; + while (*mode) { + if (*mode >= '0' && *mode <= '9') + state->level = *mode - '0'; + else + switch (*mode) { + case 'r': + state->mode = GZ_READ; + break; +#ifndef NO_GZCOMPRESS + case 'w': + state->mode = GZ_WRITE; + break; + case 'a': + state->mode = GZ_APPEND; + break; +#endif + case '+': /* can't read and write at the same time */ + free(state); + return NULL; + case 'b': /* ignore -- will request binary anyway */ + break; +#ifdef O_CLOEXEC + case 'e': + cloexec = 1; + break; +#endif +#ifdef O_EXCL + case 'x': + exclusive = 1; + break; +#endif + case 'f': + state->strategy = Z_FILTERED; + break; + case 'h': + state->strategy = Z_HUFFMAN_ONLY; + break; + case 'R': + state->strategy = Z_RLE; + break; + case 'F': + state->strategy = Z_FIXED; + break; + case 'T': + state->direct = 1; + break; + default: /* could consider as an error, but just ignore */ + ; + } + mode++; + } + + /* must provide an "r", "w", or "a" */ + if (state->mode == GZ_NONE) { + free(state); + return NULL; + } + + /* can't force transparent read */ + if (state->mode == GZ_READ) { + if (state->direct) { + free(state); + return NULL; + } + state->direct = 1; /* for empty file */ + } + + /* save the path name for error messages */ +#ifdef WIDECHAR + if (fd == -2) { + len = wcstombs(NULL, path, 0); + if (len == (z_size_t)-1) + len = 0; + } + else +#endif + len = strlen((const char *)path); + state->path = (char *)malloc(len + 1); + if (state->path == NULL) { + free(state); + return NULL; + } +#ifdef WIDECHAR + if (fd == -2) + if (len) + wcstombs(state->path, path, len + 1); + else + *(state->path) = 0; + else +#endif +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->path, len + 1, "%s", (const char *)path); +#else + strcpy(state->path, path); +#endif + + /* compute the flags for open() */ + oflag = +#ifdef O_LARGEFILE + O_LARGEFILE | +#endif +#ifdef O_BINARY + O_BINARY | +#endif +#ifdef O_CLOEXEC + (cloexec ? O_CLOEXEC : 0) | +#endif + (state->mode == GZ_READ ? + O_RDONLY : + (O_WRONLY | O_CREAT | +#ifdef O_EXCL + (exclusive ? O_EXCL : 0) | +#endif + (state->mode == GZ_WRITE ? + O_TRUNC : + O_APPEND))); + + /* open the file with the appropriate flags (or just use fd) */ + state->fd = fd > -1 ? fd : ( +#ifdef WIDECHAR + fd == -2 ? _wopen(path, oflag, 0666) : +#endif + open((const char *)path, oflag, 0666)); + if (state->fd == -1) { + free(state->path); + free(state); + return NULL; + } + if (state->mode == GZ_APPEND) { + LSEEK(state->fd, 0, SEEK_END); /* so gzoffset() is correct */ + state->mode = GZ_WRITE; /* simplify later checks */ + } + + /* save the current position for rewinding (only if reading) */ + if (state->mode == GZ_READ) { + state->start = LSEEK(state->fd, 0, SEEK_CUR); + if (state->start == -1) state->start = 0; + } + + /* initialize stream */ + gz_reset(state); + + /* return stream */ + return (gzFile)state; +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen64(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzdopen(fd, mode) + int fd; + const char *mode; +{ + char *path; /* identifier for error messages */ + gzFile gz; + + if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) + return NULL; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(path, 7 + 3 * sizeof(int), "", fd); +#else + sprintf(path, "", fd); /* for debugging */ +#endif + gz = gz_open(path, fd, mode); + free(path); + return gz; +} + +/* -- see zlib.h -- */ +#ifdef WIDECHAR +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ + return gz_open(path, -2, mode); +} +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzbuffer(file, size) + gzFile file; + unsigned size; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* make sure we haven't already allocated memory */ + if (state->size != 0) + return -1; + + /* check and set requested size */ + if ((size << 1) < size) + return -1; /* need to be able to double it */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ + state->want = size; + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzrewind(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* back up and start over */ + if (LSEEK(state->fd, state->start, SEEK_SET) == -1) + return -1; + gz_reset(state); + return 0; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzseek64(file, offset, whence) + gzFile file; + z_off64_t offset; + int whence; +{ + unsigned n; + z_off64_t ret; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* check that there's no error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* can only seek from start or relative to current position */ + if (whence != SEEK_SET && whence != SEEK_CUR) + return -1; + + /* normalize offset to a SEEK_CUR specification */ + if (whence == SEEK_SET) + offset -= state->x.pos; + else if (state->seek) + offset += state->skip; + state->seek = 0; + + /* if within raw area while reading, just go there */ + if (state->mode == GZ_READ && state->how == COPY && + state->x.pos + offset >= 0) { + ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); + if (ret == -1) + return -1; + state->x.have = 0; + state->eof = 0; + state->past = 0; + state->seek = 0; + gz_error(state, Z_OK, NULL); + state->strm.avail_in = 0; + state->x.pos += offset; + return state->x.pos; + } + + /* calculate skip amount, rewinding if needed for back seek when reading */ + if (offset < 0) { + if (state->mode != GZ_READ) /* writing -- can't go backwards */ + return -1; + offset += state->x.pos; + if (offset < 0) /* before start of file! */ + return -1; + if (gzrewind(file) == -1) /* rewind, then skip to offset */ + return -1; + } + + /* if reading, skip what's in output buffer (one less gzgetc() check) */ + if (state->mode == GZ_READ) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ? + (unsigned)offset : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + offset -= n; + } + + /* request skip (if not zero) */ + if (offset) { + state->seek = 1; + state->skip = offset; + } + return state->x.pos + offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzseek(file, offset, whence) + gzFile file; + z_off_t offset; + int whence; +{ + z_off64_t ret; + + ret = gzseek64(file, (z_off64_t)offset, whence); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gztell64(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* return position */ + return state->x.pos + (state->seek ? state->skip : 0); +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gztell(file) + gzFile file; +{ + z_off64_t ret; + + ret = gztell64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzoffset64(file) + gzFile file; +{ + z_off64_t offset; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* compute and return effective offset in file */ + offset = LSEEK(state->fd, 0, SEEK_CUR); + if (offset == -1) + return -1; + if (state->mode == GZ_READ) /* reading */ + offset -= state->strm.avail_in; /* don't count buffered input */ + return offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzoffset(file) + gzFile file; +{ + z_off64_t ret; + + ret = gzoffset64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzeof(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return 0; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return 0; + + /* return end-of-file state */ + return state->mode == GZ_READ ? state->past : 0; +} + +/* -- see zlib.h -- */ +const char * ZEXPORT gzerror(file, errnum) + gzFile file; + int *errnum; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return NULL; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return NULL; + + /* return error information */ + if (errnum != NULL) + *errnum = state->err; + return state->err == Z_MEM_ERROR ? "out of memory" : + (state->msg == NULL ? "" : state->msg); +} + +/* -- see zlib.h -- */ +void ZEXPORT gzclearerr(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return; + + /* clear error and end-of-file */ + if (state->mode == GZ_READ) { + state->eof = 0; + state->past = 0; + } + gz_error(state, Z_OK, NULL); +} + +/* Create an error message in allocated memory and set state->err and + state->msg accordingly. Free any previous error message already there. Do + not try to free or allocate space if the error is Z_MEM_ERROR (out of + memory). Simply save the error message as a static string. If there is an + allocation failure constructing the error message, then convert the error to + out of memory. */ +void ZLIB_INTERNAL gz_error(state, err, msg) + gz_statep state; + int err; + const char *msg; +{ + /* free previously allocated message and clear */ + if (state->msg != NULL) { + if (state->err != Z_MEM_ERROR) + free(state->msg); + state->msg = NULL; + } + + /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ + if (err != Z_OK && err != Z_BUF_ERROR) + state->x.have = 0; + + /* set error code, and if no message, then done */ + state->err = err; + if (msg == NULL) + return; + + /* for an out of memory error, return literal string when requested */ + if (err == Z_MEM_ERROR) + return; + + /* construct error message with path */ + if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) == + NULL) { + state->err = Z_MEM_ERROR; + return; + } +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, + "%s%s%s", state->path, ": ", msg); +#else + strcpy(state->msg, state->path); + strcat(state->msg, ": "); + strcat(state->msg, msg); +#endif +} + +#ifndef INT_MAX +/* portably return maximum value for an int (when limits.h presumed not + available) -- we need to do this to cover cases where 2's complement not + used, since C standard permits 1's complement and sign-bit representations, + otherwise we could just use ((unsigned)-1) >> 1 */ +unsigned ZLIB_INTERNAL gz_intmax() +{ + unsigned p, q; + + p = 1; + do { + q = p; + p <<= 1; + p++; + } while (p > q); + return q >> 1; +} +#endif diff --git a/zlib/gzread.c b/zlib/gzread.c new file mode 100644 index 00000000000..956b91ea7d9 --- /dev/null +++ b/zlib/gzread.c @@ -0,0 +1,654 @@ +/* gzread.c -- zlib functions for reading gzip files + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); +local int gz_avail OF((gz_statep)); +local int gz_look OF((gz_statep)); +local int gz_decomp OF((gz_statep)); +local int gz_fetch OF((gz_statep)); +local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); + +/* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from + state->fd, and update state->eof, state->err, and state->msg as appropriate. + This function needs to loop on read(), since read() is not guaranteed to + read the number of bytes requested, depending on the type of descriptor. */ +local int gz_load(state, buf, len, have) + gz_statep state; + unsigned char *buf; + unsigned len; + unsigned *have; +{ + int ret; + unsigned get, max = ((unsigned)-1 >> 2) + 1; + + *have = 0; + do { + get = len - *have; + if (get > max) + get = max; + ret = read(state->fd, buf + *have, get); + if (ret <= 0) + break; + *have += (unsigned)ret; + } while (*have < len); + if (ret < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + if (ret == 0) + state->eof = 1; + return 0; +} + +/* Load up input buffer and set eof flag if last data loaded -- return -1 on + error, 0 otherwise. Note that the eof flag is set when the end of the input + file is reached, even though there may be unused data in the buffer. Once + that data has been used, no more attempts will be made to read the file. + If strm->avail_in != 0, then the current data is moved to the beginning of + the input buffer, and then the remainder of the buffer is loaded with the + available data from the input file. */ +local int gz_avail(state) + gz_statep state; +{ + unsigned got; + z_streamp strm = &(state->strm); + + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + if (state->eof == 0) { + if (strm->avail_in) { /* copy what's there to the start */ + unsigned char *p = state->in; + unsigned const char *q = strm->next_in; + unsigned n = strm->avail_in; + do { + *p++ = *q++; + } while (--n); + } + if (gz_load(state, state->in + strm->avail_in, + state->size - strm->avail_in, &got) == -1) + return -1; + strm->avail_in += got; + strm->next_in = state->in; + } + return 0; +} + +/* Look for gzip header, set up for inflate or copy. state->x.have must be 0. + If this is the first time in, allocate required memory. state->how will be + left unchanged if there is no more input data available, will be set to COPY + if there is no gzip header and direct copying will be performed, or it will + be set to GZIP for decompression. If direct copying, then leftover input + data from the input buffer will be copied to the output buffer. In that + case, all further file reads will be directly to either the output buffer or + a user buffer. If decompressing, the inflate state will be initialized. + gz_look() will return 0 on success or -1 on failure. */ +local int gz_look(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + /* allocate read buffers and inflate memory */ + if (state->size == 0) { + /* allocate buffers */ + state->in = (unsigned char *)malloc(state->want); + state->out = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL || state->out == NULL) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + state->size = state->want; + + /* allocate inflate memory */ + state->strm.zalloc = Z_NULL; + state->strm.zfree = Z_NULL; + state->strm.opaque = Z_NULL; + state->strm.avail_in = 0; + state->strm.next_in = Z_NULL; + if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */ + free(state->out); + free(state->in); + state->size = 0; + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + } + + /* get at least the magic bytes in the input buffer */ + if (strm->avail_in < 2) { + if (gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) + return 0; + } + + /* look for gzip magic bytes -- if there, do gzip decoding (note: there is + a logical dilemma here when considering the case of a partially written + gzip file, to wit, if a single 31 byte is written, then we cannot tell + whether this is a single-byte file, or just a partially written gzip + file -- for here we assume that if a gzip file is being written, then + the header will be written in a single operation, so that reading a + single byte is sufficient indication that it is not a gzip file) */ + if (strm->avail_in > 1 && + strm->next_in[0] == 31 && strm->next_in[1] == 139) { + inflateReset(strm); + state->how = GZIP; + state->direct = 0; + return 0; + } + + /* no gzip header -- if we were decoding gzip before, then this is trailing + garbage. Ignore the trailing garbage and finish. */ + if (state->direct == 0) { + strm->avail_in = 0; + state->eof = 1; + state->x.have = 0; + return 0; + } + + /* doing raw i/o, copy any leftover input to output -- this assumes that + the output buffer is larger than the input buffer, which also assures + space for gzungetc() */ + state->x.next = state->out; + if (strm->avail_in) { + memcpy(state->x.next, strm->next_in, strm->avail_in); + state->x.have = strm->avail_in; + strm->avail_in = 0; + } + state->how = COPY; + state->direct = 1; + return 0; +} + +/* Decompress from input to the provided next_out and avail_out in the state. + On return, state->x.have and state->x.next point to the just decompressed + data. If the gzip stream completes, state->how is reset to LOOK to look for + the next gzip stream or raw data, once state->x.have is depleted. Returns 0 + on success, -1 on failure. */ +local int gz_decomp(state) + gz_statep state; +{ + int ret = Z_OK; + unsigned had; + z_streamp strm = &(state->strm); + + /* fill output buffer up to end of deflate stream */ + had = strm->avail_out; + do { + /* get more input for inflate() */ + if (strm->avail_in == 0 && gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) { + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + break; + } + + /* decompress and handle errors */ + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { + gz_error(state, Z_STREAM_ERROR, + "internal error: inflate stream corrupt"); + return -1; + } + if (ret == Z_MEM_ERROR) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ + gz_error(state, Z_DATA_ERROR, + strm->msg == NULL ? "compressed data error" : strm->msg); + return -1; + } + } while (strm->avail_out && ret != Z_STREAM_END); + + /* update available output */ + state->x.have = had - strm->avail_out; + state->x.next = strm->next_out - state->x.have; + + /* if the gzip stream completed successfully, look for another */ + if (ret == Z_STREAM_END) + state->how = LOOK; + + /* good decompression */ + return 0; +} + +/* Fetch data and put it in the output buffer. Assumes state->x.have is 0. + Data is either copied from the input file or decompressed from the input + file depending on state->how. If state->how is LOOK, then a gzip header is + looked for to determine whether to copy or decompress. Returns -1 on error, + otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the + end of the input file has been reached and all data has been processed. */ +local int gz_fetch(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + do { + switch(state->how) { + case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ + if (gz_look(state) == -1) + return -1; + if (state->how == LOOK) + return 0; + break; + case COPY: /* -> COPY */ + if (gz_load(state, state->out, state->size << 1, &(state->x.have)) + == -1) + return -1; + state->x.next = state->out; + return 0; + case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ + strm->avail_out = state->size << 1; + strm->next_out = state->out; + if (gz_decomp(state) == -1) + return -1; + } + } while (state->x.have == 0 && (!state->eof || strm->avail_in)); + return 0; +} + +/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ +local int gz_skip(state, len) + gz_statep state; + z_off64_t len; +{ + unsigned n; + + /* skip over len bytes or reach end-of-file, whichever comes first */ + while (len) + /* skip over whatever is in output buffer */ + if (state->x.have) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? + (unsigned)len : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + len -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && state->strm.avail_in == 0) + break; + + /* need more data to skip -- load up output buffer */ + else { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return -1; + } + return 0; +} + +/* Read len bytes into buf from file, or less than len up to the end of the + input. Return the number of bytes read. If zero is returned, either the + end of file was reached, or there was an error. state->err must be + consulted in that case to determine which. */ +local z_size_t gz_read(state, buf, len) + gz_statep state; + voidp buf; + z_size_t len; +{ + z_size_t got; + unsigned n; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return 0; + } + + /* get len bytes to buf, or less than len if at the end */ + got = 0; + do { + /* set n to the maximum amount of len that fits in an unsigned int */ + n = -1; + if (n > len) + n = len; + + /* first just try copying data from the output buffer */ + if (state->x.have) { + if (state->x.have < n) + n = state->x.have; + memcpy(buf, state->x.next, n); + state->x.next += n; + state->x.have -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && state->strm.avail_in == 0) { + state->past = 1; /* tried to read past end */ + break; + } + + /* need output data -- for small len or new stream load up our output + buffer */ + else if (state->how == LOOK || n < (state->size << 1)) { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return 0; + continue; /* no progress yet -- go back to copy above */ + /* the copy above assures that we will leave with space in the + output buffer, allowing at least one gzungetc() to succeed */ + } + + /* large len -- read directly into user buffer */ + else if (state->how == COPY) { /* read directly */ + if (gz_load(state, (unsigned char *)buf, n, &n) == -1) + return 0; + } + + /* large len -- decompress directly into user buffer */ + else { /* state->how == GZIP */ + state->strm.avail_out = n; + state->strm.next_out = (unsigned char *)buf; + if (gz_decomp(state) == -1) + return 0; + n = state->x.have; + state->x.have = 0; + } + + /* update progress */ + len -= n; + buf = (char *)buf + n; + got += n; + state->x.pos += n; + } while (len); + + /* return number of bytes read into user buffer */ + return got; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in an int"); + return -1; + } + + /* read len or fewer bytes to buf */ + len = gz_read(state, buf, len); + + /* check for an error */ + if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* return the number of bytes read (this is assured to fit in an int) */ + return (int)len; +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* read len or fewer bytes to buf, return the number of full items read */ + return len ? gz_read(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +#else +# undef gzgetc +#endif +int ZEXPORT gzgetc(file) + gzFile file; +{ + int ret; + unsigned char buf[1]; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* try output buffer (no need to check for skip request) */ + if (state->x.have) { + state->x.have--; + state->x.pos++; + return *(state->x.next)++; + } + + /* nothing there -- try gz_read() */ + ret = gz_read(state, buf, 1); + return ret < 1 ? -1 : buf[0]; +} + +int ZEXPORT gzgetc_(file) +gzFile file; +{ + return gzgetc(file); +} + +/* -- see zlib.h -- */ +int ZEXPORT gzungetc(c, file) + int c; + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return -1; + } + + /* can't push EOF */ + if (c < 0) + return -1; + + /* if output buffer empty, put byte at end (allows more pushing) */ + if (state->x.have == 0) { + state->x.have = 1; + state->x.next = state->out + (state->size << 1) - 1; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; + return c; + } + + /* if no room, give up (must have already done a gzungetc()) */ + if (state->x.have == (state->size << 1)) { + gz_error(state, Z_DATA_ERROR, "out of room to push characters"); + return -1; + } + + /* slide output data if needed and insert byte before existing data */ + if (state->x.next == state->out) { + unsigned char *src = state->out + state->x.have; + unsigned char *dest = state->out + (state->size << 1); + while (src > state->out) + *--dest = *--src; + state->x.next = dest; + } + state->x.have++; + state->x.next--; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; + return c; +} + +/* -- see zlib.h -- */ +char * ZEXPORT gzgets(file, buf, len) + gzFile file; + char *buf; + int len; +{ + unsigned left, n; + char *str; + unsigned char *eol; + gz_statep state; + + /* check parameters and get internal structure */ + if (file == NULL || buf == NULL || len < 1) + return NULL; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return NULL; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return NULL; + } + + /* copy output bytes up to new line or len - 1, whichever comes first -- + append a terminating zero to the string (we don't check for a zero in + the contents, let the user worry about that) */ + str = buf; + left = (unsigned)len - 1; + if (left) do { + /* assure that something is in the output buffer */ + if (state->x.have == 0 && gz_fetch(state) == -1) + return NULL; /* error */ + if (state->x.have == 0) { /* end of file */ + state->past = 1; /* read past end */ + break; /* return what we have */ + } + + /* look for end-of-line in current output buffer */ + n = state->x.have > left ? left : state->x.have; + eol = (unsigned char *)memchr(state->x.next, '\n', n); + if (eol != NULL) + n = (unsigned)(eol - state->x.next) + 1; + + /* copy through end-of-line, or remainder if not found */ + memcpy(buf, state->x.next, n); + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + left -= n; + buf += n; + } while (left && eol == NULL); + + /* return terminated string, or if nothing, end of file */ + if (buf == str) + return NULL; + buf[0] = 0; + return str; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzdirect(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* if the state is not known, but we can find out, then do so (this is + mainly for right after a gzopen() or gzdopen()) */ + if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) + (void)gz_look(state); + + /* return 1 if transparent, 0 if processing a gzip stream */ + return state->direct; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_r(file) + gzFile file; +{ + int ret, err; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're reading */ + if (state->mode != GZ_READ) + return Z_STREAM_ERROR; + + /* free memory and close file */ + if (state->size) { + inflateEnd(&(state->strm)); + free(state->out); + free(state->in); + } + err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; + gz_error(state, Z_OK, NULL); + free(state->path); + ret = close(state->fd); + free(state); + return ret ? Z_ERRNO : err; +} diff --git a/zlib/gzwrite.c b/zlib/gzwrite.c new file mode 100644 index 00000000000..c7b5651d70b --- /dev/null +++ b/zlib/gzwrite.c @@ -0,0 +1,665 @@ +/* gzwrite.c -- zlib functions for writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_init OF((gz_statep)); +local int gz_comp OF((gz_statep, int)); +local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); + +/* Initialize state for writing a gzip file. Mark initialization by setting + state->size to non-zero. Return -1 on a memory allocation failure, or 0 on + success. */ +local int gz_init(state) + gz_statep state; +{ + int ret; + z_streamp strm = &(state->strm); + + /* allocate input buffer (double size for gzprintf) */ + state->in = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* only need output buffer and deflate state if compressing */ + if (!state->direct) { + /* allocate output buffer */ + state->out = (unsigned char *)malloc(state->want); + if (state->out == NULL) { + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* allocate deflate memory, set up for gzip compression */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + ret = deflateInit2(strm, state->level, Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy); + if (ret != Z_OK) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + strm->next_in = NULL; + } + + /* mark state as initialized */ + state->size = state->want; + + /* initialize write buffer if compressing */ + if (!state->direct) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = strm->next_out; + } + return 0; +} + +/* Compress whatever is at avail_in and next_in and write to the output file. + Return -1 if there is an error writing to the output file or if gz_init() + fails to allocate memory, otherwise 0. flush is assumed to be a valid + deflate() flush value. If flush is Z_FINISH, then the deflate() state is + reset to start a new gzip stream. If gz->direct is true, then simply write + to the output file without compressing, and ignore flush. */ +local int gz_comp(state, flush) + gz_statep state; + int flush; +{ + int ret, writ; + unsigned have, put, max = ((unsigned)-1 >> 2) + 1; + z_streamp strm = &(state->strm); + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return -1; + + /* write directly if requested */ + if (state->direct) { + while (strm->avail_in) { + put = strm->avail_in > max ? max : strm->avail_in; + writ = write(state->fd, strm->next_in, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in -= (unsigned)writ; + strm->next_in += writ; + } + return 0; + } + + /* run deflate() on provided input until it produces no more output */ + ret = Z_OK; + do { + /* write out current buffer contents if full, or if flushing, but if + doing Z_FINISH then don't write until we get to Z_STREAM_END */ + if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && + (flush != Z_FINISH || ret == Z_STREAM_END))) { + while (strm->next_out > state->x.next) { + put = strm->next_out - state->x.next > (int)max ? max : + (unsigned)(strm->next_out - state->x.next); + writ = write(state->fd, state->x.next, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + state->x.next += writ; + } + if (strm->avail_out == 0) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = state->out; + } + } + + /* compress */ + have = strm->avail_out; + ret = deflate(strm, flush); + if (ret == Z_STREAM_ERROR) { + gz_error(state, Z_STREAM_ERROR, + "internal error: deflate stream corrupt"); + return -1; + } + have -= strm->avail_out; + } while (have); + + /* if that completed a deflate stream, allow another to start */ + if (flush == Z_FINISH) + deflateReset(strm); + + /* all done, no errors */ + return 0; +} + +/* Compress len zeros to output. Return -1 on a write error or memory + allocation failure by gz_comp(), or 0 on success. */ +local int gz_zero(state, len) + gz_statep state; + z_off64_t len; +{ + int first; + unsigned n; + z_streamp strm = &(state->strm); + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + + /* compress len zeros (len guaranteed > 0) */ + first = 1; + while (len) { + n = GT_OFF(state->size) || (z_off64_t)state->size > len ? + (unsigned)len : state->size; + if (first) { + memset(state->in, 0, n); + first = 0; + } + strm->avail_in = n; + strm->next_in = state->in; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + len -= n; + } + return 0; +} + +/* Write len bytes from buf to file. Return the number of bytes written. If + the returned value is less than len, then there was an error. */ +local z_size_t gz_write(state, buf, len) + gz_statep state; + voidpc buf; + z_size_t len; +{ + z_size_t put = len; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return 0; + } + + /* for small len, copy to input buffer, otherwise compress directly */ + if (len < state->size) { + /* copy to input buffer, compress when full */ + do { + unsigned have, copy; + + if (state->strm.avail_in == 0) + state->strm.next_in = state->in; + have = (unsigned)((state->strm.next_in + state->strm.avail_in) - + state->in); + copy = state->size - have; + if (copy > len) + copy = len; + memcpy(state->in + have, buf, copy); + state->strm.avail_in += copy; + state->x.pos += copy; + buf = (const char *)buf + copy; + len -= copy; + if (len && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + } while (len); + } + else { + /* consume whatever's left in the input buffer */ + if (state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* directly compress user buffer to file */ + state->strm.next_in = (z_const Bytef *)buf; + do { + unsigned n = (unsigned)-1; + if (n > len) + n = len; + state->strm.avail_in = n; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + len -= n; + } while (len); + } + + /* input was all buffered or compressed */ + return put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; + } + + /* write len bytes from buf (the return value will fit in an int) */ + return (int)gz_write(state, buf, len); +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* write len bytes to buf, return the number of full items written */ + return len ? gz_write(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputc(file, c) + gzFile file; + int c; +{ + unsigned have; + unsigned char buf[1]; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return -1; + } + + /* try writing to input buffer for speed (state->size == 0 if buffer not + initialized) */ + if (state->size) { + if (strm->avail_in == 0) + strm->next_in = state->in; + have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + if (have < state->size) { + state->in[have] = (unsigned char)c; + strm->avail_in++; + state->x.pos++; + return c & 0xff; + } + } + + /* no room in buffer or not initialized, use gz_write() */ + buf[0] = (unsigned char)c; + if (gz_write(state, buf, 1) != 1) + return -1; + return c & 0xff; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputs(file, str) + gzFile file; + const char *str; +{ + int ret; + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; + + /* write string */ + len = strlen(str); + ret = gz_write(state, str, len); + return ret == 0 && len != 0 ? -1 : ret; +} + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +#include + +/* -- see zlib.h -- */ +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) +{ + int len; + unsigned left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return state->err; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(state->in + (strm->next_in - state->in) + strm->avail_in); + next[state->size - 1] = 0; +#ifdef NO_vsnprintf +# ifdef HAS_vsprintf_void + (void)vsprintf(next, format, va); + for (len = 0; len < state->size; len++) + if (next[len] == 0) break; +# else + len = vsprintf(next, format, va); +# endif +#else +# ifdef HAS_vsnprintf_void + (void)vsnprintf(next, state->size, format, va); + len = strlen(next); +# else + len = vsnprintf(next, state->size, format, va); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || (unsigned)len >= state->size || next[state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += (unsigned)len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return len; +} + +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ + va_list va; + int ret; + + va_start(va, format); + ret = gzvprintf(file, format, va); + va_end(va); + return ret; +} + +#else /* !STDC && !Z_HAVE_STDARG_H */ + +/* -- see zlib.h -- */ +int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) + gzFile file; + const char *format; + int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; +{ + unsigned len, left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that can really pass pointer in ints */ + if (sizeof(int) != sizeof(void *)) + return Z_STREAM_ERROR; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return state->error; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->error; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(strm->next_in + strm->avail_in); + next[state->size - 1] = 0; +#ifdef NO_snprintf +# ifdef HAS_sprintf_void + sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + a13, a14, a15, a16, a17, a18, a19, a20); + for (len = 0; len < size; len++) + if (next[len] == 0) + break; +# else + len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, + a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#else +# ifdef HAS_snprintf_void + snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen(next); +# else + len = snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || len >= state->size || next[state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return (int)len; +} + +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzflush(file, flush) + gzFile file; + int flush; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* check flush parameter */ + if (flush < 0 || flush > Z_FINISH) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* compress remaining data with requested flush */ + (void)gz_comp(state, flush); + return state->err; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzsetparams(file, level, strategy) + gzFile file; + int level; + int strategy; +{ + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* if no change is requested, then do nothing */ + if (level == state->level && strategy == state->strategy) + return Z_OK; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* change compression parameters for subsequent input */ + if (state->size) { + /* flush previous input with previous parameters before changing */ + if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1) + return state->err; + deflateParams(strm, level, strategy); + } + state->level = level; + state->strategy = strategy; + return Z_OK; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_w(file) + gzFile file; +{ + int ret = Z_OK; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're writing */ + if (state->mode != GZ_WRITE) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + ret = state->err; + } + + /* flush, free memory, and close file */ + if (gz_comp(state, Z_FINISH) == -1) + ret = state->err; + if (state->size) { + if (!state->direct) { + (void)deflateEnd(&(state->strm)); + free(state->out); + } + free(state->in); + } + gz_error(state, Z_OK, NULL); + free(state->path); + if (close(state->fd) == -1) + ret = Z_ERRNO; + free(state); + return ret; +} diff --git a/zlib/infback.c b/zlib/infback.c index 455dbc9ee84..59679ecbfc5 100644 --- a/zlib/infback.c +++ b/zlib/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -42,20 +42,29 @@ int stream_size; return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif state = (struct inflate_state FAR *)ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->dmax = 32768U; - state->wbits = windowBits; + state->wbits = (uInt)windowBits; state->wsize = 1U << windowBits; state->window = window; - state->write = 0; + state->wnext = 0; state->whave = 0; return Z_OK; } @@ -246,14 +255,14 @@ out_func out; void FAR *out_desc; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ - code this; /* current decoding table entry */ + code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ @@ -389,19 +398,18 @@ void FAR *out_desc; state->have = 0; while (state->have < state->nlen + state->ndist) { for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.val < 16) { - NEEDBITS(this.bits); - DROPBITS(this.bits); - state->lens[state->have++] = this.val; + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; } else { - if (this.val == 16) { - NEEDBITS(this.bits + 2); - DROPBITS(this.bits); + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); if (state->have == 0) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; @@ -411,16 +419,16 @@ void FAR *out_desc; copy = 3 + BITS(2); DROPBITS(2); } - else if (this.val == 17) { - NEEDBITS(this.bits + 3); - DROPBITS(this.bits); + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); len = 0; copy = 3 + BITS(3); DROPBITS(3); } else { - NEEDBITS(this.bits + 7); - DROPBITS(this.bits); + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); len = 0; copy = 11 + BITS(7); DROPBITS(7); @@ -438,7 +446,16 @@ void FAR *out_desc; /* handle error breaks in while */ if (state->mode == BAD) break; - /* build code tables */ + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (code const FAR *)(state->next); state->lenbits = 9; @@ -474,28 +491,28 @@ void FAR *out_desc; /* get a literal, length, or end-of-block code */ for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.op && (this.op & 0xf0) == 0) { - last = this; + if (here.op && (here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->lencode[last.val + + here = state->lencode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } - DROPBITS(this.bits); - state->length = (unsigned)this.val; + DROPBITS(here.bits); + state->length = (unsigned)here.val; /* process literal */ - if (this.op == 0) { - Tracevv((stderr, this.val >= 0x20 && this.val < 0x7f ? + if (here.op == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", this.val)); + "inflate: literal 0x%02x\n", here.val)); ROOM(); *put++ = (unsigned char)(state->length); left--; @@ -504,21 +521,21 @@ void FAR *out_desc; } /* process end of block */ - if (this.op & 32) { + if (here.op & 32) { Tracevv((stderr, "inflate: end of block\n")); state->mode = TYPE; break; } /* invalid code */ - if (this.op & 64) { + if (here.op & 64) { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } /* length code -- get extra bits, if any */ - state->extra = (unsigned)(this.op) & 15; + state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->length += BITS(state->extra); @@ -528,30 +545,30 @@ void FAR *out_desc; /* get distance code */ for (;;) { - this = state->distcode[BITS(state->distbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if ((this.op & 0xf0) == 0) { - last = this; + if ((here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->distcode[last.val + + here = state->distcode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } - DROPBITS(this.bits); - if (this.op & 64) { + DROPBITS(here.bits); + if (here.op & 64) { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } - state->offset = (unsigned)this.val; + state->offset = (unsigned)here.val; /* get distance extra bits, if any */ - state->extra = (unsigned)(this.op) & 15; + state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->offset += BITS(state->extra); diff --git a/zlib/inffast.c b/zlib/inffast.c index bbee92ed1e6..0dbd1dbc09f 100644 --- a/zlib/inffast.c +++ b/zlib/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2004 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,26 +8,9 @@ #include "inflate.h" #include "inffast.h" -#ifndef ASMINF - -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") #else -# define OFF 1 -# define PUP(a) *++(a) -#endif /* Decode literal, length, and distance codes and write out the resulting @@ -64,13 +47,13 @@ requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ -void inflate_fast(strm, start) +void ZLIB_INTERNAL inflate_fast(strm, start) z_streamp strm; unsigned start; /* inflate()'s starting value for strm->avail_out */ { struct inflate_state FAR *state; - unsigned char FAR *in; /* local strm->next_in */ - unsigned char FAR *last; /* while in < last, enough input available */ + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ unsigned char FAR *out; /* local strm->next_out */ unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ unsigned char FAR *end; /* while out < end, enough space available */ @@ -79,7 +62,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #endif unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ - unsigned write; /* window write index */ + unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ unsigned long hold; /* local strm->hold */ unsigned bits; /* local strm->bits */ @@ -87,7 +70,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ code const FAR *dcode; /* local strm->distcode */ unsigned lmask; /* mask for first level of length codes */ unsigned dmask; /* mask for first level of distance codes */ - code this; /* retrieved table entry */ + code here; /* retrieved table entry */ unsigned op; /* code bits, operation, extra bits, or */ /* window position, window bytes to copy */ unsigned len; /* match length, unused bytes */ @@ -96,9 +79,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -106,7 +89,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #endif wsize = state->wsize; whave = state->whave; - write = state->write; + wnext = state->wnext; window = state->window; hold = state->hold; bits = state->bits; @@ -119,29 +102,29 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } - this = lcode[hold & lmask]; + here = lcode[hold & lmask]; dolen: - op = (unsigned)(this.bits); + op = (unsigned)(here.bits); hold >>= op; bits -= op; - op = (unsigned)(this.op); + op = (unsigned)(here.op); if (op == 0) { /* literal */ - Tracevv((stderr, this.val >= 0x20 && this.val < 0x7f ? + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", this.val)); - PUP(out) = (unsigned char)(this.val); + "inflate: literal 0x%02x\n", here.val)); + *out++ = (unsigned char)(here.val); } else if (op & 16) { /* length base */ - len = (unsigned)(this.val); + len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -150,25 +133,25 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } - this = dcode[hold & dmask]; + here = dcode[hold & dmask]; dodist: - op = (unsigned)(this.bits); + op = (unsigned)(here.bits); hold >>= op; bits -= op; - op = (unsigned)(this.op); + op = (unsigned)(here.op); if (op & 16) { /* distance base */ - dist = (unsigned)(this.val); + dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -187,79 +170,101 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (dist > op) { /* see if copy from window */ op = dist - op; /* distance back in window */ if (op > whave) { - strm->msg = (char *)"invalid distance too far back"; - state->mode = BAD; - break; + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif } - from = window - OFF; - if (write == 0) { /* very common case */ + from = window; + if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } - else if (write < op) { /* wrap around window */ - from += wsize + write - op; - op -= write; + else if (wnext < op) { /* wrap around window */ + from += wsize + wnext - op; + op -= wnext; if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; - if (write < len) { /* some from start of window */ - op = write; + from = window; + if (wnext < len) { /* some from start of window */ + op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } } else { /* contiguous in window */ - from += write - op; + from += wnext - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } else if ((op & 64) == 0) { /* 2nd level distance code */ - this = dcode[this.val + (hold & ((1U << op) - 1))]; + here = dcode[here.val + (hold & ((1U << op) - 1))]; goto dodist; } else { @@ -269,7 +274,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } } else if ((op & 64) == 0) { /* 2nd level length code */ - this = lcode[this.val + (hold & ((1U << op) - 1))]; + here = lcode[here.val + (hold & ((1U << op) - 1))]; goto dolen; } else if (op & 32) { /* end-of-block */ @@ -291,8 +296,8 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); @@ -305,7 +310,7 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): - Using bit fields for code structure - Different op definition to avoid & for extra bits (do & for table bits) - - Three separate decoding do-loops for direct, window, and write == 0 + - Three separate decoding do-loops for direct, window, and wnext == 0 - Special case for distance > 1 copies to do overlapped load and store copy - Explicit branch predictions (based on measured branch probabilities) - Deferring match copy and interspersed it with decoding subsequent codes diff --git a/zlib/inffast.h b/zlib/inffast.h index 1e88d2d97b5..e5c1aa4ca8c 100644 --- a/zlib/inffast.h +++ b/zlib/inffast.h @@ -1,5 +1,5 @@ /* inffast.h -- header to use inffast.c - * Copyright (C) 1995-2003 Mark Adler + * Copyright (C) 1995-2003, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,4 +8,4 @@ subject to change. Applications should only use zlib.h. */ -void inflate_fast OF((z_streamp strm, unsigned start)); +void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/zlib/inffixed.h b/zlib/inffixed.h index 75ed4b5978d..d6283277694 100644 --- a/zlib/inffixed.h +++ b/zlib/inffixed.h @@ -2,9 +2,9 @@ * Generated automatically by makefixed(). */ - /* WARNING: this file should *not* be used by applications. It - is part of the implementation of the compression library and - is subject to change. Applications should only use zlib.h. + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. */ static const code lenfix[512] = { diff --git a/zlib/inflate.c b/zlib/inflate.c index 792fdee8e9c..ac333e8c2ed 100644 --- a/zlib/inflate.c +++ b/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -45,7 +45,7 @@ * - Rearrange window copies in inflate_fast() for speed and simplification * - Unroll last copy for window match in inflate_fast() * - Use local copies of window variables in inflate_fast() for speed - * - Pull out common write == 0 case for speed in inflate_fast() + * - Pull out common wnext == 0 case for speed in inflate_fast() * - Make op and len in inflate_fast() unsigned for consistency * - Add FAR to lcode and dcode declarations in inflate_fast() * - Simplified bad distance check in inflate_fast() @@ -92,37 +92,156 @@ #endif /* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); local void fixedtables OF((struct inflate_state FAR *state)); -local int updatewindow OF((z_streamp strm, unsigned out)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); #ifdef BUILDFIXED void makefixed OF((void)); #endif -local unsigned syncsearch OF((unsigned FAR *have, unsigned char FAR *buf, +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + int ZEXPORT inflateReset(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - strm->total_in = strm->total_out = state->total = 0; - strm->msg = Z_NULL; - strm->adler = 1; /* to support ill-conceived Java test suite */ - state->mode = HEAD; - state->last = 0; - state->havedict = 0; - state->dmax = 32768U; - state->head = Z_NULL; state->wsize = 0; state->whave = 0; - state->write = 0; - state->hold = 0; - state->bits = 0; - state->lencode = state->distcode = state->next = state->codes; - Tracev((stderr, "inflate: reset\n")); - return Z_OK; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); } int ZEXPORT inflatePrime(strm, bits, value) @@ -132,66 +251,20 @@ int value; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += value << state->bits; - state->bits += bits; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; return Z_OK; } -int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) -z_streamp strm; -int windowBits; -const char *version; -int stream_size; -{ - struct inflate_state FAR *state; - - if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || - stream_size != (int)(sizeof(z_stream))) - return Z_VERSION_ERROR; - if (strm == Z_NULL) return Z_STREAM_ERROR; - strm->msg = Z_NULL; /* in case we return an error */ - if (strm->zalloc == (alloc_func)0) { - strm->zalloc = zcalloc; - strm->opaque = (voidpf)0; - } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; - state = (struct inflate_state FAR *) - ZALLOC(strm, 1, sizeof(struct inflate_state)); - if (state == Z_NULL) return Z_MEM_ERROR; - Tracev((stderr, "inflate: allocated\n")); - strm->state = (struct internal_state FAR *)state; - if (windowBits < 0) { - state->wrap = 0; - windowBits = -windowBits; - } - else { - state->wrap = (windowBits >> 4) + 1; -#ifdef GUNZIP - if (windowBits < 48) windowBits &= 15; -#endif - } - if (windowBits < 8 || windowBits > 15) { - ZFREE(strm, state); - strm->state = Z_NULL; - return Z_STREAM_ERROR; - } - state->wbits = (unsigned)windowBits; - state->window = Z_NULL; - return inflateReset(strm); -} - -int ZEXPORT inflateInit_(strm, version, stream_size) -z_streamp strm; -const char *version; -int stream_size; -{ - return inflateInit2_(strm, DEF_WBITS, version, stream_size); -} - /* Return state with length and distance decoding tables and index sizes set to fixed code decoding. Normally this returns fixed tables from inffixed.h. @@ -286,8 +359,8 @@ void makefixed() low = 0; for (;;) { if ((low % 7) == 0) printf("\n "); - printf("{%u,%u,%d}", state.lencode[low].op, state.lencode[low].bits, - state.lencode[low].val); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); if (++low == size) break; putchar(','); } @@ -320,12 +393,13 @@ void makefixed() output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(strm, out) +local int updatewindow(strm, end, copy) z_streamp strm; -unsigned out; +const Bytef *end; +unsigned copy; { struct inflate_state FAR *state; - unsigned copy, dist; + unsigned dist; state = (struct inflate_state FAR *)strm->state; @@ -340,30 +414,29 @@ unsigned out; /* if window not in use yet, initialize */ if (state->wsize == 0) { state->wsize = 1U << state->wbits; - state->write = 0; + state->wnext = 0; state->whave = 0; } /* copy state->wsize or less output bytes into the circular window */ - copy = out - strm->avail_out; if (copy >= state->wsize) { - zmemcpy(state->window, strm->next_out - state->wsize, state->wsize); - state->write = 0; + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; state->whave = state->wsize; } else { - dist = state->wsize - state->write; + dist = state->wsize - state->wnext; if (dist > copy) dist = copy; - zmemcpy(state->window + state->write, strm->next_out - copy, dist); + zmemcpy(state->window + state->wnext, end - copy, dist); copy -= dist; if (copy) { - zmemcpy(state->window, strm->next_out - copy, copy); - state->write = copy; + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; state->whave = state->wsize; } else { - state->write += dist; - if (state->write == state->wsize) state->write = 0; + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; if (state->whave < state->wsize) state->whave += dist; } } @@ -464,11 +537,6 @@ unsigned out; bits -= bits & 7; \ } while (0) -/* Reverse the bytes in a 32-bit value */ -#define REVERSE(q) \ - ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is @@ -556,7 +624,7 @@ z_streamp strm; int flush; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ @@ -564,7 +632,7 @@ int flush; unsigned in, out; /* save starting available input and output */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ - code this; /* current decoding table entry */ + code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ @@ -574,7 +642,7 @@ int flush; static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; @@ -594,6 +662,8 @@ int flush; NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); @@ -619,7 +689,9 @@ int flush; } DROPBITS(4); len = BITS(4) + 8; - if (len > state->wbits) { + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; @@ -646,14 +718,16 @@ int flush; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; - if (state->flags & 0x0200) CRC4(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: @@ -662,7 +736,8 @@ int flush; state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: @@ -671,7 +746,8 @@ int flush; state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) @@ -689,7 +765,7 @@ int flush; len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -708,9 +784,9 @@ int flush; if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) - state->head->name[state->length++] = len; + state->head->name[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -729,9 +805,9 @@ int flush; if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) - state->head->comment[state->length++] = len; + state->head->comment[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -743,7 +819,7 @@ int flush; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); - if (hold != (state->check & 0xffff)) { + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { strm->msg = (char *)"header crc mismatch"; state->mode = BAD; break; @@ -760,7 +836,7 @@ int flush; #endif case DICTID: NEEDBITS(32); - strm->adler = state->check = REVERSE(hold); + strm->adler = state->check = ZSWAP32(hold); INITBITS(); state->mode = DICT; case DICT: @@ -771,7 +847,7 @@ int flush; strm->adler = state->check = adler32(0L, Z_NULL, 0); state->mode = TYPE; case TYPE: - if (flush == Z_BLOCK) goto inf_leave; + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; case TYPEDO: if (state->last) { BYTEBITS(); @@ -791,7 +867,11 @@ int flush; fixedtables(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); - state->mode = LEN; /* decode codes */ + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } break; case 2: /* dynamic block */ Tracev((stderr, "inflate: dynamic codes block%s\n", @@ -816,6 +896,9 @@ int flush; Tracev((stderr, "inflate: stored length %u\n", state->length)); INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + case COPY_: state->mode = COPY; case COPY: copy = state->length; @@ -861,7 +944,7 @@ int flush; while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; - state->lencode = (code const FAR *)(state->next); + state->lencode = (const code FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); @@ -876,19 +959,18 @@ int flush; case CODELENS: while (state->have < state->nlen + state->ndist) { for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.val < 16) { - NEEDBITS(this.bits); - DROPBITS(this.bits); - state->lens[state->have++] = this.val; + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; } else { - if (this.val == 16) { - NEEDBITS(this.bits + 2); - DROPBITS(this.bits); + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); if (state->have == 0) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; @@ -898,16 +980,16 @@ int flush; copy = 3 + BITS(2); DROPBITS(2); } - else if (this.val == 17) { - NEEDBITS(this.bits + 3); - DROPBITS(this.bits); + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); len = 0; copy = 3 + BITS(3); DROPBITS(3); } else { - NEEDBITS(this.bits + 7); - DROPBITS(this.bits); + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); len = 0; copy = 11 + BITS(7); DROPBITS(7); @@ -925,9 +1007,18 @@ int flush; /* handle error breaks in while */ if (state->mode == BAD) break; - /* build code tables */ + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; - state->lencode = (code const FAR *)(state->next); + state->lencode = (const code FAR *)(state->next); state->lenbits = 9; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); @@ -936,7 +1027,7 @@ int flush; state->mode = BAD; break; } - state->distcode = (code const FAR *)(state->next); + state->distcode = (const code FAR *)(state->next); state->distbits = 6; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); @@ -946,88 +1037,102 @@ int flush; break; } Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + case LEN_: state->mode = LEN; case LEN: if (have >= 6 && left >= 258) { RESTORE(); inflate_fast(strm, out); LOAD(); + if (state->mode == TYPE) + state->back = -1; break; } + state->back = 0; for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.op && (this.op & 0xf0) == 0) { - last = this; + if (here.op && (here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->lencode[last.val + + here = state->lencode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); + state->back += last.bits; } - DROPBITS(this.bits); - state->length = (unsigned)this.val; - if ((int)(this.op) == 0) { - Tracevv((stderr, this.val >= 0x20 && this.val < 0x7f ? + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", this.val)); + "inflate: literal 0x%02x\n", here.val)); state->mode = LIT; break; } - if (this.op & 32) { + if (here.op & 32) { Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; state->mode = TYPE; break; } - if (this.op & 64) { + if (here.op & 64) { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } - state->extra = (unsigned)(this.op) & 15; + state->extra = (unsigned)(here.op) & 15; state->mode = LENEXT; case LENEXT: if (state->extra) { NEEDBITS(state->extra); state->length += BITS(state->extra); DROPBITS(state->extra); + state->back += state->extra; } Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; state->mode = DIST; case DIST: for (;;) { - this = state->distcode[BITS(state->distbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if ((this.op & 0xf0) == 0) { - last = this; + if ((here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->distcode[last.val + + here = state->distcode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); + state->back += last.bits; } - DROPBITS(this.bits); - if (this.op & 64) { + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } - state->offset = (unsigned)this.val; - state->extra = (unsigned)(this.op) & 15; + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; state->mode = DISTEXT; case DISTEXT: if (state->extra) { NEEDBITS(state->extra); state->offset += BITS(state->extra); DROPBITS(state->extra); + state->back += state->extra; } #ifdef INFLATE_STRICT if (state->offset > state->dmax) { @@ -1036,11 +1141,6 @@ int flush; break; } #endif - if (state->offset > state->whave + out - left) { - strm->msg = (char *)"invalid distance too far back"; - state->mode = BAD; - break; - } Tracevv((stderr, "inflate: distance %u\n", state->offset)); state->mode = MATCH; case MATCH: @@ -1048,12 +1148,32 @@ int flush; copy = out - left; if (state->offset > copy) { /* copy from window */ copy = state->offset - copy; - if (copy > state->write) { - copy -= state->write; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; from = state->window + (state->wsize - copy); } else - from = state->window + (state->write - copy); + from = state->window + (state->wnext - copy); if (copy > state->length) copy = state->length; } else { /* copy from output */ @@ -1080,15 +1200,15 @@ int flush; out -= left; strm->total_out += out; state->total += out; - if (out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; - if (( + if ((state->wrap & 4) && ( #ifdef GUNZIP state->flags ? hold : #endif - REVERSE(hold)) != state->check) { + ZSWAP32(hold)) != state->check) { strm->msg = (char *)"incorrect data check"; state->mode = BAD; break; @@ -1132,8 +1252,9 @@ int flush; */ inf_leave: RESTORE(); - if (state->wsize || (state->mode < CHECK && out != strm->avail_out)) - if (updatewindow(strm, out)) { + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { state->mode = MEM; return Z_MEM_ERROR; } @@ -1142,11 +1263,12 @@ int flush; strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); - strm->data_type = state->bits + (state->last ? 64 : 0) + - (state->mode == TYPE ? 128 : 0); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) ret = Z_BUF_ERROR; return ret; @@ -1156,7 +1278,7 @@ int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); @@ -1166,43 +1288,59 @@ z_streamp strm; return Z_OK; } +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) z_streamp strm; const Bytef *dictionary; uInt dictLength; { struct inflate_state FAR *state; - unsigned long id; + unsigned long dictid; + int ret; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; - /* check for correct dictionary id */ + /* check for correct dictionary identifier */ if (state->mode == DICT) { - id = adler32(0L, Z_NULL, 0); - id = adler32(id, dictionary, dictLength); - if (id != state->check) + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) return Z_DATA_ERROR; } - /* copy dictionary to window */ - if (updatewindow(strm, strm->avail_out)) { + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { state->mode = MEM; return Z_MEM_ERROR; } - if (dictLength > state->wsize) { - zmemcpy(state->window, dictionary + dictLength - state->wsize, - state->wsize); - state->whave = state->wsize; - } - else { - zmemcpy(state->window + state->wsize - dictLength, dictionary, - dictLength); - state->whave = dictLength; - } state->havedict = 1; Tracev((stderr, "inflate: dictionary set\n")); return Z_OK; @@ -1215,7 +1353,7 @@ gz_headerp head; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; @@ -1238,7 +1376,7 @@ gz_headerp head; */ local unsigned syncsearch(have, buf, len) unsigned FAR *have; -unsigned char FAR *buf; +const unsigned char FAR *buf; unsigned len; { unsigned got; @@ -1268,7 +1406,7 @@ z_streamp strm; struct inflate_state FAR *state; /* check parameters */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; @@ -1315,7 +1453,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1330,8 +1468,7 @@ z_streamp source; unsigned wsize; /* check input */ - if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || - source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) + if (inflateStateCheck(source) || dest == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; @@ -1350,8 +1487,9 @@ z_streamp source; } /* copy state */ - zmemcpy(dest, source, sizeof(z_stream)); - zmemcpy(copy, state, sizeof(struct inflate_state)); + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); @@ -1366,3 +1504,58 @@ z_streamp source; dest->state = (struct internal_state FAR *)copy; return Z_OK; } + +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/zlib/inflate.h b/zlib/inflate.h index 07bd3e78a7c..a46cce6b6d0 100644 --- a/zlib/inflate.h +++ b/zlib/inflate.h @@ -1,5 +1,5 @@ /* inflate.h -- internal inflate state definition - * Copyright (C) 1995-2004 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,7 +18,7 @@ /* Possible inflate modes between inflate() calls */ typedef enum { - HEAD, /* i: waiting for magic header */ + HEAD = 16180, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ @@ -32,11 +32,13 @@ typedef enum { TYPE, /* i: waiting for type bits, including last-flag bit */ TYPEDO, /* i: same, but skip check to exit inflate on new block */ STORED, /* i: waiting for stored size (length and complement) */ + COPY_, /* i/o: same as COPY below, but only first time in */ COPY, /* i/o: waiting for input or output to copy stored block */ TABLE, /* i: waiting for dynamic block table lengths */ LENLENS, /* i: waiting for code length code lengths */ CODELENS, /* i: waiting for length/lit and distance code lengths */ - LEN, /* i: waiting for length/lit code */ + LEN_, /* i: same as LEN below, but only first time in */ + LEN, /* i: waiting for length/lit/eob code */ LENEXT, /* i: waiting for length extra bits */ DIST, /* i: waiting for distance code */ DISTEXT, /* i: waiting for distance extra bits */ @@ -53,19 +55,21 @@ typedef enum { /* State transitions between above modes - - (most modes can go to the BAD or MEM mode -- not shown for clarity) + (most modes can go to BAD or MEM on error -- not shown for clarity) Process header: - HEAD -> (gzip) or (zlib) - (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME - NAME -> COMMENT -> HCRC -> TYPE + HEAD -> (gzip) or (zlib) or (raw) + (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT -> + HCRC -> TYPE (zlib) -> DICTID or TYPE DICTID -> DICT -> TYPE + (raw) -> TYPEDO Read deflate blocks: - TYPE -> STORED or TABLE or LEN or CHECK - STORED -> COPY -> TYPE - TABLE -> LENLENS -> CODELENS -> LEN - Read deflate codes: + TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK + STORED -> COPY_ -> COPY -> TYPE + TABLE -> LENLENS -> CODELENS -> LEN_ + LEN_ -> LEN + Read deflate codes in fixed or dynamic block: LEN -> LENEXT or LIT or TYPE LENEXT -> DIST -> DISTEXT -> MATCH -> LEN LIT -> LEN @@ -73,11 +77,14 @@ typedef enum { CHECK -> LENGTH -> DONE */ -/* state maintained between inflate() calls. Approximately 7K bytes. */ +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ - int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ @@ -88,7 +95,7 @@ struct inflate_state { unsigned wbits; /* log base 2 of requested window size */ unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ - unsigned write; /* window write index */ + unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if needed */ /* bit accumulator */ unsigned long hold; /* input bit accumulator */ @@ -112,4 +119,7 @@ struct inflate_state { unsigned short lens[320]; /* temporary storage for code lengths */ unsigned short work[288]; /* work area for code table building */ code codes[ENOUGH]; /* space for code tables */ + int sane; /* if false, allow invalid distance too far */ + int back; /* bits back of last unprocessed length/lit */ + unsigned was; /* initial length of match */ }; diff --git a/zlib/inftrees.c b/zlib/inftrees.c index 8a9c13ff03d..2ea08fc13ea 100644 --- a/zlib/inftrees.c +++ b/zlib/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,7 +9,7 @@ #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.2.3 Copyright 1995-2005 Mark Adler "; + " inflate 1.2.11 Copyright 1995-2017 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -29,7 +29,7 @@ const char inflate_copyright[] = table index bits. It will differ if the request is greater than the longest code or if it is less than the shortest code. */ -int inflate_table(type, lens, codes, table, bits, work) +int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) codetype type; unsigned short FAR *lens; unsigned codes; @@ -50,11 +50,11 @@ unsigned short FAR *work; unsigned fill; /* index for replicating entries */ unsigned low; /* low bits for current root entry */ unsigned mask; /* mask for low root bits */ - code this; /* table entry for duplication */ + code here; /* table entry for duplication */ code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -62,7 +62,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 201, 196}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -115,15 +115,15 @@ unsigned short FAR *work; if (count[max] != 0) break; if (root > max) root = max; if (max == 0) { /* no symbols to code at all */ - this.op = (unsigned char)64; /* invalid code marker */ - this.bits = (unsigned char)1; - this.val = (unsigned short)0; - *(*table)++ = this; /* make a table to force an error */ - *(*table)++ = this; + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)1; + here.val = (unsigned short)0; + *(*table)++ = here; /* make a table to force an error */ + *(*table)++ = here; *bits = 1; return 0; /* no symbols, but wait for decoding to report error */ } - for (min = 1; min <= MAXBITS; min++) + for (min = 1; min < max; min++) if (count[min] != 0) break; if (root < min) root = min; @@ -166,11 +166,10 @@ unsigned short FAR *work; entered in the tables. used keeps track of how many table entries have been allocated from the - provided *table space. It is checked when a LENS table is being made - against the space in *table, ENOUGH, minus the maximum space needed by - the worst case distance code, MAXD. This should never happen, but the - sufficiency of ENOUGH has not been proven exhaustively, hence the check. - This assumes that when type == LENS, bits == 9. + provided *table space. It is checked for LENS and DIST tables against + the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in + the initial root table size constants. See the comments in inftrees.h + for more information. sym increments through all symbols, and the loop terminates when all codes of length max, i.e. all codes, have been processed. This @@ -182,19 +181,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; - default: /* DISTS */ + default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -209,24 +206,25 @@ unsigned short FAR *work; mask = used - 1; /* mask for comparing low */ /* check available table space */ - if (type == LENS && used >= ENOUGH - MAXD) + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) return 1; /* process all codes and make table entries */ for (;;) { /* create table entry */ - this.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { - this.op = (unsigned char)0; - this.val = work[sym]; + here.bits = (unsigned char)(len - drop); + if (work[sym] + 1U < match) { + here.op = (unsigned char)0; + here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - this.op = (unsigned char)(extra[work[sym]]); - this.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { - this.op = (unsigned char)(32 + 64); /* end of block */ - this.val = 0; + here.op = (unsigned char)(32 + 64); /* end of block */ + here.val = 0; } /* replicate for those indices with low len bits equal to huff */ @@ -235,7 +233,7 @@ unsigned short FAR *work; min = fill; /* save offset to next table */ do { fill -= incr; - next[(huff >> drop) + fill] = this; + next[(huff >> drop) + fill] = here; } while (fill != 0); /* backwards increment the len-bit code huff */ @@ -277,7 +275,8 @@ unsigned short FAR *work; /* check for enough space */ used += 1U << curr; - if (type == LENS && used >= ENOUGH - MAXD) + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) return 1; /* point entry in root table to sub-table */ @@ -288,38 +287,14 @@ unsigned short FAR *work; } } - /* - Fill in rest of table for incomplete codes. This loop is similar to the - loop above in incrementing huff for table indices. It is assumed that - len is equal to curr + drop, so there is no loop needed to increment - through high index bits. When the current sub-table is filled, the loop - drops back to the root table to fill in any remaining entries there. - */ - this.op = (unsigned char)64; /* invalid code marker */ - this.bits = (unsigned char)(len - drop); - this.val = (unsigned short)0; - while (huff != 0) { - /* when done with sub-table, drop back to root table */ - if (drop != 0 && (huff & mask) != low) { - drop = 0; - len = root; - next = *table; - this.bits = (unsigned char)len; - } - - /* put invalid code marker in table */ - next[huff >> drop] = this; - - /* backwards increment the len-bit code huff */ - incr = 1U << (len - 1); - while (huff & incr) - incr >>= 1; - if (incr != 0) { - huff &= incr - 1; - huff += incr; - } - else - huff = 0; + /* fill in remaining table entry if code is incomplete (guaranteed to have + at most one remaining entry, since if the code is incomplete, the + maximum code length that was allowed to get this far is one bit) */ + if (huff != 0) { + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)(len - drop); + here.val = (unsigned short)0; + next[huff] = here; } /* set return parameters */ diff --git a/zlib/inftrees.h b/zlib/inftrees.h index b1104c87e76..baa53a0b1a1 100644 --- a/zlib/inftrees.h +++ b/zlib/inftrees.h @@ -1,5 +1,5 @@ /* inftrees.h -- header to use inftrees.c - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2005, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -35,21 +35,28 @@ typedef struct { 01000000 - invalid code */ -/* Maximum size of dynamic tree. The maximum found in a long but non- - exhaustive search was 1444 code structures (852 for length/literals - and 592 for distances, the latter actually the result of an - exhaustive search). The true maximum is not known, but the value - below is more than safe. */ -#define ENOUGH 2048 -#define MAXD 592 +/* Maximum size of the dynamic table. The maximum number of code structures is + 1444, which is the sum of 852 for literal/length codes and 592 for distance + codes. These values were found by exhaustive searches using the program + examples/enough.c found in the zlib distribtution. The arguments to that + program are the number of symbols, the initial root table size, and the + maximum bit length of a code. "enough 286 9 15" for literal/length codes + returns returns 852, and "enough 30 6 15" for distance codes returns 592. + The initial root table size (9 or 6) is found in the fifth argument of the + inflate_table() calls in inflate.c and infback.c. If the root table size is + changed, then these maximum sizes would be need to be recalculated and + updated. */ +#define ENOUGH_LENS 852 +#define ENOUGH_DISTS 592 +#define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) -/* Type of code to build for inftable() */ +/* Type of code to build for inflate_table() */ typedef enum { CODES, LENS, DISTS } codetype; -extern int inflate_table OF((codetype type, unsigned short FAR *lens, +int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, unsigned codes, code FAR * FAR *table, unsigned FAR *bits, unsigned short FAR *work)); diff --git a/zlib/make_vms.com b/zlib/make_vms.com new file mode 100644 index 00000000000..65e9d0cbc8e --- /dev/null +++ b/zlib/make_vms.com @@ -0,0 +1,867 @@ +$! make libz under VMS written by +$! Martin P.J. Zinser +$! +$! In case of problems with the install you might contact me at +$! zinser@zinser.no-ip.info(preferred) or +$! martin.zinser@eurexchange.com (work) +$! +$! Make procedure history for Zlib +$! +$!------------------------------------------------------------------------------ +$! Version history +$! 0.01 20060120 First version to receive a number +$! 0.02 20061008 Adapt to new Makefile.in +$! 0.03 20091224 Add support for large file check +$! 0.04 20100110 Add new gzclose, gzlib, gzread, gzwrite +$! 0.05 20100221 Exchange zlibdefs.h by zconf.h.in +$! 0.06 20120111 Fix missing amiss_err, update zconf_h.in, fix new exmples +$! subdir path, update module search in makefile.in +$! 0.07 20120115 Triggered by work done by Alexey Chupahin completly redesigned +$! shared image creation +$! 0.08 20120219 Make it work on VAX again, pre-load missing symbols to shared +$! image +$! 0.09 20120305 SMS. P1 sets builder ("MMK", "MMS", " " (built-in)). +$! "" -> automatic, preference: MMK, MMS, built-in. +$! +$ on error then goto err_exit +$! +$ true = 1 +$ false = 0 +$ tmpnam = "temp_" + f$getjpi("","pid") +$ tt = tmpnam + ".txt" +$ tc = tmpnam + ".c" +$ th = tmpnam + ".h" +$ define/nolog tconfig 'th' +$ its_decc = false +$ its_vaxc = false +$ its_gnuc = false +$ s_case = False +$! +$! Setup variables holding "config" information +$! +$ Make = "''p1'" +$ name = "Zlib" +$ version = "?.?.?" +$ v_string = "ZLIB_VERSION" +$ v_file = "zlib.h" +$ ccopt = "/include = []" +$ lopts = "" +$ dnsrl = "" +$ aconf_in_file = "zconf.h.in#zconf.h_in#zconf_h.in" +$ conf_check_string = "" +$ linkonly = false +$ optfile = name + ".opt" +$ mapfile = name + ".map" +$ libdefs = "" +$ vax = f$getsyi("HW_MODEL").lt.1024 +$ axp = f$getsyi("HW_MODEL").ge.1024 .and. f$getsyi("HW_MODEL").lt.4096 +$ ia64 = f$getsyi("HW_MODEL").ge.4096 +$! +$! 2012-03-05 SMS. +$! Why is this needed? And if it is needed, why not simply ".not. vax"? +$! +$!!! if axp .or. ia64 then set proc/parse=extended +$! +$ whoami = f$parse(f$environment("Procedure"),,,,"NO_CONCEAL") +$ mydef = F$parse(whoami,,,"DEVICE") +$ mydir = f$parse(whoami,,,"DIRECTORY") - "][" +$ myproc = f$parse(whoami,,,"Name") + f$parse(whoami,,,"type") +$! +$! Check for MMK/MMS +$! +$ if (Make .eqs. "") +$ then +$ If F$Search ("Sys$System:MMS.EXE") .nes. "" Then Make = "MMS" +$ If F$Type (MMK) .eqs. "STRING" Then Make = "MMK" +$ else +$ Make = f$edit( Make, "trim") +$ endif +$! +$ gosub find_version +$! +$ open/write topt tmp.opt +$ open/write optf 'optfile' +$! +$ gosub check_opts +$! +$! Look for the compiler used +$! +$ gosub check_compiler +$ close topt +$ close optf +$! +$ if its_decc +$ then +$ ccopt = "/prefix=all" + ccopt +$ if f$trnlnm("SYS") .eqs. "" +$ then +$ if axp +$ then +$ define sys sys$library: +$ else +$ ccopt = "/decc" + ccopt +$ define sys decc$library_include: +$ endif +$ endif +$! +$! 2012-03-05 SMS. +$! Why /NAMES = AS_IS? Why not simply ".not. vax"? And why not on VAX? +$! +$ if axp .or. ia64 +$ then +$ ccopt = ccopt + "/name=as_is/opt=(inline=speed)" +$ s_case = true +$ endif +$ endif +$ if its_vaxc .or. its_gnuc +$ then +$ if f$trnlnm("SYS").eqs."" then define sys sys$library: +$ endif +$! +$! Build a fake configure input header +$! +$ open/write conf_hin config.hin +$ write conf_hin "#undef _LARGEFILE64_SOURCE" +$ close conf_hin +$! +$! +$ i = 0 +$FIND_ACONF: +$ fname = f$element(i,"#",aconf_in_file) +$ if fname .eqs. "#" then goto AMISS_ERR +$ if f$search(fname) .eqs. "" +$ then +$ i = i + 1 +$ goto find_aconf +$ endif +$ open/read/err=aconf_err aconf_in 'fname' +$ open/write aconf zconf.h +$ACONF_LOOP: +$ read/end_of_file=aconf_exit aconf_in line +$ work = f$edit(line, "compress,trim") +$ if f$extract(0,6,work) .nes. "#undef" +$ then +$ if f$extract(0,12,work) .nes. "#cmakedefine" +$ then +$ write aconf line +$ endif +$ else +$ cdef = f$element(1," ",work) +$ gosub check_config +$ endif +$ goto aconf_loop +$ACONF_EXIT: +$ write aconf "" +$ write aconf "/* VMS specifics added by make_vms.com: */" +$ write aconf "#define VMS 1" +$ write aconf "#include " +$ write aconf "#include " +$ write aconf "#ifdef _LARGEFILE" +$ write aconf "# define off64_t __off64_t" +$ write aconf "# define fopen64 fopen" +$ write aconf "# define fseeko64 fseeko" +$ write aconf "# define lseek64 lseek" +$ write aconf "# define ftello64 ftell" +$ write aconf "#endif" +$ write aconf "#if !defined( __VAX) && (__CRTL_VER >= 70312000)" +$ write aconf "# define HAVE_VSNPRINTF" +$ write aconf "#endif" +$ close aconf_in +$ close aconf +$ if f$search("''th'") .nes. "" then delete 'th';* +$! Build the thing plain or with mms +$! +$ write sys$output "Compiling Zlib sources ..." +$ if make.eqs."" +$ then +$ if (f$search( "example.obj;*") .nes. "") then delete example.obj;* +$ if (f$search( "minigzip.obj;*") .nes. "") then delete minigzip.obj;* +$ CALL MAKE adler32.OBJ "CC ''CCOPT' adler32" - + adler32.c zlib.h zconf.h +$ CALL MAKE compress.OBJ "CC ''CCOPT' compress" - + compress.c zlib.h zconf.h +$ CALL MAKE crc32.OBJ "CC ''CCOPT' crc32" - + crc32.c zlib.h zconf.h +$ CALL MAKE deflate.OBJ "CC ''CCOPT' deflate" - + deflate.c deflate.h zutil.h zlib.h zconf.h +$ CALL MAKE gzclose.OBJ "CC ''CCOPT' gzclose" - + gzclose.c zutil.h zlib.h zconf.h +$ CALL MAKE gzlib.OBJ "CC ''CCOPT' gzlib" - + gzlib.c zutil.h zlib.h zconf.h +$ CALL MAKE gzread.OBJ "CC ''CCOPT' gzread" - + gzread.c zutil.h zlib.h zconf.h +$ CALL MAKE gzwrite.OBJ "CC ''CCOPT' gzwrite" - + gzwrite.c zutil.h zlib.h zconf.h +$ CALL MAKE infback.OBJ "CC ''CCOPT' infback" - + infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h +$ CALL MAKE inffast.OBJ "CC ''CCOPT' inffast" - + inffast.c zutil.h zlib.h zconf.h inffast.h +$ CALL MAKE inflate.OBJ "CC ''CCOPT' inflate" - + inflate.c zutil.h zlib.h zconf.h infblock.h +$ CALL MAKE inftrees.OBJ "CC ''CCOPT' inftrees" - + inftrees.c zutil.h zlib.h zconf.h inftrees.h +$ CALL MAKE trees.OBJ "CC ''CCOPT' trees" - + trees.c deflate.h zutil.h zlib.h zconf.h +$ CALL MAKE uncompr.OBJ "CC ''CCOPT' uncompr" - + uncompr.c zlib.h zconf.h +$ CALL MAKE zutil.OBJ "CC ''CCOPT' zutil" - + zutil.c zutil.h zlib.h zconf.h +$ write sys$output "Building Zlib ..." +$ CALL MAKE libz.OLB "lib/crea libz.olb *.obj" *.OBJ +$ write sys$output "Building example..." +$ CALL MAKE example.OBJ "CC ''CCOPT' [.test]example" - + [.test]example.c zlib.h zconf.h +$ call make example.exe "LINK example,libz.olb/lib" example.obj libz.olb +$ write sys$output "Building minigzip..." +$ CALL MAKE minigzip.OBJ "CC ''CCOPT' [.test]minigzip" - + [.test]minigzip.c zlib.h zconf.h +$ call make minigzip.exe - + "LINK minigzip,libz.olb/lib" - + minigzip.obj libz.olb +$ else +$ gosub crea_mms +$ write sys$output "Make ''name' ''version' with ''Make' " +$ 'make' +$ endif +$! +$! Create shareable image +$! +$ gosub crea_olist +$ write sys$output "Creating libzshr.exe" +$ call map_2_shopt 'mapfile' 'optfile' +$ LINK_'lopts'/SHARE=libzshr.exe modules.opt/opt,'optfile'/opt +$ write sys$output "Zlib build completed" +$ delete/nolog tmp.opt;* +$ exit +$AMISS_ERR: +$ write sys$output "No source for config.hin found." +$ write sys$output "Tried any of ''aconf_in_file'" +$ goto err_exit +$CC_ERR: +$ write sys$output "C compiler required to build ''name'" +$ goto err_exit +$ERR_EXIT: +$ set message/facil/ident/sever/text +$ close/nolog optf +$ close/nolog topt +$ close/nolog aconf_in +$ close/nolog aconf +$ close/nolog out +$ close/nolog min +$ close/nolog mod +$ close/nolog h_in +$ write sys$output "Exiting..." +$ exit 2 +$! +$! +$MAKE: SUBROUTINE !SUBROUTINE TO CHECK DEPENDENCIES +$ V = 'F$Verify(0) +$! P1 = What we are trying to make +$! P2 = Command to make it +$! P3 - P8 What it depends on +$ +$ If F$Search(P1) .Eqs. "" Then Goto Makeit +$ Time = F$CvTime(F$File(P1,"RDT")) +$arg=3 +$Loop: +$ Argument = P'arg +$ If Argument .Eqs. "" Then Goto Exit +$ El=0 +$Loop2: +$ File = F$Element(El," ",Argument) +$ If File .Eqs. " " Then Goto Endl +$ AFile = "" +$Loop3: +$ OFile = AFile +$ AFile = F$Search(File) +$ If AFile .Eqs. "" .Or. AFile .Eqs. OFile Then Goto NextEl +$ If F$CvTime(F$File(AFile,"RDT")) .Ges. Time Then Goto Makeit +$ Goto Loop3 +$NextEL: +$ El = El + 1 +$ Goto Loop2 +$EndL: +$ arg=arg+1 +$ If arg .Le. 8 Then Goto Loop +$ Goto Exit +$ +$Makeit: +$ VV=F$VERIFY(0) +$ write sys$output P2 +$ 'P2 +$ VV='F$Verify(VV) +$Exit: +$ If V Then Set Verify +$ENDSUBROUTINE +$!------------------------------------------------------------------------------ +$! +$! Check command line options and set symbols accordingly +$! +$!------------------------------------------------------------------------------ +$! Version history +$! 0.01 20041206 First version to receive a number +$! 0.02 20060126 Add new "HELP" target +$ CHECK_OPTS: +$ i = 1 +$ OPT_LOOP: +$ if i .lt. 9 +$ then +$ cparm = f$edit(p'i',"upcase") +$! +$! Check if parameter actually contains something +$! +$ if f$edit(cparm,"trim") .nes. "" +$ then +$ if cparm .eqs. "DEBUG" +$ then +$ ccopt = ccopt + "/noopt/deb" +$ lopts = lopts + "/deb" +$ endif +$ if f$locate("CCOPT=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ ccopt = ccopt + f$extract(start,len,cparm) +$ if f$locate("AS_IS",f$edit(ccopt,"UPCASE")) .lt. f$length(ccopt) - + then s_case = true +$ endif +$ if cparm .eqs. "LINK" then linkonly = true +$ if f$locate("LOPTS=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ lopts = lopts + f$extract(start,len,cparm) +$ endif +$ if f$locate("CC=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ cc_com = f$extract(start,len,cparm) + if (cc_com .nes. "DECC") .and. - + (cc_com .nes. "VAXC") .and. - + (cc_com .nes. "GNUC") +$ then +$ write sys$output "Unsupported compiler choice ''cc_com' ignored" +$ write sys$output "Use DECC, VAXC, or GNUC instead" +$ else +$ if cc_com .eqs. "DECC" then its_decc = true +$ if cc_com .eqs. "VAXC" then its_vaxc = true +$ if cc_com .eqs. "GNUC" then its_gnuc = true +$ endif +$ endif +$ if f$locate("MAKE=",cparm) .lt. f$length(cparm) +$ then +$ start = f$locate("=",cparm) + 1 +$ len = f$length(cparm) - start +$ mmks = f$extract(start,len,cparm) +$ if (mmks .eqs. "MMK") .or. (mmks .eqs. "MMS") +$ then +$ make = mmks +$ else +$ write sys$output "Unsupported make choice ''mmks' ignored" +$ write sys$output "Use MMK or MMS instead" +$ endif +$ endif +$ if cparm .eqs. "HELP" then gosub bhelp +$ endif +$ i = i + 1 +$ goto opt_loop +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Look for the compiler used +$! +$! Version history +$! 0.01 20040223 First version to receive a number +$! 0.02 20040229 Save/set value of decc$no_rooted_search_lists +$! 0.03 20060202 Extend handling of GNU C +$! 0.04 20090402 Compaq -> hp +$CHECK_COMPILER: +$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) +$ then +$ its_decc = (f$search("SYS$SYSTEM:DECC$COMPILER.EXE") .nes. "") +$ its_vaxc = .not. its_decc .and. (F$Search("SYS$System:VAXC.Exe") .nes. "") +$ its_gnuc = .not. (its_decc .or. its_vaxc) .and. (f$trnlnm("gnu_cc") .nes. "") +$ endif +$! +$! Exit if no compiler available +$! +$ if (.not. (its_decc .or. its_vaxc .or. its_gnuc)) +$ then goto CC_ERR +$ else +$ if its_decc +$ then +$ write sys$output "CC compiler check ... hp C" +$ if f$trnlnm("decc$no_rooted_search_lists") .nes. "" +$ then +$ dnrsl = f$trnlnm("decc$no_rooted_search_lists") +$ endif +$ define/nolog decc$no_rooted_search_lists 1 +$ else +$ if its_vaxc then write sys$output "CC compiler check ... VAX C" +$ if its_gnuc +$ then +$ write sys$output "CC compiler check ... GNU C" +$ if f$trnlnm(topt) then write topt "gnu_cc:[000000]gcclib.olb/lib" +$ if f$trnlnm(optf) then write optf "gnu_cc:[000000]gcclib.olb/lib" +$ cc = "gcc" +$ endif +$ if f$trnlnm(topt) then write topt "sys$share:vaxcrtl.exe/share" +$ if f$trnlnm(optf) then write optf "sys$share:vaxcrtl.exe/share" +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! If MMS/MMK are available dump out the descrip.mms if required +$! +$CREA_MMS: +$ write sys$output "Creating descrip.mms..." +$ create descrip.mms +$ open/append out descrip.mms +$ copy sys$input: out +$ deck +# descrip.mms: MMS description file for building zlib on VMS +# written by Martin P.J. Zinser +# + +OBJS = adler32.obj, compress.obj, crc32.obj, gzclose.obj, gzlib.obj\ + gzread.obj, gzwrite.obj, uncompr.obj, infback.obj\ + deflate.obj, trees.obj, zutil.obj, inflate.obj, \ + inftrees.obj, inffast.obj + +$ eod +$ write out "CFLAGS=", ccopt +$ write out "LOPTS=", lopts +$ write out "all : example.exe minigzip.exe libz.olb" +$ copy sys$input: out +$ deck + @ write sys$output " Example applications available" + +libz.olb : libz.olb($(OBJS)) + @ write sys$output " libz available" + +example.exe : example.obj libz.olb + link $(LOPTS) example,libz.olb/lib + +minigzip.exe : minigzip.obj libz.olb + link $(LOPTS) minigzip,libz.olb/lib + +clean : + delete *.obj;*,libz.olb;*,*.opt;*,*.exe;* + + +# Other dependencies. +adler32.obj : adler32.c zutil.h zlib.h zconf.h +compress.obj : compress.c zlib.h zconf.h +crc32.obj : crc32.c zutil.h zlib.h zconf.h +deflate.obj : deflate.c deflate.h zutil.h zlib.h zconf.h +example.obj : [.test]example.c zlib.h zconf.h +gzclose.obj : gzclose.c zutil.h zlib.h zconf.h +gzlib.obj : gzlib.c zutil.h zlib.h zconf.h +gzread.obj : gzread.c zutil.h zlib.h zconf.h +gzwrite.obj : gzwrite.c zutil.h zlib.h zconf.h +inffast.obj : inffast.c zutil.h zlib.h zconf.h inftrees.h inffast.h +inflate.obj : inflate.c zutil.h zlib.h zconf.h +inftrees.obj : inftrees.c zutil.h zlib.h zconf.h inftrees.h +minigzip.obj : [.test]minigzip.c zlib.h zconf.h +trees.obj : trees.c deflate.h zutil.h zlib.h zconf.h +uncompr.obj : uncompr.c zlib.h zconf.h +zutil.obj : zutil.c zutil.h zlib.h zconf.h +infback.obj : infback.c zutil.h inftrees.h inflate.h inffast.h inffixed.h +$ eod +$ close out +$ return +$!------------------------------------------------------------------------------ +$! +$! Read list of core library sources from makefile.in and create options +$! needed to build shareable image +$! +$CREA_OLIST: +$ open/read min makefile.in +$ open/write mod modules.opt +$ src_check_list = "OBJZ =#OBJG =" +$MRLOOP: +$ read/end=mrdone min rec +$ i = 0 +$SRC_CHECK_LOOP: +$ src_check = f$element(i, "#", src_check_list) +$ i = i+1 +$ if src_check .eqs. "#" then goto mrloop +$ if (f$extract(0,6,rec) .nes. src_check) then goto src_check_loop +$ rec = rec - src_check +$ gosub extra_filnam +$ if (f$element(1,"\",rec) .eqs. "\") then goto mrloop +$MRSLOOP: +$ read/end=mrdone min rec +$ gosub extra_filnam +$ if (f$element(1,"\",rec) .nes. "\") then goto mrsloop +$MRDONE: +$ close min +$ close mod +$ return +$!------------------------------------------------------------------------------ +$! +$! Take record extracted in crea_olist and split it into single filenames +$! +$EXTRA_FILNAM: +$ myrec = f$edit(rec - "\", "trim,compress") +$ i = 0 +$FELOOP: +$ srcfil = f$element(i," ", myrec) +$ if (srcfil .nes. " ") +$ then +$ write mod f$parse(srcfil,,,"NAME"), ".obj" +$ i = i + 1 +$ goto feloop +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Find current Zlib version number +$! +$FIND_VERSION: +$ open/read h_in 'v_file' +$hloop: +$ read/end=hdone h_in rec +$ rec = f$edit(rec,"TRIM") +$ if (f$extract(0,1,rec) .nes. "#") then goto hloop +$ rec = f$edit(rec - "#", "TRIM") +$ if f$element(0," ",rec) .nes. "define" then goto hloop +$ if f$element(1," ",rec) .eqs. v_string +$ then +$ version = 'f$element(2," ",rec)' +$ goto hdone +$ endif +$ goto hloop +$hdone: +$ close h_in +$ return +$!------------------------------------------------------------------------------ +$! +$CHECK_CONFIG: +$! +$ in_ldef = f$locate(cdef,libdefs) +$ if (in_ldef .lt. f$length(libdefs)) +$ then +$ write aconf "#define ''cdef' 1" +$ libdefs = f$extract(0,in_ldef,libdefs) + - + f$extract(in_ldef + f$length(cdef) + 1, - + f$length(libdefs) - in_ldef - f$length(cdef) - 1, - + libdefs) +$ else +$ if (f$type('cdef') .eqs. "INTEGER") +$ then +$ write aconf "#define ''cdef' ", 'cdef' +$ else +$ if (f$type('cdef') .eqs. "STRING") +$ then +$ write aconf "#define ''cdef' ", """", '''cdef'', """" +$ else +$ gosub check_cc_def +$ endif +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Check if this is a define relating to the properties of the C/C++ +$! compiler +$! +$ CHECK_CC_DEF: +$ if (cdef .eqs. "_LARGEFILE64_SOURCE") +$ then +$ copy sys$input: 'tc' +$ deck +#include "tconfig" +#define _LARGEFILE +#include + +int main(){ +FILE *fp; + fp = fopen("temp.txt","r"); + fseeko(fp,1,SEEK_SET); + fclose(fp); +} + +$ eod +$ test_inv = false +$ comm_h = false +$ gosub cc_prop_check +$ return +$ endif +$ write aconf "/* ", line, " */" +$ return +$!------------------------------------------------------------------------------ +$! +$! Check for properties of C/C++ compiler +$! +$! Version history +$! 0.01 20031020 First version to receive a number +$! 0.02 20031022 Added logic for defines with value +$! 0.03 20040309 Make sure local config file gets not deleted +$! 0.04 20041230 Also write include for configure run +$! 0.05 20050103 Add processing of "comment defines" +$CC_PROP_CHECK: +$ cc_prop = true +$ is_need = false +$ is_need = (f$extract(0,4,cdef) .eqs. "NEED") .or. (test_inv .eq. true) +$ if f$search(th) .eqs. "" then create 'th' +$ set message/nofac/noident/nosever/notext +$ on error then continue +$ cc 'tmpnam' +$ if .not. ($status) then cc_prop = false +$ on error then continue +$! The headers might lie about the capabilities of the RTL +$ link 'tmpnam',tmp.opt/opt +$ if .not. ($status) then cc_prop = false +$ set message/fac/ident/sever/text +$ on error then goto err_exit +$ delete/nolog 'tmpnam'.*;*/exclude='th' +$ if (cc_prop .and. .not. is_need) .or. - + (.not. cc_prop .and. is_need) +$ then +$ write sys$output "Checking for ''cdef'... yes" +$ if f$type('cdef_val'_yes) .nes. "" +$ then +$ if f$type('cdef_val'_yes) .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_yes) +$ if f$type('cdef_val'_yes) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_yes) +$ else +$ call write_config f$fao("#define !AS 1",cdef) +$ endif +$ if (cdef .eqs. "HAVE_FSEEKO") .or. (cdef .eqs. "_LARGE_FILES") .or. - + (cdef .eqs. "_LARGEFILE64_SOURCE") then - + call write_config f$string("#define _LARGEFILE 1") +$ else +$ write sys$output "Checking for ''cdef'... no" +$ if (comm_h) +$ then + call write_config f$fao("/* !AS */",line) +$ else +$ if f$type('cdef_val'_no) .nes. "" +$ then +$ if f$type('cdef_val'_no) .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,'cdef_val'_no) +$ if f$type('cdef_val'_no) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,'cdef_val'_no) +$ else +$ call write_config f$fao("#undef !AS",cdef) +$ endif +$ endif +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Check for properties of C/C++ compiler with multiple result values +$! +$! Version history +$! 0.01 20040127 First version +$! 0.02 20050103 Reconcile changes from cc_prop up to version 0.05 +$CC_MPROP_CHECK: +$ cc_prop = true +$ i = 1 +$ idel = 1 +$ MT_LOOP: +$ if f$type(result_'i') .eqs. "STRING" +$ then +$ set message/nofac/noident/nosever/notext +$ on error then continue +$ cc 'tmpnam'_'i' +$ if .not. ($status) then cc_prop = false +$ on error then continue +$! The headers might lie about the capabilities of the RTL +$ link 'tmpnam'_'i',tmp.opt/opt +$ if .not. ($status) then cc_prop = false +$ set message/fac/ident/sever/text +$ on error then goto err_exit +$ delete/nolog 'tmpnam'_'i'.*;* +$ if (cc_prop) +$ then +$ write sys$output "Checking for ''cdef'... ", mdef_'i' +$ if f$type(mdef_'i') .eqs. "INTEGER" - + then call write_config f$fao("#define !AS !UL",cdef,mdef_'i') +$ if f$type('cdef_val'_yes) .eqs. "STRING" - + then call write_config f$fao("#define !AS !AS",cdef,mdef_'i') +$ goto msym_clean +$ else +$ i = i + 1 +$ goto mt_loop +$ endif +$ endif +$ write sys$output "Checking for ''cdef'... no" +$ call write_config f$fao("#undef !AS",cdef) +$ MSYM_CLEAN: +$ if (idel .le. msym_max) +$ then +$ delete/sym mdef_'idel' +$ idel = idel + 1 +$ goto msym_clean +$ endif +$ return +$!------------------------------------------------------------------------------ +$! +$! Write configuration to both permanent and temporary config file +$! +$! Version history +$! 0.01 20031029 First version to receive a number +$! +$WRITE_CONFIG: SUBROUTINE +$ write aconf 'p1' +$ open/append confh 'th' +$ write confh 'p1' +$ close confh +$ENDSUBROUTINE +$!------------------------------------------------------------------------------ +$! +$! Analyze the project map file and create the symbol vector for a shareable +$! image from it +$! +$! Version history +$! 0.01 20120128 First version +$! 0.02 20120226 Add pre-load logic +$! +$ MAP_2_SHOPT: Subroutine +$! +$ SAY := "WRITE_ SYS$OUTPUT" +$! +$ IF F$SEARCH("''P1'") .EQS. "" +$ THEN +$ SAY "MAP_2_SHOPT-E-NOSUCHFILE: Error, inputfile ''p1' not available" +$ goto exit_m2s +$ ENDIF +$ IF "''P2'" .EQS. "" +$ THEN +$ SAY "MAP_2_SHOPT: Error, no output file provided" +$ goto exit_m2s +$ ENDIF +$! +$ module1 = "deflate#deflateEnd#deflateInit_#deflateParams#deflateSetDictionary" +$ module2 = "gzclose#gzerror#gzgetc#gzgets#gzopen#gzprintf#gzputc#gzputs#gzread" +$ module3 = "gzseek#gztell#inflate#inflateEnd#inflateInit_#inflateSetDictionary" +$ module4 = "inflateSync#uncompress#zlibVersion#compress" +$ open/read map 'p1 +$ if axp .or. ia64 +$ then +$ open/write aopt a.opt +$ open/write bopt b.opt +$ write aopt " CASE_SENSITIVE=YES" +$ write bopt "SYMBOL_VECTOR= (-" +$ mod_sym_num = 1 +$ MOD_SYM_LOOP: +$ if f$type(module'mod_sym_num') .nes. "" +$ then +$ mod_in = 0 +$ MOD_SYM_IN: +$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') +$ if shared_proc .nes. "#" +$ then +$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- + f$edit(shared_proc,"upcase"),shared_proc) +$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) +$ mod_in = mod_in + 1 +$ goto mod_sym_in +$ endif +$ mod_sym_num = mod_sym_num + 1 +$ goto mod_sym_loop +$ endif +$MAP_LOOP: +$ read/end=map_end map line +$ if (f$locate("{",line).lt. f$length(line)) .or. - + (f$locate("global:", line) .lt. f$length(line)) +$ then +$ proc = true +$ goto map_loop +$ endif +$ if f$locate("}",line).lt. f$length(line) then proc = false +$ if f$locate("local:", line) .lt. f$length(line) then proc = false +$ if proc +$ then +$ shared_proc = f$edit(line,"collapse") +$ chop_semi = f$locate(";", shared_proc) +$ if chop_semi .lt. f$length(shared_proc) then - + shared_proc = f$extract(0, chop_semi, shared_proc) +$ write aopt f$fao(" symbol_vector=(!AS/!AS=PROCEDURE)",- + f$edit(shared_proc,"upcase"),shared_proc) +$ write bopt f$fao("!AS=PROCEDURE,-",shared_proc) +$ endif +$ goto map_loop +$MAP_END: +$ close/nolog aopt +$ close/nolog bopt +$ open/append libopt 'p2' +$ open/read aopt a.opt +$ open/read bopt b.opt +$ALOOP: +$ read/end=aloop_end aopt line +$ write libopt line +$ goto aloop +$ALOOP_END: +$ close/nolog aopt +$ sv = "" +$BLOOP: +$ read/end=bloop_end bopt svn +$ if (svn.nes."") +$ then +$ if (sv.nes."") then write libopt sv +$ sv = svn +$ endif +$ goto bloop +$BLOOP_END: +$ write libopt f$extract(0,f$length(sv)-2,sv), "-" +$ write libopt ")" +$ close/nolog bopt +$ delete/nolog/noconf a.opt;*,b.opt;* +$ else +$ if vax +$ then +$ open/append libopt 'p2' +$ mod_sym_num = 1 +$ VMOD_SYM_LOOP: +$ if f$type(module'mod_sym_num') .nes. "" +$ then +$ mod_in = 0 +$ VMOD_SYM_IN: +$ shared_proc = f$element(mod_in, "#", module'mod_sym_num') +$ if shared_proc .nes. "#" +$ then +$ write libopt f$fao("UNIVERSAL=!AS",- + f$edit(shared_proc,"upcase")) +$ mod_in = mod_in + 1 +$ goto vmod_sym_in +$ endif +$ mod_sym_num = mod_sym_num + 1 +$ goto vmod_sym_loop +$ endif +$VMAP_LOOP: +$ read/end=vmap_end map line +$ if (f$locate("{",line).lt. f$length(line)) .or. - + (f$locate("global:", line) .lt. f$length(line)) +$ then +$ proc = true +$ goto vmap_loop +$ endif +$ if f$locate("}",line).lt. f$length(line) then proc = false +$ if f$locate("local:", line) .lt. f$length(line) then proc = false +$ if proc +$ then +$ shared_proc = f$edit(line,"collapse") +$ chop_semi = f$locate(";", shared_proc) +$ if chop_semi .lt. f$length(shared_proc) then - + shared_proc = f$extract(0, chop_semi, shared_proc) +$ write libopt f$fao("UNIVERSAL=!AS",- + f$edit(shared_proc,"upcase")) +$ endif +$ goto vmap_loop +$VMAP_END: +$ else +$ write sys$output "Unknown Architecture (Not VAX, AXP, or IA64)" +$ write sys$output "No options file created" +$ endif +$ endif +$ EXIT_M2S: +$ close/nolog map +$ close/nolog libopt +$ endsubroutine diff --git a/zlib/msdos/Makefile.bor b/zlib/msdos/Makefile.bor new file mode 100644 index 00000000000..3d12a2c2527 --- /dev/null +++ b/zlib/msdos/Makefile.bor @@ -0,0 +1,115 @@ +# Makefile for zlib +# Borland C++ +# Last updated: 15-Mar-2003 + +# To use, do "make -fmakefile.bor" +# To compile in small model, set below: MODEL=s + +# WARNING: the small model is supported but only for small values of +# MAX_WBITS and MAX_MEM_LEVEL. For example: +# -DMAX_WBITS=11 -DDEF_WBITS=11 -DMAX_MEM_LEVEL=3 +# If you wish to reduce the memory requirements (default 256K for big +# objects plus a few K), you can add to the LOC macro below: +# -DMAX_MEM_LEVEL=7 -DMAX_WBITS=14 +# See zconf.h for details about the memory requirements. + +# ------------ Turbo C++, Borland C++ ------------ + +# Optional nonstandard preprocessor flags (e.g. -DMAX_MEM_LEVEL=7) +# should be added to the environment via "set LOCAL_ZLIB=-DFOO" or added +# to the declaration of LOC here: +LOC = $(LOCAL_ZLIB) + +# type for CPU required: 0: 8086, 1: 80186, 2: 80286, 3: 80386, etc. +CPU_TYP = 0 + +# memory model: one of s, m, c, l (small, medium, compact, large) +MODEL=l + +# replace bcc with tcc for Turbo C++ 1.0, with bcc32 for the 32 bit version +CC=bcc +LD=bcc +AR=tlib + +# compiler flags +# replace "-O2" by "-O -G -a -d" for Turbo C++ 1.0 +CFLAGS=-O2 -Z -m$(MODEL) $(LOC) + +LDFLAGS=-m$(MODEL) -f- + + +# variables +ZLIB_LIB = zlib_$(MODEL).lib + +OBJ1 = adler32.obj compress.obj crc32.obj deflate.obj gzclose.obj gzlib.obj gzread.obj +OBJ2 = gzwrite.obj infback.obj inffast.obj inflate.obj inftrees.obj trees.obj uncompr.obj zutil.obj +OBJP1 = +adler32.obj+compress.obj+crc32.obj+deflate.obj+gzclose.obj+gzlib.obj+gzread.obj +OBJP2 = +gzwrite.obj+infback.obj+inffast.obj+inflate.obj+inftrees.obj+trees.obj+uncompr.obj+zutil.obj + + +# targets +all: $(ZLIB_LIB) example.exe minigzip.exe + +.c.obj: + $(CC) -c $(CFLAGS) $*.c + +adler32.obj: adler32.c zlib.h zconf.h + +compress.obj: compress.c zlib.h zconf.h + +crc32.obj: crc32.c zlib.h zconf.h crc32.h + +deflate.obj: deflate.c deflate.h zutil.h zlib.h zconf.h + +gzclose.obj: gzclose.c zlib.h zconf.h gzguts.h + +gzlib.obj: gzlib.c zlib.h zconf.h gzguts.h + +gzread.obj: gzread.c zlib.h zconf.h gzguts.h + +gzwrite.obj: gzwrite.c zlib.h zconf.h gzguts.h + +infback.obj: infback.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inffast.obj: inffast.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h + +inflate.obj: inflate.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inftrees.obj: inftrees.c zutil.h zlib.h zconf.h inftrees.h + +trees.obj: trees.c zutil.h zlib.h zconf.h deflate.h trees.h + +uncompr.obj: uncompr.c zlib.h zconf.h + +zutil.obj: zutil.c zutil.h zlib.h zconf.h + +example.obj: test/example.c zlib.h zconf.h + +minigzip.obj: test/minigzip.c zlib.h zconf.h + + +# the command line is cut to fit in the MS-DOS 128 byte limit: +$(ZLIB_LIB): $(OBJ1) $(OBJ2) + -del $(ZLIB_LIB) + $(AR) $(ZLIB_LIB) $(OBJP1) + $(AR) $(ZLIB_LIB) $(OBJP2) + +example.exe: example.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) example.obj $(ZLIB_LIB) + +minigzip.exe: minigzip.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) minigzip.obj $(ZLIB_LIB) + +test: example.exe minigzip.exe + example + echo hello world | minigzip | minigzip -d + +clean: + -del *.obj + -del *.lib + -del *.exe + -del zlib_*.bak + -del foo.gz diff --git a/zlib/msdos/Makefile.dj2 b/zlib/msdos/Makefile.dj2 new file mode 100644 index 00000000000..59d2037d69c --- /dev/null +++ b/zlib/msdos/Makefile.dj2 @@ -0,0 +1,104 @@ +# Makefile for zlib. Modified for djgpp v2.0 by F. J. Donahoe, 3/15/96. +# Copyright (C) 1995-1998 Jean-loup Gailly. +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile, or to compile and test, type: +# +# make -fmakefile.dj2; make test -fmakefile.dj2 +# +# To install libz.a, zconf.h and zlib.h in the djgpp directories, type: +# +# make install -fmakefile.dj2 +# +# after first defining LIBRARY_PATH and INCLUDE_PATH in djgpp.env as +# in the sample below if the pattern of the DJGPP distribution is to +# be followed. Remember that, while 'es around <=> are ignored in +# makefiles, they are *not* in batch files or in djgpp.env. +# - - - - - +# [make] +# INCLUDE_PATH=%\>;INCLUDE_PATH%%\DJDIR%\include +# LIBRARY_PATH=%\>;LIBRARY_PATH%%\DJDIR%\lib +# BUTT=-m486 +# - - - - - +# Alternately, these variables may be defined below, overriding the values +# in djgpp.env, as +# INCLUDE_PATH=c:\usr\include +# LIBRARY_PATH=c:\usr\lib + +CC=gcc + +#CFLAGS=-MMD -O +#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#CFLAGS=-MMD -g -DZLIB_DEBUG +CFLAGS=-MMD -O3 $(BUTT) -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ + -Wstrict-prototypes -Wmissing-prototypes + +# If cp.exe is available, replace "copy /Y" with "cp -fp" . +CP=copy /Y +# If gnu install.exe is available, replace $(CP) with ginstall. +INSTALL=$(CP) +# The default value of RM is "rm -f." If "rm.exe" is found, comment out: +RM=del +LDLIBS=-L. -lz +LD=$(CC) -s -o +LDSHARED=$(CC) + +INCL=zlib.h zconf.h +LIBS=libz.a + +AR=ar rcs + +prefix=/usr/local +exec_prefix = $(prefix) + +OBJS = adler32.o compress.o crc32.o gzclose.o gzlib.o gzread.o gzwrite.o \ + uncompr.o deflate.o trees.o zutil.o inflate.o infback.o inftrees.o inffast.o + +OBJA = +# to use the asm code: make OBJA=match.o + +TEST_OBJS = example.o minigzip.o + +all: example.exe minigzip.exe + +check: test +test: all + ./example + echo hello world | .\minigzip | .\minigzip -d + +%.o : %.c + $(CC) $(CFLAGS) -c $< -o $@ + +libz.a: $(OBJS) $(OBJA) + $(AR) $@ $(OBJS) $(OBJA) + +%.exe : %.o $(LIBS) + $(LD) $@ $< $(LDLIBS) + +# INCLUDE_PATH and LIBRARY_PATH were set for [make] in djgpp.env . + +.PHONY : uninstall clean + +install: $(INCL) $(LIBS) + -@if not exist $(INCLUDE_PATH)\nul mkdir $(INCLUDE_PATH) + -@if not exist $(LIBRARY_PATH)\nul mkdir $(LIBRARY_PATH) + $(INSTALL) zlib.h $(INCLUDE_PATH) + $(INSTALL) zconf.h $(INCLUDE_PATH) + $(INSTALL) libz.a $(LIBRARY_PATH) + +uninstall: + $(RM) $(INCLUDE_PATH)\zlib.h + $(RM) $(INCLUDE_PATH)\zconf.h + $(RM) $(LIBRARY_PATH)\libz.a + +clean: + $(RM) *.d + $(RM) *.o + $(RM) *.exe + $(RM) libz.a + $(RM) foo.gz + +DEPS := $(wildcard *.d) +ifneq ($(DEPS),) +include $(DEPS) +endif diff --git a/zlib/msdos/Makefile.emx b/zlib/msdos/Makefile.emx new file mode 100644 index 00000000000..e30f67bed68 --- /dev/null +++ b/zlib/msdos/Makefile.emx @@ -0,0 +1,69 @@ +# Makefile for zlib. Modified for emx 0.9c by Chr. Spieler, 6/17/98. +# Copyright (C) 1995-1998 Jean-loup Gailly. +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile, or to compile and test, type: +# +# make -fmakefile.emx; make test -fmakefile.emx +# + +CC=gcc + +#CFLAGS=-MMD -O +#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#CFLAGS=-MMD -g -DZLIB_DEBUG +CFLAGS=-MMD -O3 $(BUTT) -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ + -Wstrict-prototypes -Wmissing-prototypes + +# If cp.exe is available, replace "copy /Y" with "cp -fp" . +CP=copy /Y +# If gnu install.exe is available, replace $(CP) with ginstall. +INSTALL=$(CP) +# The default value of RM is "rm -f." If "rm.exe" is found, comment out: +RM=del +LDLIBS=-L. -lzlib +LD=$(CC) -s -o +LDSHARED=$(CC) + +INCL=zlib.h zconf.h +LIBS=zlib.a + +AR=ar rcs + +prefix=/usr/local +exec_prefix = $(prefix) + +OBJS = adler32.o compress.o crc32.o gzclose.o gzlib.o gzread.o gzwrite.o \ + uncompr.o deflate.o trees.o zutil.o inflate.o infback.o inftrees.o inffast.o + +TEST_OBJS = example.o minigzip.o + +all: example.exe minigzip.exe + +test: all + ./example + echo hello world | .\minigzip | .\minigzip -d + +%.o : %.c + $(CC) $(CFLAGS) -c $< -o $@ + +zlib.a: $(OBJS) + $(AR) $@ $(OBJS) + +%.exe : %.o $(LIBS) + $(LD) $@ $< $(LDLIBS) + + +.PHONY : clean + +clean: + $(RM) *.d + $(RM) *.o + $(RM) *.exe + $(RM) zlib.a + $(RM) foo.gz + +DEPS := $(wildcard *.d) +ifneq ($(DEPS),) +include $(DEPS) +endif diff --git a/zlib/msdos/Makefile.msc b/zlib/msdos/Makefile.msc new file mode 100644 index 00000000000..ae8378615ec --- /dev/null +++ b/zlib/msdos/Makefile.msc @@ -0,0 +1,112 @@ +# Makefile for zlib +# Microsoft C 5.1 or later +# Last updated: 19-Mar-2003 + +# To use, do "make makefile.msc" +# To compile in small model, set below: MODEL=S + +# If you wish to reduce the memory requirements (default 256K for big +# objects plus a few K), you can add to the LOC macro below: +# -DMAX_MEM_LEVEL=7 -DMAX_WBITS=14 +# See zconf.h for details about the memory requirements. + +# ------------- Microsoft C 5.1 and later ------------- + +# Optional nonstandard preprocessor flags (e.g. -DMAX_MEM_LEVEL=7) +# should be added to the environment via "set LOCAL_ZLIB=-DFOO" or added +# to the declaration of LOC here: +LOC = $(LOCAL_ZLIB) + +# Type for CPU required: 0: 8086, 1: 80186, 2: 80286, 3: 80386, etc. +CPU_TYP = 0 + +# Memory model: one of S, M, C, L (small, medium, compact, large) +MODEL=L + +CC=cl +CFLAGS=-nologo -A$(MODEL) -G$(CPU_TYP) -W3 -Oait -Gs $(LOC) +#-Ox generates bad code with MSC 5.1 +LIB_CFLAGS=-Zl $(CFLAGS) + +LD=link +LDFLAGS=/noi/e/st:0x1500/noe/farcall/packcode +# "/farcall/packcode" are only useful for `large code' memory models +# but should be a "no-op" for small code models. + + +# variables +ZLIB_LIB = zlib_$(MODEL).lib + +OBJ1 = adler32.obj compress.obj crc32.obj deflate.obj gzclose.obj gzlib.obj gzread.obj +OBJ2 = gzwrite.obj infback.obj inffast.obj inflate.obj inftrees.obj trees.obj uncompr.obj zutil.obj + + +# targets +all: $(ZLIB_LIB) example.exe minigzip.exe + +.c.obj: + $(CC) -c $(LIB_CFLAGS) $*.c + +adler32.obj: adler32.c zlib.h zconf.h + +compress.obj: compress.c zlib.h zconf.h + +crc32.obj: crc32.c zlib.h zconf.h crc32.h + +deflate.obj: deflate.c deflate.h zutil.h zlib.h zconf.h + +gzclose.obj: gzclose.c zlib.h zconf.h gzguts.h + +gzlib.obj: gzlib.c zlib.h zconf.h gzguts.h + +gzread.obj: gzread.c zlib.h zconf.h gzguts.h + +gzwrite.obj: gzwrite.c zlib.h zconf.h gzguts.h + +infback.obj: infback.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inffast.obj: inffast.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h + +inflate.obj: inflate.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inftrees.obj: inftrees.c zutil.h zlib.h zconf.h inftrees.h + +trees.obj: trees.c zutil.h zlib.h zconf.h deflate.h trees.h + +uncompr.obj: uncompr.c zlib.h zconf.h + +zutil.obj: zutil.c zutil.h zlib.h zconf.h + +example.obj: test/example.c zlib.h zconf.h + $(CC) -c $(CFLAGS) $*.c + +minigzip.obj: test/minigzip.c zlib.h zconf.h + $(CC) -c $(CFLAGS) $*.c + + +# the command line is cut to fit in the MS-DOS 128 byte limit: +$(ZLIB_LIB): $(OBJ1) $(OBJ2) + if exist $(ZLIB_LIB) del $(ZLIB_LIB) + lib $(ZLIB_LIB) $(OBJ1); + lib $(ZLIB_LIB) $(OBJ2); + +example.exe: example.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) example.obj,,,$(ZLIB_LIB); + +minigzip.exe: minigzip.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) minigzip.obj,,,$(ZLIB_LIB); + +test: example.exe minigzip.exe + example + echo hello world | minigzip | minigzip -d + +clean: + -del *.obj + -del *.lib + -del *.exe + -del *.map + -del zlib_*.bak + -del foo.gz diff --git a/zlib/msdos/Makefile.tc b/zlib/msdos/Makefile.tc new file mode 100644 index 00000000000..5aec82a9d58 --- /dev/null +++ b/zlib/msdos/Makefile.tc @@ -0,0 +1,100 @@ +# Makefile for zlib +# Turbo C 2.01, Turbo C++ 1.01 +# Last updated: 15-Mar-2003 + +# To use, do "make -fmakefile.tc" +# To compile in small model, set below: MODEL=s + +# WARNING: the small model is supported but only for small values of +# MAX_WBITS and MAX_MEM_LEVEL. For example: +# -DMAX_WBITS=11 -DMAX_MEM_LEVEL=3 +# If you wish to reduce the memory requirements (default 256K for big +# objects plus a few K), you can add to CFLAGS below: +# -DMAX_MEM_LEVEL=7 -DMAX_WBITS=14 +# See zconf.h for details about the memory requirements. + +# ------------ Turbo C 2.01, Turbo C++ 1.01 ------------ +MODEL=l +CC=tcc +LD=tcc +AR=tlib +# CFLAGS=-O2 -G -Z -m$(MODEL) -DMAX_WBITS=11 -DMAX_MEM_LEVEL=3 +CFLAGS=-O2 -G -Z -m$(MODEL) +LDFLAGS=-m$(MODEL) -f- + + +# variables +ZLIB_LIB = zlib_$(MODEL).lib + +OBJ1 = adler32.obj compress.obj crc32.obj deflate.obj gzclose.obj gzlib.obj gzread.obj +OBJ2 = gzwrite.obj infback.obj inffast.obj inflate.obj inftrees.obj trees.obj uncompr.obj zutil.obj +OBJP1 = +adler32.obj+compress.obj+crc32.obj+deflate.obj+gzclose.obj+gzlib.obj+gzread.obj +OBJP2 = +gzwrite.obj+infback.obj+inffast.obj+inflate.obj+inftrees.obj+trees.obj+uncompr.obj+zutil.obj + + +# targets +all: $(ZLIB_LIB) example.exe minigzip.exe + +.c.obj: + $(CC) -c $(CFLAGS) $*.c + +adler32.obj: adler32.c zlib.h zconf.h + +compress.obj: compress.c zlib.h zconf.h + +crc32.obj: crc32.c zlib.h zconf.h crc32.h + +deflate.obj: deflate.c deflate.h zutil.h zlib.h zconf.h + +gzclose.obj: gzclose.c zlib.h zconf.h gzguts.h + +gzlib.obj: gzlib.c zlib.h zconf.h gzguts.h + +gzread.obj: gzread.c zlib.h zconf.h gzguts.h + +gzwrite.obj: gzwrite.c zlib.h zconf.h gzguts.h + +infback.obj: infback.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inffast.obj: inffast.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h + +inflate.obj: inflate.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inftrees.obj: inftrees.c zutil.h zlib.h zconf.h inftrees.h + +trees.obj: trees.c zutil.h zlib.h zconf.h deflate.h trees.h + +uncompr.obj: uncompr.c zlib.h zconf.h + +zutil.obj: zutil.c zutil.h zlib.h zconf.h + +example.obj: test/example.c zlib.h zconf.h + +minigzip.obj: test/minigzip.c zlib.h zconf.h + + +# the command line is cut to fit in the MS-DOS 128 byte limit: +$(ZLIB_LIB): $(OBJ1) $(OBJ2) + -del $(ZLIB_LIB) + $(AR) $(ZLIB_LIB) $(OBJP1) + $(AR) $(ZLIB_LIB) $(OBJP2) + +example.exe: example.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) example.obj $(ZLIB_LIB) + +minigzip.exe: minigzip.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) minigzip.obj $(ZLIB_LIB) + +test: example.exe minigzip.exe + example + echo hello world | minigzip | minigzip -d + +clean: + -del *.obj + -del *.lib + -del *.exe + -del zlib_*.bak + -del foo.gz diff --git a/zlib/nintendods/README b/zlib/nintendods/README new file mode 100644 index 00000000000..ba7a37dbe8e --- /dev/null +++ b/zlib/nintendods/README @@ -0,0 +1,5 @@ +This Makefile requires devkitARM (http://www.devkitpro.org/category/devkitarm/) and works inside "contrib/nds". It is based on a devkitARM template. + +Eduardo Costa +January 3, 2009 + diff --git a/zlib/old/Makefile.emx b/zlib/old/Makefile.emx new file mode 100644 index 00000000000..612b0379158 --- /dev/null +++ b/zlib/old/Makefile.emx @@ -0,0 +1,69 @@ +# Makefile for zlib. Modified for emx/rsxnt by Chr. Spieler, 6/16/98. +# Copyright (C) 1995-1998 Jean-loup Gailly. +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile, or to compile and test, type: +# +# make -fmakefile.emx; make test -fmakefile.emx +# + +CC=gcc -Zwin32 + +#CFLAGS=-MMD -O +#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#CFLAGS=-MMD -g -DZLIB_DEBUG +CFLAGS=-MMD -O3 $(BUTT) -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ + -Wstrict-prototypes -Wmissing-prototypes + +# If cp.exe is available, replace "copy /Y" with "cp -fp" . +CP=copy /Y +# If gnu install.exe is available, replace $(CP) with ginstall. +INSTALL=$(CP) +# The default value of RM is "rm -f." If "rm.exe" is found, comment out: +RM=del +LDLIBS=-L. -lzlib +LD=$(CC) -s -o +LDSHARED=$(CC) + +INCL=zlib.h zconf.h +LIBS=zlib.a + +AR=ar rcs + +prefix=/usr/local +exec_prefix = $(prefix) + +OBJS = adler32.o compress.o crc32.o deflate.o gzclose.o gzlib.o gzread.o \ + gzwrite.o infback.o inffast.o inflate.o inftrees.o trees.o uncompr.o zutil.o + +TEST_OBJS = example.o minigzip.o + +all: example.exe minigzip.exe + +test: all + ./example + echo hello world | .\minigzip | .\minigzip -d + +%.o : %.c + $(CC) $(CFLAGS) -c $< -o $@ + +zlib.a: $(OBJS) + $(AR) $@ $(OBJS) + +%.exe : %.o $(LIBS) + $(LD) $@ $< $(LDLIBS) + + +.PHONY : clean + +clean: + $(RM) *.d + $(RM) *.o + $(RM) *.exe + $(RM) zlib.a + $(RM) foo.gz + +DEPS := $(wildcard *.d) +ifneq ($(DEPS),) +include $(DEPS) +endif diff --git a/zlib/old/Makefile.riscos b/zlib/old/Makefile.riscos new file mode 100644 index 00000000000..57e29d3fba9 --- /dev/null +++ b/zlib/old/Makefile.riscos @@ -0,0 +1,151 @@ +# Project: zlib_1_03 +# Patched for zlib 1.1.2 rw@shadow.org.uk 19980430 +# test works out-of-the-box, installs `somewhere' on demand + +# Toolflags: +CCflags = -c -depend !Depend -IC: -g -throwback -DRISCOS -fah +C++flags = -c -depend !Depend -IC: -throwback +Linkflags = -aif -c++ -o $@ +ObjAsmflags = -throwback -NoCache -depend !Depend +CMHGflags = +LibFileflags = -c -l -o $@ +Squeezeflags = -o $@ + +# change the line below to where _you_ want the library installed. +libdest = lib:zlib + +# Final targets: +@.lib: @.o.adler32 @.o.compress @.o.crc32 @.o.deflate @.o.gzio \ + @.o.infblock @.o.infcodes @.o.inffast @.o.inflate @.o.inftrees @.o.infutil @.o.trees \ + @.o.uncompr @.o.zutil + LibFile $(LibFileflags) @.o.adler32 @.o.compress @.o.crc32 @.o.deflate \ + @.o.gzio @.o.infblock @.o.infcodes @.o.inffast @.o.inflate @.o.inftrees @.o.infutil \ + @.o.trees @.o.uncompr @.o.zutil +test: @.minigzip @.example @.lib + @copy @.lib @.libc A~C~DF~L~N~P~Q~RS~TV + @echo running tests: hang on. + @/@.minigzip -f -9 libc + @/@.minigzip -d libc-gz + @/@.minigzip -f -1 libc + @/@.minigzip -d libc-gz + @/@.minigzip -h -9 libc + @/@.minigzip -d libc-gz + @/@.minigzip -h -1 libc + @/@.minigzip -d libc-gz + @/@.minigzip -9 libc + @/@.minigzip -d libc-gz + @/@.minigzip -1 libc + @/@.minigzip -d libc-gz + @diff @.lib @.libc + @echo that should have reported '@.lib and @.libc identical' if you have diff. + @/@.example @.fred @.fred + @echo that will have given lots of hello!'s. + +@.minigzip: @.o.minigzip @.lib C:o.Stubs + Link $(Linkflags) @.o.minigzip @.lib C:o.Stubs +@.example: @.o.example @.lib C:o.Stubs + Link $(Linkflags) @.o.example @.lib C:o.Stubs + +install: @.lib + cdir $(libdest) + cdir $(libdest).h + @copy @.h.zlib $(libdest).h.zlib A~C~DF~L~N~P~Q~RS~TV + @copy @.h.zconf $(libdest).h.zconf A~C~DF~L~N~P~Q~RS~TV + @copy @.lib $(libdest).lib A~C~DF~L~N~P~Q~RS~TV + @echo okay, installed zlib in $(libdest) + +clean:; remove @.minigzip + remove @.example + remove @.libc + -wipe @.o.* F~r~cV + remove @.fred + +# User-editable dependencies: +.c.o: + cc $(ccflags) -o $@ $< + +# Static dependencies: + +# Dynamic dependencies: +o.example: c.example +o.example: h.zlib +o.example: h.zconf +o.minigzip: c.minigzip +o.minigzip: h.zlib +o.minigzip: h.zconf +o.adler32: c.adler32 +o.adler32: h.zlib +o.adler32: h.zconf +o.compress: c.compress +o.compress: h.zlib +o.compress: h.zconf +o.crc32: c.crc32 +o.crc32: h.zlib +o.crc32: h.zconf +o.deflate: c.deflate +o.deflate: h.deflate +o.deflate: h.zutil +o.deflate: h.zlib +o.deflate: h.zconf +o.gzio: c.gzio +o.gzio: h.zutil +o.gzio: h.zlib +o.gzio: h.zconf +o.infblock: c.infblock +o.infblock: h.zutil +o.infblock: h.zlib +o.infblock: h.zconf +o.infblock: h.infblock +o.infblock: h.inftrees +o.infblock: h.infcodes +o.infblock: h.infutil +o.infcodes: c.infcodes +o.infcodes: h.zutil +o.infcodes: h.zlib +o.infcodes: h.zconf +o.infcodes: h.inftrees +o.infcodes: h.infblock +o.infcodes: h.infcodes +o.infcodes: h.infutil +o.infcodes: h.inffast +o.inffast: c.inffast +o.inffast: h.zutil +o.inffast: h.zlib +o.inffast: h.zconf +o.inffast: h.inftrees +o.inffast: h.infblock +o.inffast: h.infcodes +o.inffast: h.infutil +o.inffast: h.inffast +o.inflate: c.inflate +o.inflate: h.zutil +o.inflate: h.zlib +o.inflate: h.zconf +o.inflate: h.infblock +o.inftrees: c.inftrees +o.inftrees: h.zutil +o.inftrees: h.zlib +o.inftrees: h.zconf +o.inftrees: h.inftrees +o.inftrees: h.inffixed +o.infutil: c.infutil +o.infutil: h.zutil +o.infutil: h.zlib +o.infutil: h.zconf +o.infutil: h.infblock +o.infutil: h.inftrees +o.infutil: h.infcodes +o.infutil: h.infutil +o.trees: c.trees +o.trees: h.deflate +o.trees: h.zutil +o.trees: h.zlib +o.trees: h.zconf +o.trees: h.trees +o.uncompr: c.uncompr +o.uncompr: h.zlib +o.uncompr: h.zconf +o.zutil: c.zutil +o.zutil: h.zutil +o.zutil: h.zlib +o.zutil: h.zconf diff --git a/zlib/old/README b/zlib/old/README new file mode 100644 index 00000000000..800bf079827 --- /dev/null +++ b/zlib/old/README @@ -0,0 +1,3 @@ +This directory contains files that have not been updated for zlib 1.2.x + +(Volunteers are encouraged to help clean this up. Thanks.) diff --git a/zlib/old/descrip.mms b/zlib/old/descrip.mms new file mode 100644 index 00000000000..7066da5b557 --- /dev/null +++ b/zlib/old/descrip.mms @@ -0,0 +1,48 @@ +# descrip.mms: MMS description file for building zlib on VMS +# written by Martin P.J. Zinser + +cc_defs = +c_deb = + +.ifdef __DECC__ +pref = /prefix=all +.endif + +OBJS = adler32.obj, compress.obj, crc32.obj, gzio.obj, uncompr.obj,\ + deflate.obj, trees.obj, zutil.obj, inflate.obj, infblock.obj,\ + inftrees.obj, infcodes.obj, infutil.obj, inffast.obj + +CFLAGS= $(C_DEB) $(CC_DEFS) $(PREF) + +all : example.exe minigzip.exe + @ write sys$output " Example applications available" +libz.olb : libz.olb($(OBJS)) + @ write sys$output " libz available" + +example.exe : example.obj libz.olb + link example,libz.olb/lib + +minigzip.exe : minigzip.obj libz.olb + link minigzip,libz.olb/lib,x11vms:xvmsutils.olb/lib + +clean : + delete *.obj;*,libz.olb;* + + +# Other dependencies. +adler32.obj : zutil.h zlib.h zconf.h +compress.obj : zlib.h zconf.h +crc32.obj : zutil.h zlib.h zconf.h +deflate.obj : deflate.h zutil.h zlib.h zconf.h +example.obj : zlib.h zconf.h +gzio.obj : zutil.h zlib.h zconf.h +infblock.obj : zutil.h zlib.h zconf.h infblock.h inftrees.h infcodes.h infutil.h +infcodes.obj : zutil.h zlib.h zconf.h inftrees.h infutil.h infcodes.h inffast.h +inffast.obj : zutil.h zlib.h zconf.h inftrees.h infutil.h inffast.h +inflate.obj : zutil.h zlib.h zconf.h infblock.h +inftrees.obj : zutil.h zlib.h zconf.h inftrees.h +infutil.obj : zutil.h zlib.h zconf.h inftrees.h infutil.h +minigzip.obj : zlib.h zconf.h +trees.obj : deflate.h zutil.h zlib.h zconf.h +uncompr.obj : zlib.h zconf.h +zutil.obj : zutil.h zlib.h zconf.h diff --git a/zlib/old/os2/Makefile.os2 b/zlib/old/os2/Makefile.os2 new file mode 100644 index 00000000000..bb426c0d8e6 --- /dev/null +++ b/zlib/old/os2/Makefile.os2 @@ -0,0 +1,136 @@ +# Makefile for zlib under OS/2 using GCC (PGCC) +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile and test, type: +# cp Makefile.os2 .. +# cd .. +# make -f Makefile.os2 test + +# This makefile will build a static library z.lib, a shared library +# z.dll and a import library zdll.lib. You can use either z.lib or +# zdll.lib by specifying either -lz or -lzdll on gcc's command line + +CC=gcc -Zomf -s + +CFLAGS=-O6 -Wall +#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 +#CFLAGS=-g -DZLIB_DEBUG +#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ +# -Wstrict-prototypes -Wmissing-prototypes + +#################### BUG WARNING: ##################### +## infcodes.c hits a bug in pgcc-1.0, so you have to use either +## -O# where # <= 4 or one of (-fno-ommit-frame-pointer or -fno-force-mem) +## This bug is reportedly fixed in pgcc >1.0, but this was not tested +CFLAGS+=-fno-force-mem + +LDFLAGS=-s -L. -lzdll -Zcrtdll +LDSHARED=$(CC) -s -Zomf -Zdll -Zcrtdll + +VER=1.1.0 +ZLIB=z.lib +SHAREDLIB=z.dll +SHAREDLIBIMP=zdll.lib +LIBS=$(ZLIB) $(SHAREDLIB) $(SHAREDLIBIMP) + +AR=emxomfar cr +IMPLIB=emximp +RANLIB=echo +TAR=tar +SHELL=bash + +prefix=/usr/local +exec_prefix = $(prefix) + +OBJS = adler32.o compress.o crc32.o gzio.o uncompr.o deflate.o trees.o \ + zutil.o inflate.o infblock.o inftrees.o infcodes.o infutil.o inffast.o + +TEST_OBJS = example.o minigzip.o + +DISTFILES = README INDEX ChangeLog configure Make*[a-z0-9] *.[ch] descrip.mms \ + algorithm.txt zlib.3 msdos/Make*[a-z0-9] msdos/zlib.def msdos/zlib.rc \ + nt/Makefile.nt nt/zlib.dnt contrib/README.contrib contrib/*.txt \ + contrib/asm386/*.asm contrib/asm386/*.c \ + contrib/asm386/*.bat contrib/asm386/zlibvc.d?? contrib/iostream/*.cpp \ + contrib/iostream/*.h contrib/iostream2/*.h contrib/iostream2/*.cpp \ + contrib/untgz/Makefile contrib/untgz/*.c contrib/untgz/*.w32 + +all: example.exe minigzip.exe + +test: all + @LD_LIBRARY_PATH=.:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ + echo hello world | ./minigzip | ./minigzip -d || \ + echo ' *** minigzip test FAILED ***' ; \ + if ./example; then \ + echo ' *** zlib test OK ***'; \ + else \ + echo ' *** zlib test FAILED ***'; \ + fi + +$(ZLIB): $(OBJS) + $(AR) $@ $(OBJS) + -@ ($(RANLIB) $@ || true) >/dev/null 2>&1 + +$(SHAREDLIB): $(OBJS) os2/z.def + $(LDSHARED) -o $@ $^ + +$(SHAREDLIBIMP): os2/z.def + $(IMPLIB) -o $@ $^ + +example.exe: example.o $(LIBS) + $(CC) $(CFLAGS) -o $@ example.o $(LDFLAGS) + +minigzip.exe: minigzip.o $(LIBS) + $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) + +clean: + rm -f *.o *~ example minigzip libz.a libz.so* foo.gz + +distclean: clean + +zip: + mv Makefile Makefile~; cp -p Makefile.in Makefile + rm -f test.c ztest*.c + v=`sed -n -e 's/\.//g' -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < zlib.h`;\ + zip -ul9 zlib$$v $(DISTFILES) + mv Makefile~ Makefile + +dist: + mv Makefile Makefile~; cp -p Makefile.in Makefile + rm -f test.c ztest*.c + d=zlib-`sed -n '/VERSION "/s/.*"\(.*\)".*/\1/p' < zlib.h`;\ + rm -f $$d.tar.gz; \ + if test ! -d ../$$d; then rm -f ../$$d; ln -s `pwd` ../$$d; fi; \ + files=""; \ + for f in $(DISTFILES); do files="$$files $$d/$$f"; done; \ + cd ..; \ + GZIP=-9 $(TAR) chofz $$d/$$d.tar.gz $$files; \ + if test ! -d $$d; then rm -f $$d; fi + mv Makefile~ Makefile + +tags: + etags *.[ch] + +depend: + makedepend -- $(CFLAGS) -- *.[ch] + +# DO NOT DELETE THIS LINE -- make depend depends on it. + +adler32.o: zlib.h zconf.h +compress.o: zlib.h zconf.h +crc32.o: zlib.h zconf.h +deflate.o: deflate.h zutil.h zlib.h zconf.h +example.o: zlib.h zconf.h +gzio.o: zutil.h zlib.h zconf.h +infblock.o: infblock.h inftrees.h infcodes.h infutil.h zutil.h zlib.h zconf.h +infcodes.o: zutil.h zlib.h zconf.h +infcodes.o: inftrees.h infblock.h infcodes.h infutil.h inffast.h +inffast.o: zutil.h zlib.h zconf.h inftrees.h +inffast.o: infblock.h infcodes.h infutil.h inffast.h +inflate.o: zutil.h zlib.h zconf.h infblock.h +inftrees.o: zutil.h zlib.h zconf.h inftrees.h +infutil.o: zutil.h zlib.h zconf.h infblock.h inftrees.h infcodes.h infutil.h +minigzip.o: zlib.h zconf.h +trees.o: deflate.h zutil.h zlib.h zconf.h trees.h +uncompr.o: zlib.h zconf.h +zutil.o: zutil.h zlib.h zconf.h diff --git a/zlib/old/os2/zlib.def b/zlib/old/os2/zlib.def new file mode 100644 index 00000000000..4c753f1a3b9 --- /dev/null +++ b/zlib/old/os2/zlib.def @@ -0,0 +1,51 @@ +; +; Slightly modified version of ../nt/zlib.dnt :-) +; + +LIBRARY Z +DESCRIPTION "Zlib compression library for OS/2" +CODE PRELOAD MOVEABLE DISCARDABLE +DATA PRELOAD MOVEABLE MULTIPLE + +EXPORTS + adler32 + compress + crc32 + deflate + deflateCopy + deflateEnd + deflateInit2_ + deflateInit_ + deflateParams + deflateReset + deflateSetDictionary + gzclose + gzdopen + gzerror + gzflush + gzopen + gzread + gzwrite + inflate + inflateEnd + inflateInit2_ + inflateInit_ + inflateReset + inflateSetDictionary + inflateSync + uncompress + zlibVersion + gzprintf + gzputc + gzgetc + gzseek + gzrewind + gztell + gzeof + gzsetparams + zError + inflateSyncPoint + get_crc_table + compress2 + gzputs + gzgets diff --git a/zlib/old/visual-basic.txt b/zlib/old/visual-basic.txt new file mode 100644 index 00000000000..57efe58124e --- /dev/null +++ b/zlib/old/visual-basic.txt @@ -0,0 +1,160 @@ +See below some functions declarations for Visual Basic. + +Frequently Asked Question: + +Q: Each time I use the compress function I get the -5 error (not enough + room in the output buffer). + +A: Make sure that the length of the compressed buffer is passed by + reference ("as any"), not by value ("as long"). Also check that + before the call of compress this length is equal to the total size of + the compressed buffer and not zero. + + +From: "Jon Caruana" +Subject: Re: How to port zlib declares to vb? +Date: Mon, 28 Oct 1996 18:33:03 -0600 + +Got the answer! (I haven't had time to check this but it's what I got, and +looks correct): + +He has the following routines working: + compress + uncompress + gzopen + gzwrite + gzread + gzclose + +Declares follow: (Quoted from Carlos Rios , in Vb4 form) + +#If Win16 Then 'Use Win16 calls. +Declare Function compress Lib "ZLIB.DLL" (ByVal compr As + String, comprLen As Any, ByVal buf As String, ByVal buflen + As Long) As Integer +Declare Function uncompress Lib "ZLIB.DLL" (ByVal uncompr + As String, uncomprLen As Any, ByVal compr As String, ByVal + lcompr As Long) As Integer +Declare Function gzopen Lib "ZLIB.DLL" (ByVal filePath As + String, ByVal mode As String) As Long +Declare Function gzread Lib "ZLIB.DLL" (ByVal file As + Long, ByVal uncompr As String, ByVal uncomprLen As Integer) + As Integer +Declare Function gzwrite Lib "ZLIB.DLL" (ByVal file As + Long, ByVal uncompr As String, ByVal uncomprLen As Integer) + As Integer +Declare Function gzclose Lib "ZLIB.DLL" (ByVal file As + Long) As Integer +#Else +Declare Function compress Lib "ZLIB32.DLL" + (ByVal compr As String, comprLen As Any, ByVal buf As + String, ByVal buflen As Long) As Integer +Declare Function uncompress Lib "ZLIB32.DLL" + (ByVal uncompr As String, uncomprLen As Any, ByVal compr As + String, ByVal lcompr As Long) As Long +Declare Function gzopen Lib "ZLIB32.DLL" + (ByVal file As String, ByVal mode As String) As Long +Declare Function gzread Lib "ZLIB32.DLL" + (ByVal file As Long, ByVal uncompr As String, ByVal + uncomprLen As Long) As Long +Declare Function gzwrite Lib "ZLIB32.DLL" + (ByVal file As Long, ByVal uncompr As String, ByVal + uncomprLen As Long) As Long +Declare Function gzclose Lib "ZLIB32.DLL" + (ByVal file As Long) As Long +#End If + +-Jon Caruana +jon-net@usa.net +Microsoft Sitebuilder Network Level 1 Member - HTML Writer's Guild Member + + +Here is another example from Michael that he +says conforms to the VB guidelines, and that solves the problem of not +knowing the uncompressed size by storing it at the end of the file: + +'Calling the functions: +'bracket meaning: [optional] {Range of possible values} +'Call subCompressFile( [, , [level of compression {1..9}]]) +'Call subUncompressFile() + +Option Explicit +Private lngpvtPcnSml As Long 'Stores value for 'lngPercentSmaller' +Private Const SUCCESS As Long = 0 +Private Const strFilExt As String = ".cpr" +Private Declare Function lngfncCpr Lib "zlib.dll" Alias "compress2" (ByRef +dest As Any, ByRef destLen As Any, ByRef src As Any, ByVal srcLen As Long, +ByVal level As Integer) As Long +Private Declare Function lngfncUcp Lib "zlib.dll" Alias "uncompress" (ByRef +dest As Any, ByRef destLen As Any, ByRef src As Any, ByVal srcLen As Long) +As Long + +Public Sub subCompressFile(ByVal strargOriFilPth As String, Optional ByVal +strargCprFilPth As String, Optional ByVal intLvl As Integer = 9) + Dim strCprPth As String + Dim lngOriSiz As Long + Dim lngCprSiz As Long + Dim bytaryOri() As Byte + Dim bytaryCpr() As Byte + lngOriSiz = FileLen(strargOriFilPth) + ReDim bytaryOri(lngOriSiz - 1) + Open strargOriFilPth For Binary Access Read As #1 + Get #1, , bytaryOri() + Close #1 + strCprPth = IIf(strargCprFilPth = "", strargOriFilPth, strargCprFilPth) +'Select file path and name + strCprPth = strCprPth & IIf(Right(strCprPth, Len(strFilExt)) = +strFilExt, "", strFilExt) 'Add file extension if not exists + lngCprSiz = (lngOriSiz * 1.01) + 12 'Compression needs temporary a bit +more space then original file size + ReDim bytaryCpr(lngCprSiz - 1) + If lngfncCpr(bytaryCpr(0), lngCprSiz, bytaryOri(0), lngOriSiz, intLvl) = +SUCCESS Then + lngpvtPcnSml = (1# - (lngCprSiz / lngOriSiz)) * 100 + ReDim Preserve bytaryCpr(lngCprSiz - 1) + Open strCprPth For Binary Access Write As #1 + Put #1, , bytaryCpr() + Put #1, , lngOriSiz 'Add the the original size value to the end +(last 4 bytes) + Close #1 + Else + MsgBox "Compression error" + End If + Erase bytaryCpr + Erase bytaryOri +End Sub + +Public Sub subUncompressFile(ByVal strargFilPth As String) + Dim bytaryCpr() As Byte + Dim bytaryOri() As Byte + Dim lngOriSiz As Long + Dim lngCprSiz As Long + Dim strOriPth As String + lngCprSiz = FileLen(strargFilPth) + ReDim bytaryCpr(lngCprSiz - 1) + Open strargFilPth For Binary Access Read As #1 + Get #1, , bytaryCpr() + Close #1 + 'Read the original file size value: + lngOriSiz = bytaryCpr(lngCprSiz - 1) * (2 ^ 24) _ + + bytaryCpr(lngCprSiz - 2) * (2 ^ 16) _ + + bytaryCpr(lngCprSiz - 3) * (2 ^ 8) _ + + bytaryCpr(lngCprSiz - 4) + ReDim Preserve bytaryCpr(lngCprSiz - 5) 'Cut of the original size value + ReDim bytaryOri(lngOriSiz - 1) + If lngfncUcp(bytaryOri(0), lngOriSiz, bytaryCpr(0), lngCprSiz) = SUCCESS +Then + strOriPth = Left(strargFilPth, Len(strargFilPth) - Len(strFilExt)) + Open strOriPth For Binary Access Write As #1 + Put #1, , bytaryOri() + Close #1 + Else + MsgBox "Uncompression error" + End If + Erase bytaryCpr + Erase bytaryOri +End Sub +Public Property Get lngPercentSmaller() As Long + lngPercentSmaller = lngpvtPcnSml +End Property diff --git a/zlib/os400/README400 b/zlib/os400/README400 new file mode 100644 index 00000000000..4f98334f5ac --- /dev/null +++ b/zlib/os400/README400 @@ -0,0 +1,48 @@ + ZLIB version 1.2.11 for OS/400 installation instructions + +1) Download and unpack the zlib tarball to some IFS directory. + (i.e.: /path/to/the/zlib/ifs/source/directory) + + If the installed IFS command suppors gzip format, this is straightforward, +else you have to unpack first to some directory on a system supporting it, +then move the whole directory to the IFS via the network (via SMB or FTP). + +2) Edit the configuration parameters in the compilation script. + + EDTF STMF('/path/to/the/zlib/ifs/source/directory/os400/make.sh') + +Tune the parameters according to your needs if not matching the defaults. +Save the file and exit after edition. + +3) Enter qshell, then work in the zlib OS/400 specific directory. + + QSH + cd /path/to/the/zlib/ifs/source/directory/os400 + +4) Compile and install + + sh make.sh + +The script will: +- create the libraries, objects and IFS directories for the zlib environment, +- compile all modules, +- create a service program, +- create a static and a dynamic binding directory, +- install header files for C/C++ and for ILE/RPG, both for compilation in + DB2 and IFS environments. + +That's all. + + +Notes: For OS/400 ILE RPG programmers, a /copy member defining the ZLIB + API prototypes for ILE RPG can be found in ZLIB/H(ZLIB.INC). + In the ILE environment, the same definitions are available from + file zlib.inc located in the same IFS include directory as the + C/C++ header files. + Please read comments in this member for more information. + + Remember that most foreign textual data are ASCII coded: this + implementation does not handle conversion from/to ASCII, so + text data code conversions must be done explicitely. + + Mainly for the reason above, always open zipped files in binary mode. diff --git a/zlib/os400/bndsrc b/zlib/os400/bndsrc new file mode 100644 index 00000000000..5e6e0a2f0af --- /dev/null +++ b/zlib/os400/bndsrc @@ -0,0 +1,119 @@ +STRPGMEXP PGMLVL(*CURRENT) SIGNATURE('ZLIB') + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.1.3 entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("adler32") + EXPORT SYMBOL("compress") + EXPORT SYMBOL("compress2") + EXPORT SYMBOL("crc32") + EXPORT SYMBOL("get_crc_table") + EXPORT SYMBOL("deflate") + EXPORT SYMBOL("deflateEnd") + EXPORT SYMBOL("deflateSetDictionary") + EXPORT SYMBOL("deflateCopy") + EXPORT SYMBOL("deflateReset") + EXPORT SYMBOL("deflateParams") + EXPORT SYMBOL("deflatePrime") + EXPORT SYMBOL("deflateInit_") + EXPORT SYMBOL("deflateInit2_") + EXPORT SYMBOL("gzopen") + EXPORT SYMBOL("gzdopen") + EXPORT SYMBOL("gzsetparams") + EXPORT SYMBOL("gzread") + EXPORT SYMBOL("gzwrite") + EXPORT SYMBOL("gzprintf") + EXPORT SYMBOL("gzputs") + EXPORT SYMBOL("gzgets") + EXPORT SYMBOL("gzputc") + EXPORT SYMBOL("gzgetc") + EXPORT SYMBOL("gzflush") + EXPORT SYMBOL("gzseek") + EXPORT SYMBOL("gzrewind") + EXPORT SYMBOL("gztell") + EXPORT SYMBOL("gzeof") + EXPORT SYMBOL("gzclose") + EXPORT SYMBOL("gzerror") + EXPORT SYMBOL("inflate") + EXPORT SYMBOL("inflateEnd") + EXPORT SYMBOL("inflateSetDictionary") + EXPORT SYMBOL("inflateSync") + EXPORT SYMBOL("inflateReset") + EXPORT SYMBOL("inflateInit_") + EXPORT SYMBOL("inflateInit2_") + EXPORT SYMBOL("inflateSyncPoint") + EXPORT SYMBOL("uncompress") + EXPORT SYMBOL("zlibVersion") + EXPORT SYMBOL("zError") + EXPORT SYMBOL("z_errmsg") + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.2.1 additional entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("compressBound") + EXPORT SYMBOL("deflateBound") + EXPORT SYMBOL("deflatePending") + EXPORT SYMBOL("gzungetc") + EXPORT SYMBOL("gzclearerr") + EXPORT SYMBOL("inflateBack") + EXPORT SYMBOL("inflateBackEnd") + EXPORT SYMBOL("inflateBackInit_") + EXPORT SYMBOL("inflateCopy") + EXPORT SYMBOL("zlibCompileFlags") + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.2.4 additional entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("adler32_combine") + EXPORT SYMBOL("adler32_combine64") + EXPORT SYMBOL("crc32_combine") + EXPORT SYMBOL("crc32_combine64") + EXPORT SYMBOL("deflateSetHeader") + EXPORT SYMBOL("deflateTune") + EXPORT SYMBOL("gzbuffer") + EXPORT SYMBOL("gzclose_r") + EXPORT SYMBOL("gzclose_w") + EXPORT SYMBOL("gzdirect") + EXPORT SYMBOL("gzoffset") + EXPORT SYMBOL("gzoffset64") + EXPORT SYMBOL("gzopen64") + EXPORT SYMBOL("gzseek64") + EXPORT SYMBOL("gztell64") + EXPORT SYMBOL("inflateGetHeader") + EXPORT SYMBOL("inflateMark") + EXPORT SYMBOL("inflatePrime") + EXPORT SYMBOL("inflateReset2") + EXPORT SYMBOL("inflateUndermine") + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.2.6 additional entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("deflateResetKeep") + EXPORT SYMBOL("gzgetc_") + EXPORT SYMBOL("inflateResetKeep") + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.2.8 additional entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("gzvprintf") + EXPORT SYMBOL("inflateGetDictionary") + +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ +/* Version 1.2.9 additional entry points. */ +/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/ + + EXPORT SYMBOL("adler32_z") + EXPORT SYMBOL("crc32_z") + EXPORT SYMBOL("deflateGetDictionary") + EXPORT SYMBOL("gzfread") + EXPORT SYMBOL("gzfwrite") + EXPORT SYMBOL("inflateCodesUsed") + EXPORT SYMBOL("inflateValidate") + EXPORT SYMBOL("uncompress2") + +ENDPGMEXP diff --git a/zlib/os400/make.sh b/zlib/os400/make.sh new file mode 100644 index 00000000000..19eec117a67 --- /dev/null +++ b/zlib/os400/make.sh @@ -0,0 +1,366 @@ +#!/bin/sh +# +# ZLIB compilation script for the OS/400. +# +# +# This is a shell script since make is not a standard component of OS/400. + + +################################################################################ +# +# Tunable configuration parameters. +# +################################################################################ + +TARGETLIB='ZLIB' # Target OS/400 program library +STATBNDDIR='ZLIB_A' # Static binding directory. +DYNBNDDIR='ZLIB' # Dynamic binding directory. +SRVPGM="ZLIB" # Service program. +IFSDIR='/zlib' # IFS support base directory. +TGTCCSID='500' # Target CCSID of objects +DEBUG='*NONE' # Debug level +OPTIMIZE='40' # Optimisation level +OUTPUT='*NONE' # Compilation output option. +TGTRLS='V6R1M0' # Target OS release + +export TARGETLIB STATBNDDIR DYNBNDDIR SRVPGM IFSDIR +export TGTCCSID DEBUG OPTIMIZE OUTPUT TGTRLS + + +################################################################################ +# +# OS/400 specific definitions. +# +################################################################################ + +LIBIFSNAME="/QSYS.LIB/${TARGETLIB}.LIB" + + +################################################################################ +# +# Procedures. +# +################################################################################ + +# action_needed dest [src] +# +# dest is an object to build +# if specified, src is an object on which dest depends. +# +# exit 0 (succeeds) if some action has to be taken, else 1. + +action_needed() + +{ + [ ! -e "${1}" ] && return 0 + [ "${2}" ] || return 1 + [ "${1}" -ot "${2}" ] && return 0 + return 1 +} + + +# make_module module_name source_name [additional_definitions] +# +# Compile source name into module if needed. +# As side effect, append the module name to variable MODULES. +# Set LINK to "YES" if the module has been compiled. + +make_module() + +{ + MODULES="${MODULES} ${1}" + MODIFSNAME="${LIBIFSNAME}/${1}.MODULE" + CSRC="`basename \"${2}\"`" + + if action_needed "${MODIFSNAME}" "${2}" + then : + elif [ ! "`sed -e \"//,/<\\\\/source>/!d\" \ + -e '/ tmphdrfile + + # Need to translate to target CCSID. + + CMD="CPY OBJ('`pwd`/tmphdrfile') TOOBJ('${DEST}')" + CMD="${CMD} TOCCSID(${TGTCCSID}) DTAFMT(*TEXT) REPLACE(*YES)" + system "${CMD}" + # touch -r "${HFILE}" "${DEST}" + rm -f tmphdrfile + fi + + IFSFILE="${IFSDIR}/include/`basename \"${HFILE}\"`" + + if action_needed "${IFSFILE}" "${DEST}" + then rm -f "${IFSFILE}" + ln -s "${DEST}" "${IFSFILE}" + fi +done + + +# Install the ILE/RPG header file. + + +HFILE="${SCRIPTDIR}/zlib.inc" +DEST="${SRCPF}/ZLIB.INC.MBR" + +if action_needed "${DEST}" "${HFILE}" +then CMD="CPY OBJ('${HFILE}') TOOBJ('${DEST}')" + CMD="${CMD} TOCCSID(${TGTCCSID}) DTAFMT(*TEXT) REPLACE(*YES)" + system "${CMD}" + # touch -r "${HFILE}" "${DEST}" +fi + +IFSFILE="${IFSDIR}/include/`basename \"${HFILE}\"`" + +if action_needed "${IFSFILE}" "${DEST}" +then rm -f "${IFSFILE}" + ln -s "${DEST}" "${IFSFILE}" +fi + + +# Create and compile the identification source file. + +echo '#pragma comment(user, "ZLIB version '"${VERSION}"'")' > os400.c +echo '#pragma comment(user, __DATE__)' >> os400.c +echo '#pragma comment(user, __TIME__)' >> os400.c +echo '#pragma comment(copyright, "Copyright (C) 1995-2017 Jean-Loup Gailly, Mark Adler. OS/400 version by P. Monnerat.")' >> os400.c +make_module OS400 os400.c +LINK= # No need to rebuild service program yet. +MODULES= + + +# Get source list. + +CSOURCES=`sed -e '/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Library + + Medium + + 2.0 + + + + zlib + zlib + alain.bonnefoy@icbt.com + Public + public + www.gzip.org/zlib + + + Jean-Loup Gailly,Mark Adler + www.gzip.org/zlib + + zlib@gzip.org + + + A massively spiffy yet delicately unobtrusive compression library. + zlib is designed to be a free, general-purpose, legally unencumbered, lossless data compression library for use on virtually any computer hardware and operating system. + http://www.gzip.org/zlib + + + + + 1.2.11 + Medium + Stable + + + + + + + No License + + + + Software Development/Libraries and Extensions/C Libraries + zlib,compression + qnx6 + qnx6 + None + Developer + + + + + + + + + + + + + + Install + Post + No + Ignore + + No + Optional + + + + + + + + + + + + + InstallOver + zlib + + + + + + + + + + + + + InstallOver + zlib-dev + + + + + + + + + diff --git a/zlib/test/example.c b/zlib/test/example.c new file mode 100644 index 00000000000..eee17ce7c15 --- /dev/null +++ b/zlib/test/example.c @@ -0,0 +1,602 @@ +/* example.c -- usage example of the zlib compression library + * Copyright (C) 1995-2006, 2011, 2016 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zlib.h" +#include + +#ifdef STDC +# include +# include +#endif + +#if defined(VMS) || defined(RISCOS) +# define TESTFILE "foo-gz" +#else +# define TESTFILE "foo.gz" +#endif + +#define CHECK_ERR(err, msg) { \ + if (err != Z_OK) { \ + fprintf(stderr, "%s error: %d\n", msg, err); \ + exit(1); \ + } \ +} + +static z_const char hello[] = "hello, hello!"; +/* "hello world" would be more standard, but the repeated "hello" + * stresses the compression code better, sorry... + */ + +static const char dictionary[] = "hello"; +static uLong dictId; /* Adler32 value of the dictionary */ + +void test_deflate OF((Byte *compr, uLong comprLen)); +void test_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_deflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_flush OF((Byte *compr, uLong *comprLen)); +void test_sync OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_dict_deflate OF((Byte *compr, uLong comprLen)); +void test_dict_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +int main OF((int argc, char *argv[])); + + +#ifdef Z_SOLO + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + (void)q; + return calloc(n, m); +} + +void myfree(void *q, void *p) +{ + (void)q; + free(p); +} + +static alloc_func zalloc = myalloc; +static free_func zfree = myfree; + +#else /* !Z_SOLO */ + +static alloc_func zalloc = (alloc_func)0; +static free_func zfree = (free_func)0; + +void test_compress OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_gzio OF((const char *fname, + Byte *uncompr, uLong uncomprLen)); + +/* =========================================================================== + * Test compress() and uncompress() + */ +void test_compress(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + uLong len = (uLong)strlen(hello)+1; + + err = compress(compr, &comprLen, (const Bytef*)hello, len); + CHECK_ERR(err, "compress"); + + strcpy((char*)uncompr, "garbage"); + + err = uncompress(uncompr, &uncomprLen, compr, comprLen); + CHECK_ERR(err, "uncompress"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad uncompress\n"); + exit(1); + } else { + printf("uncompress(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test read/write of .gz files + */ +void test_gzio(fname, uncompr, uncomprLen) + const char *fname; /* compressed file name */ + Byte *uncompr; + uLong uncomprLen; +{ +#ifdef NO_GZCOMPRESS + fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n"); +#else + int err; + int len = (int)strlen(hello)+1; + gzFile file; + z_off_t pos; + + file = gzopen(fname, "wb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + gzputc(file, 'h'); + if (gzputs(file, "ello") != 4) { + fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); + exit(1); + } + if (gzprintf(file, ", %s!", "hello") != 8) { + fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); + exit(1); + } + gzseek(file, 1L, SEEK_CUR); /* add one zero byte */ + gzclose(file); + + file = gzopen(fname, "rb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + strcpy((char*)uncompr, "garbage"); + + if (gzread(file, uncompr, (unsigned)uncomprLen) != len) { + fprintf(stderr, "gzread err: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad gzread: %s\n", (char*)uncompr); + exit(1); + } else { + printf("gzread(): %s\n", (char*)uncompr); + } + + pos = gzseek(file, -8L, SEEK_CUR); + if (pos != 6 || gztell(file) != pos) { + fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", + (long)pos, (long)gztell(file)); + exit(1); + } + + if (gzgetc(file) != ' ') { + fprintf(stderr, "gzgetc error\n"); + exit(1); + } + + if (gzungetc(' ', file) != ' ') { + fprintf(stderr, "gzungetc error\n"); + exit(1); + } + + gzgets(file, (char*)uncompr, (int)uncomprLen); + if (strlen((char*)uncompr) != 7) { /* " hello!" */ + fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello + 6)) { + fprintf(stderr, "bad gzgets after gzseek\n"); + exit(1); + } else { + printf("gzgets() after gzseek: %s\n", (char*)uncompr); + } + + gzclose(file); +#endif +} + +#endif /* Z_SOLO */ + +/* =========================================================================== + * Test deflate() with small buffers + */ +void test_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uLong len = (uLong)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + + while (c_stream.total_in != len && c_stream.total_out < comprLen) { + c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */ + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + } + /* Finish the stream, still forcing small buffers: */ + for (;;) { + c_stream.avail_out = 1; + err = deflate(&c_stream, Z_FINISH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "deflate"); + } + + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with small buffers + */ +void test_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 0; + d_stream.next_out = uncompr; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) { + d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */ + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate\n"); + exit(1); + } else { + printf("inflate(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test deflate() with large buffers and dynamic change of compression level + */ +void test_large_deflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_SPEED); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + /* At this point, uncompr is still mostly zeroes, so it should compress + * very well: + */ + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + if (c_stream.avail_in != 0) { + fprintf(stderr, "deflate not greedy\n"); + exit(1); + } + + /* Feed in already compressed data and switch to no compression: */ + deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY); + c_stream.next_in = compr; + c_stream.avail_in = (uInt)comprLen/2; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + /* Switch back to compressing mode: */ + deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED); + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with large buffers + */ +void test_large_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + for (;;) { + d_stream.next_out = uncompr; /* discard the output */ + d_stream.avail_out = (uInt)uncomprLen; + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "large inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (d_stream.total_out != 2*uncomprLen + comprLen/2) { + fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out); + exit(1); + } else { + printf("large_inflate(): OK\n"); + } +} + +/* =========================================================================== + * Test deflate() with full flush + */ +void test_flush(compr, comprLen) + Byte *compr; + uLong *comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uInt len = (uInt)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + c_stream.avail_in = 3; + c_stream.avail_out = (uInt)*comprLen; + err = deflate(&c_stream, Z_FULL_FLUSH); + CHECK_ERR(err, "deflate"); + + compr[3]++; /* force an error in first compressed block */ + c_stream.avail_in = len - 3; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + CHECK_ERR(err, "deflate"); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); + + *comprLen = c_stream.total_out; +} + +/* =========================================================================== + * Test inflateSync() + */ +void test_sync(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 2; /* just read the zlib header */ + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + err = inflate(&d_stream, Z_NO_FLUSH); + CHECK_ERR(err, "inflate"); + + d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */ + err = inflateSync(&d_stream); /* but skip the damaged part */ + CHECK_ERR(err, "inflateSync"); + + err = inflate(&d_stream, Z_FINISH); + if (err != Z_DATA_ERROR) { + fprintf(stderr, "inflate should report DATA_ERROR\n"); + /* Because of incorrect adler32 */ + exit(1); + } + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + printf("after inflateSync(): hel%s\n", (char *)uncompr); +} + +/* =========================================================================== + * Test deflate() with preset dictionary + */ +void test_dict_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + err = deflateSetDictionary(&c_stream, + (const Bytef*)dictionary, (int)sizeof(dictionary)); + CHECK_ERR(err, "deflateSetDictionary"); + + dictId = c_stream.adler; + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.avail_in = (uInt)strlen(hello)+1; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with a preset dictionary + */ +void test_dict_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + for (;;) { + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + if (err == Z_NEED_DICT) { + if (d_stream.adler != dictId) { + fprintf(stderr, "unexpected dictionary"); + exit(1); + } + err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary, + (int)sizeof(dictionary)); + } + CHECK_ERR(err, "inflate with dict"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate with dict\n"); + exit(1); + } else { + printf("inflate with dictionary: %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Usage: example [output.gz [input.gz]] + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + Byte *compr, *uncompr; + uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */ + uLong uncomprLen = comprLen; + static const char* myVersion = ZLIB_VERSION; + + if (zlibVersion()[0] != myVersion[0]) { + fprintf(stderr, "incompatible zlib version\n"); + exit(1); + + } else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) { + fprintf(stderr, "warning: different zlib version\n"); + } + + printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", + ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + + compr = (Byte*)calloc((uInt)comprLen, 1); + uncompr = (Byte*)calloc((uInt)uncomprLen, 1); + /* compr and uncompr are cleared to avoid reading uninitialized + * data and to ensure that uncompr compresses well. + */ + if (compr == Z_NULL || uncompr == Z_NULL) { + printf("out of memory\n"); + exit(1); + } + +#ifdef Z_SOLO + (void)argc; + (void)argv; +#else + test_compress(compr, comprLen, uncompr, uncomprLen); + + test_gzio((argc > 1 ? argv[1] : TESTFILE), + uncompr, uncomprLen); +#endif + + test_deflate(compr, comprLen); + test_inflate(compr, comprLen, uncompr, uncomprLen); + + test_large_deflate(compr, comprLen, uncompr, uncomprLen); + test_large_inflate(compr, comprLen, uncompr, uncomprLen); + + test_flush(compr, &comprLen); + test_sync(compr, comprLen, uncompr, uncomprLen); + comprLen = uncomprLen; + + test_dict_deflate(compr, comprLen); + test_dict_inflate(compr, comprLen, uncompr, uncomprLen); + + free(compr); + free(uncompr); + + return 0; +} diff --git a/zlib/test/infcover.c b/zlib/test/infcover.c new file mode 100644 index 00000000000..2be01646cec --- /dev/null +++ b/zlib/test/infcover.c @@ -0,0 +1,671 @@ +/* infcover.c -- test zlib's inflate routines with full code coverage + * Copyright (C) 2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* to use, do: ./configure --cover && make cover */ + +#include +#include +#include +#include +#include "zlib.h" + +/* get definition of internal structure so we can mess with it (see pull()), + and so we can call inflate_trees() (see cover5()) */ +#define ZLIB_INTERNAL +#include "inftrees.h" +#include "inflate.h" + +#define local static + +/* -- memory tracking routines -- */ + +/* + These memory tracking routines are provided to zlib and track all of zlib's + allocations and deallocations, check for LIFO operations, keep a current + and high water mark of total bytes requested, optionally set a limit on the + total memory that can be allocated, and when done check for memory leaks. + + They are used as follows: + + z_stream strm; + mem_setup(&strm) initializes the memory tracking and sets the + zalloc, zfree, and opaque members of strm to use + memory tracking for all zlib operations on strm + mem_limit(&strm, limit) sets a limit on the total bytes requested -- a + request that exceeds this limit will result in an + allocation failure (returns NULL) -- setting the + limit to zero means no limit, which is the default + after mem_setup() + mem_used(&strm, "msg") prints to stderr "msg" and the total bytes used + mem_high(&strm, "msg") prints to stderr "msg" and the high water mark + mem_done(&strm, "msg") ends memory tracking, releases all allocations + for the tracking as well as leaked zlib blocks, if + any. If there was anything unusual, such as leaked + blocks, non-FIFO frees, or frees of addresses not + allocated, then "msg" and information about the + problem is printed to stderr. If everything is + normal, nothing is printed. mem_done resets the + strm members to Z_NULL to use the default memory + allocation routines on the next zlib initialization + using strm. + */ + +/* these items are strung together in a linked list, one for each allocation */ +struct mem_item { + void *ptr; /* pointer to allocated memory */ + size_t size; /* requested size of allocation */ + struct mem_item *next; /* pointer to next item in list, or NULL */ +}; + +/* this structure is at the root of the linked list, and tracks statistics */ +struct mem_zone { + struct mem_item *first; /* pointer to first item in list, or NULL */ + size_t total, highwater; /* total allocations, and largest total */ + size_t limit; /* memory allocation limit, or 0 if no limit */ + int notlifo, rogue; /* counts of non-LIFO frees and rogue frees */ +}; + +/* memory allocation routine to pass to zlib */ +local void *mem_alloc(void *mem, unsigned count, unsigned size) +{ + void *ptr; + struct mem_item *item; + struct mem_zone *zone = mem; + size_t len = count * (size_t)size; + + /* induced allocation failure */ + if (zone == NULL || (zone->limit && zone->total + len > zone->limit)) + return NULL; + + /* perform allocation using the standard library, fill memory with a + non-zero value to make sure that the code isn't depending on zeros */ + ptr = malloc(len); + if (ptr == NULL) + return NULL; + memset(ptr, 0xa5, len); + + /* create a new item for the list */ + item = malloc(sizeof(struct mem_item)); + if (item == NULL) { + free(ptr); + return NULL; + } + item->ptr = ptr; + item->size = len; + + /* insert item at the beginning of the list */ + item->next = zone->first; + zone->first = item; + + /* update the statistics */ + zone->total += item->size; + if (zone->total > zone->highwater) + zone->highwater = zone->total; + + /* return the allocated memory */ + return ptr; +} + +/* memory free routine to pass to zlib */ +local void mem_free(void *mem, void *ptr) +{ + struct mem_item *item, *next; + struct mem_zone *zone = mem; + + /* if no zone, just do a free */ + if (zone == NULL) { + free(ptr); + return; + } + + /* point next to the item that matches ptr, or NULL if not found -- remove + the item from the linked list if found */ + next = zone->first; + if (next) { + if (next->ptr == ptr) + zone->first = next->next; /* first one is it, remove from list */ + else { + do { /* search the linked list */ + item = next; + next = item->next; + } while (next != NULL && next->ptr != ptr); + if (next) { /* if found, remove from linked list */ + item->next = next->next; + zone->notlifo++; /* not a LIFO free */ + } + + } + } + + /* if found, update the statistics and free the item */ + if (next) { + zone->total -= next->size; + free(next); + } + + /* if not found, update the rogue count */ + else + zone->rogue++; + + /* in any case, do the requested free with the standard library function */ + free(ptr); +} + +/* set up a controlled memory allocation space for monitoring, set the stream + parameters to the controlled routines, with opaque pointing to the space */ +local void mem_setup(z_stream *strm) +{ + struct mem_zone *zone; + + zone = malloc(sizeof(struct mem_zone)); + assert(zone != NULL); + zone->first = NULL; + zone->total = 0; + zone->highwater = 0; + zone->limit = 0; + zone->notlifo = 0; + zone->rogue = 0; + strm->opaque = zone; + strm->zalloc = mem_alloc; + strm->zfree = mem_free; +} + +/* set a limit on the total memory allocation, or 0 to remove the limit */ +local void mem_limit(z_stream *strm, size_t limit) +{ + struct mem_zone *zone = strm->opaque; + + zone->limit = limit; +} + +/* show the current total requested allocations in bytes */ +local void mem_used(z_stream *strm, char *prefix) +{ + struct mem_zone *zone = strm->opaque; + + fprintf(stderr, "%s: %lu allocated\n", prefix, zone->total); +} + +/* show the high water allocation in bytes */ +local void mem_high(z_stream *strm, char *prefix) +{ + struct mem_zone *zone = strm->opaque; + + fprintf(stderr, "%s: %lu high water mark\n", prefix, zone->highwater); +} + +/* release the memory allocation zone -- if there are any surprises, notify */ +local void mem_done(z_stream *strm, char *prefix) +{ + int count = 0; + struct mem_item *item, *next; + struct mem_zone *zone = strm->opaque; + + /* show high water mark */ + mem_high(strm, prefix); + + /* free leftover allocations and item structures, if any */ + item = zone->first; + while (item != NULL) { + free(item->ptr); + next = item->next; + free(item); + item = next; + count++; + } + + /* issue alerts about anything unexpected */ + if (count || zone->total) + fprintf(stderr, "** %s: %lu bytes in %d blocks not freed\n", + prefix, zone->total, count); + if (zone->notlifo) + fprintf(stderr, "** %s: %d frees not LIFO\n", prefix, zone->notlifo); + if (zone->rogue) + fprintf(stderr, "** %s: %d frees not recognized\n", + prefix, zone->rogue); + + /* free the zone and delete from the stream */ + free(zone); + strm->opaque = Z_NULL; + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; +} + +/* -- inflate test routines -- */ + +/* Decode a hexadecimal string, set *len to length, in[] to the bytes. This + decodes liberally, in that hex digits can be adjacent, in which case two in + a row writes a byte. Or they can be delimited by any non-hex character, + where the delimiters are ignored except when a single hex digit is followed + by a delimiter, where that single digit writes a byte. The returned data is + allocated and must eventually be freed. NULL is returned if out of memory. + If the length is not needed, then len can be NULL. */ +local unsigned char *h2b(const char *hex, unsigned *len) +{ + unsigned char *in, *re; + unsigned next, val; + + in = malloc((strlen(hex) + 1) >> 1); + if (in == NULL) + return NULL; + next = 0; + val = 1; + do { + if (*hex >= '0' && *hex <= '9') + val = (val << 4) + *hex - '0'; + else if (*hex >= 'A' && *hex <= 'F') + val = (val << 4) + *hex - 'A' + 10; + else if (*hex >= 'a' && *hex <= 'f') + val = (val << 4) + *hex - 'a' + 10; + else if (val != 1 && val < 32) /* one digit followed by delimiter */ + val += 240; /* make it look like two digits */ + if (val > 255) { /* have two digits */ + in[next++] = val & 0xff; /* save the decoded byte */ + val = 1; /* start over */ + } + } while (*hex++); /* go through the loop with the terminating null */ + if (len != NULL) + *len = next; + re = realloc(in, next); + return re == NULL ? in : re; +} + +/* generic inflate() run, where hex is the hexadecimal input data, what is the + text to include in an error message, step is how much input data to feed + inflate() on each call, or zero to feed it all, win is the window bits + parameter to inflateInit2(), len is the size of the output buffer, and err + is the error code expected from the first inflate() call (the second + inflate() call is expected to return Z_STREAM_END). If win is 47, then + header information is collected with inflateGetHeader(). If a zlib stream + is looking for a dictionary, then an empty dictionary is provided. + inflate() is run until all of the input data is consumed. */ +local void inf(char *hex, char *what, unsigned step, int win, unsigned len, + int err) +{ + int ret; + unsigned have; + unsigned char *in, *out; + z_stream strm, copy; + gz_header head; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, win); + if (ret != Z_OK) { + mem_done(&strm, what); + return; + } + out = malloc(len); assert(out != NULL); + if (win == 47) { + head.extra = out; + head.extra_max = len; + head.name = out; + head.name_max = len; + head.comment = out; + head.comm_max = len; + ret = inflateGetHeader(&strm, &head); assert(ret == Z_OK); + } + in = h2b(hex, &have); assert(in != NULL); + if (step == 0 || step > have) + step = have; + strm.avail_in = step; + have -= step; + strm.next_in = in; + do { + strm.avail_out = len; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); assert(err == 9 || ret == err); + if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_NEED_DICT) + break; + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary(&strm, in, 1); + assert(ret == Z_DATA_ERROR); + mem_limit(&strm, 1); + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ((struct inflate_state *)strm.state)->mode = DICT; + ret = inflateSetDictionary(&strm, out, 0); + assert(ret == Z_OK); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_BUF_ERROR); + } + ret = inflateCopy(©, &strm); assert(ret == Z_OK); + ret = inflateEnd(©); assert(ret == Z_OK); + err = 9; /* don't care next time around */ + have += strm.avail_in; + strm.avail_in = step > have ? have : step; + have -= strm.avail_in; + } while (strm.avail_in); + free(in); + free(out); + ret = inflateReset2(&strm, -8); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, what); +} + +/* cover all of the lines in inflate.c up to inflate() */ +local void cover_support(void) +{ + int ret; + z_stream strm; + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + mem_used(&strm, "inflate init"); + ret = inflatePrime(&strm, 5, 31); assert(ret == Z_OK); + ret = inflatePrime(&strm, -1, 0); assert(ret == Z_OK); + ret = inflateSetDictionary(&strm, Z_NULL, 0); + assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "prime"); + + inf("63 0", "force window allocation", 0, -15, 1, Z_OK); + inf("63 18 5", "force window replacement", 0, -8, 259, Z_OK); + inf("63 18 68 30 d0 0 0", "force split window update", 4, -8, 259, Z_OK); + inf("3 0", "use fixed blocks", 0, -15, 1, Z_STREAM_END); + inf("", "bad window size", 0, 1, 0, Z_STREAM_ERROR); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream)); + assert(ret == Z_VERSION_ERROR); + mem_done(&strm, "wrong version"); + + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); assert(ret == Z_OK); + ret = inflateEnd(&strm); assert(ret == Z_OK); + fputs("inflate built-in memory routines\n", stderr); +} + +/* cover all inflate() header and trailer cases and code after inflate() */ +local void cover_wrap(void) +{ + int ret; + z_stream strm, copy; + unsigned char dict[257]; + + ret = inflate(Z_NULL, 0); assert(ret == Z_STREAM_ERROR); + ret = inflateEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + ret = inflateCopy(Z_NULL, Z_NULL); assert(ret == Z_STREAM_ERROR); + fputs("inflate bad parameters\n", stderr); + + inf("1f 8b 0 0", "bad gzip method", 0, 31, 0, Z_DATA_ERROR); + inf("1f 8b 8 80", "bad gzip flags", 0, 31, 0, Z_DATA_ERROR); + inf("77 85", "bad zlib method", 0, 15, 0, Z_DATA_ERROR); + inf("8 99", "set window size from header", 0, 0, 0, Z_OK); + inf("78 9c", "bad zlib window size", 0, 8, 0, Z_DATA_ERROR); + inf("78 9c 63 0 0 0 1 0 1", "check adler32", 0, 15, 1, Z_STREAM_END); + inf("1f 8b 8 1e 0 0 0 0 0 0 1 0 0 0 0 0 0", "bad header crc", 0, 47, 1, + Z_DATA_ERROR); + inf("1f 8b 8 2 0 0 0 0 0 0 1d 26 3 0 0 0 0 0 0 0 0 0", "check gzip length", + 0, 47, 0, Z_STREAM_END); + inf("78 90", "bad zlib header check", 0, 47, 0, Z_DATA_ERROR); + inf("8 b8 0 0 0 1", "need dictionary", 0, 8, 0, Z_NEED_DICT); + inf("78 9c 63 0", "compute adler32", 0, 15, 1, Z_OK); + + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, -8); + strm.avail_in = 2; + strm.next_in = (void *)"\x63"; + strm.avail_out = 1; + strm.next_out = (void *)&ret; + mem_limit(&strm, 1); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + memset(dict, 0, 257); + ret = inflateSetDictionary(&strm, dict, 257); + assert(ret == Z_OK); + mem_limit(&strm, (sizeof(struct inflate_state) << 1) + 256); + ret = inflatePrime(&strm, 16, 0); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (void *)"\x80"; + ret = inflateSync(&strm); assert(ret == Z_DATA_ERROR); + ret = inflate(&strm, Z_NO_FLUSH); assert(ret == Z_STREAM_ERROR); + strm.avail_in = 4; + strm.next_in = (void *)"\0\0\xff\xff"; + ret = inflateSync(&strm); assert(ret == Z_OK); + (void)inflateSyncPoint(&strm); + ret = inflateCopy(©, &strm); assert(ret == Z_MEM_ERROR); + mem_limit(&strm, 0); + ret = inflateUndermine(&strm, 1); assert(ret == Z_DATA_ERROR); + (void)inflateMark(&strm); + ret = inflateEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "miscellaneous, force memory errors"); +} + +/* input and output functions for inflateBack() */ +local unsigned pull(void *desc, unsigned char **buf) +{ + static unsigned int next = 0; + static unsigned char dat[] = {0x63, 0, 2, 0}; + struct inflate_state *state; + + if (desc == Z_NULL) { + next = 0; + return 0; /* no input (already provided at next_in) */ + } + state = (void *)((z_stream *)desc)->state; + if (state != Z_NULL) + state->mode = SYNC; /* force an otherwise impossible situation */ + return next < sizeof(dat) ? (*buf = dat + next++, 1) : 0; +} + +local int push(void *desc, unsigned char *buf, unsigned len) +{ + buf += len; + return desc != Z_NULL; /* force error if desc not null */ +} + +/* cover inflateBack() up to common deflate data cases and after those */ +local void cover_back(void) +{ + int ret; + z_stream strm; + unsigned char win[32768]; + + ret = inflateBackInit_(Z_NULL, 0, win, 0, 0); + assert(ret == Z_VERSION_ERROR); + ret = inflateBackInit(Z_NULL, 0, win); assert(ret == Z_STREAM_ERROR); + ret = inflateBack(Z_NULL, Z_NULL, Z_NULL, Z_NULL, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(Z_NULL); assert(ret == Z_STREAM_ERROR); + fputs("inflateBack bad parameters\n", stderr); + + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + strm.avail_in = 2; + strm.next_in = (void *)"\x03"; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret == Z_STREAM_END); + /* force output error */ + strm.avail_in = 3; + strm.next_in = (void *)"\x63\x00"; + ret = inflateBack(&strm, pull, Z_NULL, push, &strm); + assert(ret == Z_BUF_ERROR); + /* force mode error by mucking with state */ + ret = inflateBack(&strm, pull, &strm, push, Z_NULL); + assert(ret == Z_STREAM_ERROR); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + mem_done(&strm, "inflateBack bad state"); + + ret = inflateBackInit(&strm, 15, win); assert(ret == Z_OK); + ret = inflateBackEnd(&strm); assert(ret == Z_OK); + fputs("inflateBack built-in memory routines\n", stderr); +} + +/* do a raw inflate of data in hexadecimal with both inflate and inflateBack */ +local int try(char *hex, char *id, int err) +{ + int ret; + unsigned len, size; + unsigned char *in, *out, *win; + char *prefix; + z_stream strm; + + /* convert to hex */ + in = h2b(hex, &len); + assert(in != NULL); + + /* allocate work areas */ + size = len << 3; + out = malloc(size); + assert(out != NULL); + win = malloc(32768); + assert(win != NULL); + prefix = malloc(strlen(id) + 6); + assert(prefix != NULL); + + /* first with inflate */ + strcpy(prefix, id); + strcat(prefix, "-late"); + mem_setup(&strm); + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, err < 0 ? 47 : -15); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + do { + strm.avail_out = size; + strm.next_out = out; + ret = inflate(&strm, Z_TREES); + assert(ret != Z_STREAM_ERROR && ret != Z_MEM_ERROR); + if (ret == Z_DATA_ERROR || ret == Z_NEED_DICT) + break; + } while (strm.avail_in || strm.avail_out == 0); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateEnd(&strm); + mem_done(&strm, prefix); + + /* then with inflateBack */ + if (err >= 0) { + strcpy(prefix, id); + strcat(prefix, "-back"); + mem_setup(&strm); + ret = inflateBackInit(&strm, 15, win); + assert(ret == Z_OK); + strm.avail_in = len; + strm.next_in = in; + ret = inflateBack(&strm, pull, Z_NULL, push, Z_NULL); + assert(ret != Z_STREAM_ERROR); + if (err) { + assert(ret == Z_DATA_ERROR); + assert(strcmp(id, strm.msg) == 0); + } + inflateBackEnd(&strm); + mem_done(&strm, prefix); + } + + /* clean up */ + free(prefix); + free(win); + free(out); + free(in); + return ret; +} + +/* cover deflate data cases in both inflate() and inflateBack() */ +local void cover_inflate(void) +{ + try("0 0 0 0 0", "invalid stored block lengths", 1); + try("3 0", "fixed", 0); + try("6", "invalid block type", 1); + try("1 1 0 fe ff 0", "stored", 0); + try("fc 0 0", "too many length or distance symbols", 1); + try("4 0 fe ff", "invalid code lengths set", 1); + try("4 0 24 49 0", "invalid bit length repeat", 1); + try("4 0 24 e9 ff ff", "invalid bit length repeat", 1); + try("4 0 24 e9 ff 6d", "invalid code -- missing end-of-block", 1); + try("4 80 49 92 24 49 92 24 71 ff ff 93 11 0", + "invalid literal/lengths set", 1); + try("4 80 49 92 24 49 92 24 f b4 ff ff c3 84", "invalid distances set", 1); + try("4 c0 81 8 0 0 0 0 20 7f eb b 0 0", "invalid literal/length code", 1); + try("2 7e ff ff", "invalid distance code", 1); + try("c c0 81 0 0 0 0 0 90 ff 6b 4 0", "invalid distance too far back", 1); + + /* also trailer mismatch just in inflate() */ + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 1", "incorrect data check", -1); + try("1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 1", + "incorrect length check", -1); + try("5 c0 21 d 0 0 0 80 b0 fe 6d 2f 91 6c", "pull 17", 0); + try("5 e0 81 91 24 cb b2 2c 49 e2 f 2e 8b 9a 47 56 9f fb fe ec d2 ff 1f", + "long code", 0); + try("ed c0 1 1 0 0 0 40 20 ff 57 1b 42 2c 4f", "length extra", 0); + try("ed cf c1 b1 2c 47 10 c4 30 fa 6f 35 1d 1 82 59 3d fb be 2e 2a fc f c", + "long distance and extra", 0); + try("ed c0 81 0 0 0 0 80 a0 fd a9 17 a9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6", "window end", 0); + inf("2 8 20 80 0 3 0", "inflate_fast TYPE return", 0, -15, 258, + Z_STREAM_END); + inf("63 18 5 40 c 0", "window wrap", 3, -8, 300, Z_OK); +} + +/* cover remaining lines in inftrees.c */ +local void cover_trees(void) +{ + int ret; + unsigned bits; + unsigned short lens[16], work[16]; + code *next, table[ENOUGH_DISTS]; + + /* we need to call inflate_table() directly in order to manifest not- + enough errors, since zlib insures that enough is always enough */ + for (bits = 0; bits < 15; bits++) + lens[bits] = (unsigned short)(bits + 1); + lens[15] = 15; + next = table; + bits = 15; + ret = inflate_table(DISTS, lens, 16, &next, &bits, work); + assert(ret == 1); + next = table; + bits = 1; + ret = inflate_table(DISTS, lens, 16, &next, &bits, work); + assert(ret == 1); + fputs("inflate_table not enough errors\n", stderr); +} + +/* cover remaining inffast.c decoding and window copying */ +local void cover_fast(void) +{ + inf("e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68" + " ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, Z_DATA_ERROR); + inf("25 fd 81 b5 6d 59 b6 6a 49 ea af 35 6 34 eb 8c b9 f6 b9 1e ef 67 49" + " 50 fe ff ff 3f 0 0", "fast distance extra bits", 0, -8, 258, + Z_DATA_ERROR); + inf("3 7e 0 0 0 0 0", "fast invalid distance code", 0, -8, 258, + Z_DATA_ERROR); + inf("1b 7 0 0 0 0 0", "fast invalid literal/length code", 0, -8, 258, + Z_DATA_ERROR); + inf("d c7 1 ae eb 38 c 4 41 a0 87 72 de df fb 1f b8 36 b1 38 5d ff ff 0", + "fast 2nd level codes and too far back", 0, -8, 258, Z_DATA_ERROR); + inf("63 18 5 8c 10 8 0 0 0 0", "very common case", 0, -8, 259, Z_OK); + inf("63 60 60 18 c9 0 8 18 18 18 26 c0 28 0 29 0 0 0", + "contiguous and wrap around window", 6, -8, 259, Z_OK); + inf("63 0 3 0 0 0 0 0", "copy direct from output", 0, -8, 259, + Z_STREAM_END); +} + +int main(void) +{ + fprintf(stderr, "%s\n", zlibVersion()); + cover_support(); + cover_wrap(); + cover_back(); + cover_inflate(); + cover_trees(); + cover_fast(); + return 0; +} diff --git a/zlib/test/minigzip.c b/zlib/test/minigzip.c new file mode 100644 index 00000000000..e22fb08c0a2 --- /dev/null +++ b/zlib/test/minigzip.c @@ -0,0 +1,651 @@ +/* minigzip.c -- simulate gzip using the zlib compression library + * Copyright (C) 1995-2006, 2010, 2011, 2016 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * minigzip is a minimal implementation of the gzip utility. This is + * only an example of using zlib and isn't meant to replace the + * full-featured gzip. No attempt is made to deal with file systems + * limiting names to 14 or 8+3 characters, etc... Error checking is + * very limited. So use minigzip only for testing; use gzip for the + * real thing. On MSDOS, use only on file names without extension + * or in pipe mode. + */ + +/* @(#) $Id$ */ + +#include "zlib.h" +#include + +#ifdef STDC +# include +# include +#endif + +#ifdef USE_MMAP +# include +# include +# include +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# ifdef UNDER_CE +# include +# endif +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifdef VMS +# define unlink delete +# define GZ_SUFFIX "-gz" +#endif +#ifdef RISCOS +# define unlink remove +# define GZ_SUFFIX "-gz" +# define fileno(file) file->__file +#endif +#if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fileno */ +#endif + +#if !defined(Z_HAVE_UNISTD_H) && !defined(_LARGEFILE64_SOURCE) +#ifndef WIN32 /* unlink already in stdio.h for WIN32 */ + extern int unlink OF((const char *)); +#endif +#endif + +#if defined(UNDER_CE) +# include +# define perror(s) pwinerror(s) + +/* Map the Windows error number in ERROR to a locale-dependent error + message string and return a pointer to it. Typically, the values + for ERROR come from GetLastError. + + The string pointed to shall not be modified by the application, + but may be overwritten by a subsequent call to strwinerror + + The strwinerror function does not change the current setting + of GetLastError. */ + +static char *strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +static void pwinerror (s) + const char *s; +{ + if (s && *s) + fprintf(stderr, "%s: %s\n", s, strwinerror(GetLastError ())); + else + fprintf(stderr, "%s\n", strwinerror(GetLastError ())); +} + +#endif /* UNDER_CE */ + +#ifndef GZ_SUFFIX +# define GZ_SUFFIX ".gz" +#endif +#define SUFFIX_LEN (sizeof(GZ_SUFFIX)-1) + +#define BUFLEN 16384 +#define MAX_NAME_LEN 1024 + +#ifdef MAXSEG_64K +# define local static + /* Needed for systems with limitation on stack size. */ +#else +# define local +#endif + +#ifdef Z_SOLO +/* for Z_SOLO, create simplified gz* functions using deflate and inflate */ + +#if defined(Z_HAVE_UNISTD_H) || defined(Z_LARGE) +# include /* for unlink() */ +#endif + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + (void)q; + return calloc(n, m); +} + +void myfree(q, p) + void *q, *p; +{ + (void)q; + free(p); +} + +typedef struct gzFile_s { + FILE *file; + int write; + int err; + char *msg; + z_stream strm; +} *gzFile; + +gzFile gzopen OF((const char *, const char *)); +gzFile gzdopen OF((int, const char *)); +gzFile gz_open OF((const char *, int, const char *)); + +gzFile gzopen(path, mode) +const char *path; +const char *mode; +{ + return gz_open(path, -1, mode); +} + +gzFile gzdopen(fd, mode) +int fd; +const char *mode; +{ + return gz_open(NULL, fd, mode); +} + +gzFile gz_open(path, fd, mode) + const char *path; + int fd; + const char *mode; +{ + gzFile gz; + int ret; + + gz = malloc(sizeof(struct gzFile_s)); + if (gz == NULL) + return NULL; + gz->write = strchr(mode, 'w') != NULL; + gz->strm.zalloc = myalloc; + gz->strm.zfree = myfree; + gz->strm.opaque = Z_NULL; + if (gz->write) + ret = deflateInit2(&(gz->strm), -1, 8, 15 + 16, 8, 0); + else { + gz->strm.next_in = 0; + gz->strm.avail_in = Z_NULL; + ret = inflateInit2(&(gz->strm), 15 + 16); + } + if (ret != Z_OK) { + free(gz); + return NULL; + } + gz->file = path == NULL ? fdopen(fd, gz->write ? "wb" : "rb") : + fopen(path, gz->write ? "wb" : "rb"); + if (gz->file == NULL) { + gz->write ? deflateEnd(&(gz->strm)) : inflateEnd(&(gz->strm)); + free(gz); + return NULL; + } + gz->err = 0; + gz->msg = ""; + return gz; +} + +int gzwrite OF((gzFile, const void *, unsigned)); + +int gzwrite(gz, buf, len) + gzFile gz; + const void *buf; + unsigned len; +{ + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL || !gz->write) + return 0; + strm = &(gz->strm); + strm->next_in = (void *)buf; + strm->avail_in = len; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_NO_FLUSH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + return len; +} + +int gzread OF((gzFile, void *, unsigned)); + +int gzread(gz, buf, len) + gzFile gz; + void *buf; + unsigned len; +{ + int ret; + unsigned got; + unsigned char in[1]; + z_stream *strm; + + if (gz == NULL || gz->write) + return 0; + if (gz->err) + return 0; + strm = &(gz->strm); + strm->next_out = (void *)buf; + strm->avail_out = len; + do { + got = fread(in, 1, 1, gz->file); + if (got == 0) + break; + strm->next_in = in; + strm->avail_in = 1; + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_DATA_ERROR) { + gz->err = Z_DATA_ERROR; + gz->msg = strm->msg; + return 0; + } + if (ret == Z_STREAM_END) + inflateReset(strm); + } while (strm->avail_out); + return len - strm->avail_out; +} + +int gzclose OF((gzFile)); + +int gzclose(gz) + gzFile gz; +{ + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL) + return Z_STREAM_ERROR; + strm = &(gz->strm); + if (gz->write) { + strm->next_in = Z_NULL; + strm->avail_in = 0; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_FINISH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + deflateEnd(strm); + } + else + inflateEnd(strm); + fclose(gz->file); + free(gz); + return Z_OK; +} + +const char *gzerror OF((gzFile, int *)); + +const char *gzerror(gz, err) + gzFile gz; + int *err; +{ + *err = gz->err; + return gz->msg; +} + +#endif + +static char *prog; + +void error OF((const char *msg)); +void gz_compress OF((FILE *in, gzFile out)); +#ifdef USE_MMAP +int gz_compress_mmap OF((FILE *in, gzFile out)); +#endif +void gz_uncompress OF((gzFile in, FILE *out)); +void file_compress OF((char *file, char *mode)); +void file_uncompress OF((char *file)); +int main OF((int argc, char *argv[])); + +/* =========================================================================== + * Display error message and exit + */ +void error(msg) + const char *msg; +{ + fprintf(stderr, "%s: %s\n", prog, msg); + exit(1); +} + +/* =========================================================================== + * Compress input to output then close both files. + */ + +void gz_compress(in, out) + FILE *in; + gzFile out; +{ + local char buf[BUFLEN]; + int len; + int err; + +#ifdef USE_MMAP + /* Try first compressing with mmap. If mmap fails (minigzip used in a + * pipe), use the normal fread loop. + */ + if (gz_compress_mmap(in, out) == Z_OK) return; +#endif + for (;;) { + len = (int)fread(buf, 1, sizeof(buf), in); + if (ferror(in)) { + perror("fread"); + exit(1); + } + if (len == 0) break; + + if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err)); + } + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); +} + +#ifdef USE_MMAP /* MMAP version, Miguel Albrecht */ + +/* Try compressing the input file at once using mmap. Return Z_OK if + * if success, Z_ERRNO otherwise. + */ +int gz_compress_mmap(in, out) + FILE *in; + gzFile out; +{ + int len; + int err; + int ifd = fileno(in); + caddr_t buf; /* mmap'ed buffer for the entire input file */ + off_t buf_len; /* length of the input file */ + struct stat sb; + + /* Determine the size of the file, needed for mmap: */ + if (fstat(ifd, &sb) < 0) return Z_ERRNO; + buf_len = sb.st_size; + if (buf_len <= 0) return Z_ERRNO; + + /* Now do the actual mmap: */ + buf = mmap((caddr_t) 0, buf_len, PROT_READ, MAP_SHARED, ifd, (off_t)0); + if (buf == (caddr_t)(-1)) return Z_ERRNO; + + /* Compress the whole file at once: */ + len = gzwrite(out, (char *)buf, (unsigned)buf_len); + + if (len != (int)buf_len) error(gzerror(out, &err)); + + munmap(buf, buf_len); + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); + return Z_OK; +} +#endif /* USE_MMAP */ + +/* =========================================================================== + * Uncompress input to output then close both files. + */ +void gz_uncompress(in, out) + gzFile in; + FILE *out; +{ + local char buf[BUFLEN]; + int len; + int err; + + for (;;) { + len = gzread(in, buf, sizeof(buf)); + if (len < 0) error (gzerror(in, &err)); + if (len == 0) break; + + if ((int)fwrite(buf, 1, (unsigned)len, out) != len) { + error("failed fwrite"); + } + } + if (fclose(out)) error("failed fclose"); + + if (gzclose(in) != Z_OK) error("failed gzclose"); +} + + +/* =========================================================================== + * Compress the given file: create a corresponding .gz file and remove the + * original. + */ +void file_compress(file, mode) + char *file; + char *mode; +{ + local char outfile[MAX_NAME_LEN]; + FILE *in; + gzFile out; + + if (strlen(file) + strlen(GZ_SUFFIX) >= sizeof(outfile)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(outfile, sizeof(outfile), "%s%s", file, GZ_SUFFIX); +#else + strcpy(outfile, file); + strcat(outfile, GZ_SUFFIX); +#endif + + in = fopen(file, "rb"); + if (in == NULL) { + perror(file); + exit(1); + } + out = gzopen(outfile, mode); + if (out == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, outfile); + exit(1); + } + gz_compress(in, out); + + unlink(file); +} + + +/* =========================================================================== + * Uncompress the given file and remove the original. + */ +void file_uncompress(file) + char *file; +{ + local char buf[MAX_NAME_LEN]; + char *infile, *outfile; + FILE *out; + gzFile in; + unsigned len = strlen(file); + + if (len + strlen(GZ_SUFFIX) >= sizeof(buf)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(buf, sizeof(buf), "%s", file); +#else + strcpy(buf, file); +#endif + + if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) { + infile = file; + outfile = buf; + outfile[len-3] = '\0'; + } else { + outfile = file; + infile = buf; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(buf + len, sizeof(buf) - len, "%s", GZ_SUFFIX); +#else + strcat(infile, GZ_SUFFIX); +#endif + } + in = gzopen(infile, "rb"); + if (in == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, infile); + exit(1); + } + out = fopen(outfile, "wb"); + if (out == NULL) { + perror(file); + exit(1); + } + + gz_uncompress(in, out); + + unlink(infile); +} + + +/* =========================================================================== + * Usage: minigzip [-c] [-d] [-f] [-h] [-r] [-1 to -9] [files...] + * -c : write to standard output + * -d : decompress + * -f : compress with Z_FILTERED + * -h : compress with Z_HUFFMAN_ONLY + * -r : compress with Z_RLE + * -1 to -9 : compression level + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + int copyout = 0; + int uncompr = 0; + gzFile file; + char *bname, outmode[20]; + +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(outmode, sizeof(outmode), "%s", "wb6 "); +#else + strcpy(outmode, "wb6 "); +#endif + + prog = argv[0]; + bname = strrchr(argv[0], '/'); + if (bname) + bname++; + else + bname = argv[0]; + argc--, argv++; + + if (!strcmp(bname, "gunzip")) + uncompr = 1; + else if (!strcmp(bname, "zcat")) + copyout = uncompr = 1; + + while (argc > 0) { + if (strcmp(*argv, "-c") == 0) + copyout = 1; + else if (strcmp(*argv, "-d") == 0) + uncompr = 1; + else if (strcmp(*argv, "-f") == 0) + outmode[3] = 'f'; + else if (strcmp(*argv, "-h") == 0) + outmode[3] = 'h'; + else if (strcmp(*argv, "-r") == 0) + outmode[3] = 'R'; + else if ((*argv)[0] == '-' && (*argv)[1] >= '1' && (*argv)[1] <= '9' && + (*argv)[2] == 0) + outmode[2] = (*argv)[1]; + else + break; + argc--, argv++; + } + if (outmode[3] == ' ') + outmode[3] = 0; + if (argc == 0) { + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + if (uncompr) { + file = gzdopen(fileno(stdin), "rb"); + if (file == NULL) error("can't gzdopen stdin"); + gz_uncompress(file, stdout); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + gz_compress(stdin, file); + } + } else { + if (copyout) { + SET_BINARY_MODE(stdout); + } + do { + if (uncompr) { + if (copyout) { + file = gzopen(*argv, "rb"); + if (file == NULL) + fprintf(stderr, "%s: can't gzopen %s\n", prog, *argv); + else + gz_uncompress(file, stdout); + } else { + file_uncompress(*argv); + } + } else { + if (copyout) { + FILE * in = fopen(*argv, "rb"); + + if (in == NULL) { + perror(*argv); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + + gz_compress(in, file); + } + + } else { + file_compress(*argv, outmode); + } + } + } while (argv++, --argc); + } + return 0; +} diff --git a/zlib/treebuild.xml b/zlib/treebuild.xml new file mode 100644 index 00000000000..fd75525f995 --- /dev/null +++ b/zlib/treebuild.xml @@ -0,0 +1,116 @@ + + + + zip compression library + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/zlib/trees.c b/zlib/trees.c index 395e4e16814..1f64ae3ec67 100644 --- a/zlib/trees.c +++ b/zlib/trees.c @@ -1,5 +1,6 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2005 Jean-loup Gailly + * Copyright (C) 1995-2017 Jean-loup Gailly + * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -35,7 +36,7 @@ #include "deflate.h" -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include #endif @@ -73,11 +74,6 @@ local const uch bl_order[BL_CODES] * probability, to avoid transmitting the lengths for unused bit length codes. */ -#define Buf_size (8 * 2*sizeof(char)) -/* Number of bits used within bi_buf. (bi_buf might be implemented on - * more than 16 bits on some systems.) - */ - /* =========================================================================== * Local data. These are initialized only once. */ @@ -126,13 +122,13 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -local static_tree_desc static_l_desc = +local const static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local static_tree_desc static_d_desc = +local const static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local static_tree_desc static_bl_desc = +local const static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== @@ -150,24 +146,22 @@ local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); local int build_bl_tree OF((deflate_state *s)); local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, int blcodes)); -local void compress_block OF((deflate_state *s, ct_data *ltree, - ct_data *dtree)); -local void set_data_type OF((deflate_state *s)); +local void compress_block OF((deflate_state *s, const ct_data *ltree, + const ct_data *dtree)); +local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); -local void copy_block OF((deflate_state *s, charf *buf, unsigned len, - int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif -#ifndef DEBUG +#ifndef ZLIB_DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ -#else /* DEBUG */ +#else /* !ZLIB_DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } @@ -186,7 +180,7 @@ local void gen_trees_header OF((void)); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) @@ -203,31 +197,31 @@ local void send_bits(s, value, length) * unused bits in value. */ if (s->bi_valid > (int)Buf_size - length) { - s->bi_buf |= (value << s->bi_valid); + s->bi_buf |= (ush)value << s->bi_valid; put_short(s, s->bi_buf); s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); s->bi_valid += length - Buf_size; } else { - s->bi_buf |= value << s->bi_valid; + s->bi_buf |= (ush)value << s->bi_valid; s->bi_valid += length; } } -#else /* !DEBUG */ +#else /* !ZLIB_DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ - s->bi_buf |= (val << s->bi_valid);\ + int val = (int)value;\ + s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ s->bi_valid += len - Buf_size;\ } else {\ - s->bi_buf |= (value) << s->bi_valid;\ + s->bi_buf |= (ush)(value) << s->bi_valid;\ s->bi_valid += len;\ }\ } -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* the arguments must not have side effects */ @@ -250,11 +244,13 @@ local void tr_static_init() if (static_init_done) return; /* For some embedded targets, global variables are not initialized: */ +#ifdef NO_INIT_GLOBAL_POINTERS static_l_desc.static_tree = static_ltree; static_l_desc.extra_bits = extra_lbits; static_d_desc.static_tree = static_dtree; static_d_desc.extra_bits = extra_dbits; static_bl_desc.extra_bits = extra_blbits; +#endif /* Initialize the mapping length (0..255) -> length code (0..28) */ length = 0; @@ -319,7 +315,7 @@ local void tr_static_init() * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H -# ifndef DEBUG +# ifndef ZLIB_DEBUG # include # endif @@ -348,13 +344,14 @@ void gen_trees_header() static_dtree[i].Len, SEPARATOR(i, D_CODES-1, 5)); } - fprintf(header, "const uch _dist_code[DIST_CODE_LEN] = {\n"); + fprintf(header, "const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = {\n"); for (i = 0; i < DIST_CODE_LEN; i++) { fprintf(header, "%2u%s", _dist_code[i], SEPARATOR(i, DIST_CODE_LEN-1, 20)); } - fprintf(header, "const uch _length_code[MAX_MATCH-MIN_MATCH+1]= {\n"); + fprintf(header, + "const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= {\n"); for (i = 0; i < MAX_MATCH-MIN_MATCH+1; i++) { fprintf(header, "%2u%s", _length_code[i], SEPARATOR(i, MAX_MATCH-MIN_MATCH, 20)); @@ -379,7 +376,7 @@ void gen_trees_header() /* =========================================================================== * Initialize the tree data structures for a new zlib stream. */ -void _tr_init(s) +void ZLIB_INTERNAL _tr_init(s) deflate_state *s; { tr_static_init(); @@ -395,8 +392,7 @@ void _tr_init(s) s->bi_buf = 0; s->bi_valid = 0; - s->last_eob_len = 8; /* enough lookahead for inflate */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif @@ -524,12 +520,12 @@ local void gen_bitlen(s, desc) xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; - s->opt_len += (ulg)f * (bits + xbits); - if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); } if (overflow == 0) return; - Trace((stderr,"\nbit length overflow\n")); + Tracev((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ @@ -556,9 +552,8 @@ local void gen_bitlen(s, desc) m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { - Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); - s->opt_len += ((long)bits - (long)tree[m].Len) - *(long)tree[m].Freq; + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; tree[m].Len = (ush)bits; } n--; @@ -580,7 +575,7 @@ local void gen_codes (tree, max_code, bl_count) ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - ush code = 0; /* running code value */ + unsigned code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ @@ -588,7 +583,8 @@ local void gen_codes (tree, max_code, bl_count) * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { - next_code[bits] = code = (code + bl_count[bits-1]) << 1; + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. @@ -601,7 +597,7 @@ local void gen_codes (tree, max_code, bl_count) int len = tree[n].Len; if (len == 0) continue; /* Now reverse the bits */ - tree[n].Code = bi_reverse(next_code[len]++, len); + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); @@ -823,7 +819,7 @@ local int build_bl_tree(s) if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ - s->opt_len += 3*(max_blindex+1) + 5+5+4; + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); @@ -864,65 +860,63 @@ local void send_all_trees(s, lcodes, dcodes, blcodes) /* =========================================================================== * Send a stored block */ -void _tr_stored_block(s, buf, stored_len, eof) +void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) deflate_state *s; charf *buf; /* input block */ ulg stored_len; /* length of input block */ - int eof; /* true if this is the last block for a file */ + int last; /* one if this is the last block for a file */ { - send_bits(s, (STORED_BLOCK<<1)+eof, 3); /* send block type */ -#ifdef DEBUG + send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + if (stored_len != 0) { + /* Avoid calling memcpy when stored_len equals to zero to + avoid UBSAN errors */ + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + } + s->pending += stored_len; +#ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; #endif - copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ +} + +/* =========================================================================== + * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) + */ +void ZLIB_INTERNAL _tr_flush_bits(s) + deflate_state *s; +{ + bi_flush(s); } /* =========================================================================== * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. - * The current inflate code requires 9 bits of lookahead. If the - * last two codes for the previous block (real code plus EOB) were coded - * on 5 bits or less, inflate may have only 5+3 bits of lookahead to decode - * the last real code. In this case we send two empty static blocks instead - * of one. (There are no problems if the previous block is stored or fixed.) - * To simplify the code, we assume the worst case of last real code encoded - * on one bit only. */ -void _tr_align(s) +void ZLIB_INTERNAL _tr_align(s) deflate_state *s; { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); - /* Of the 10 bits for the empty block, we have already sent - * (10 - bi_valid) bits. The lookahead for the last real code (before - * the EOB of the previous block) was thus at least one plus the length - * of the EOB plus what we have just sent of the empty static block. - */ - if (1 + s->last_eob_len + 10 - s->bi_valid < 9) { - send_bits(s, STATIC_TREES<<1, 3); - send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG - s->compressed_len += 10L; -#endif - bi_flush(s); - } - s->last_eob_len = 7; } /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static - * trees or store, and output the encoded block to the zip file. + * trees or store, and write out the encoded block. */ -void _tr_flush_block(s, buf, stored_len, eof) +void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; charf *buf; /* input block, or NULL if too old */ ulg stored_len; /* length of input block */ - int eof; /* true if this is the last block for a file */ + int last; /* one if this is the last block for a file */ { ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ int max_blindex = 0; /* index of last bit length code of non zero freq */ @@ -931,8 +925,8 @@ void _tr_flush_block(s, buf, stored_len, eof) if (s->level > 0) { /* Check if the file is binary or text */ - if (stored_len > 0 && s->strm->data_type == Z_UNKNOWN) - set_data_type(s); + if (s->strm->data_type == Z_UNKNOWN) + s->strm->data_type = detect_data_type(s); /* Construct the literal and distance trees */ build_tree(s, (tree_desc *)(&(s->l_desc))); @@ -978,24 +972,26 @@ void _tr_flush_block(s, buf, stored_len, eof) * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to * transform a block into a stored block. */ - _tr_stored_block(s, buf, stored_len, eof); + _tr_stored_block(s, buf, stored_len, last); #ifdef FORCE_STATIC } else if (static_lenb >= 0) { /* force static trees */ #else } else if (s->strategy == Z_FIXED || static_lenb == opt_lenb) { #endif - send_bits(s, (STATIC_TREES<<1)+eof, 3); - compress_block(s, (ct_data *)static_ltree, (ct_data *)static_dtree); -#ifdef DEBUG + send_bits(s, (STATIC_TREES<<1)+last, 3); + compress_block(s, (const ct_data *)static_ltree, + (const ct_data *)static_dtree); +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->static_len; #endif } else { - send_bits(s, (DYN_TREES<<1)+eof, 3); + send_bits(s, (DYN_TREES<<1)+last, 3); send_all_trees(s, s->l_desc.max_code+1, s->d_desc.max_code+1, max_blindex+1); - compress_block(s, (ct_data *)s->dyn_ltree, (ct_data *)s->dyn_dtree); -#ifdef DEBUG + compress_block(s, (const ct_data *)s->dyn_ltree, + (const ct_data *)s->dyn_dtree); +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->opt_len; #endif } @@ -1005,21 +1001,21 @@ void _tr_flush_block(s, buf, stored_len, eof) */ init_block(s); - if (eof) { + if (last) { bi_windup(s); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len>>3, - s->compressed_len-7*eof)); + s->compressed_len-7*last)); } /* =========================================================================== * Save the match info and tally the frequency counts. Return true if * the current block must be flushed. */ -int _tr_tally (s, dist, lc) +int ZLIB_INTERNAL _tr_tally (s, dist, lc) deflate_state *s; unsigned dist; /* distance of matched string */ unsigned lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ @@ -1071,8 +1067,8 @@ int _tr_tally (s, dist, lc) */ local void compress_block(s, ltree, dtree) deflate_state *s; - ct_data *ltree; /* literal tree */ - ct_data *dtree; /* distance tree */ + const ct_data *ltree; /* literal tree */ + const ct_data *dtree; /* distance tree */ { unsigned dist; /* distance of matched string */ int lc; /* match length or unmatched char (if dist == 0) */ @@ -1102,7 +1098,7 @@ local void compress_block(s, ltree, dtree) send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { - dist -= base_dist[code]; + dist -= (unsigned)base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ @@ -1114,28 +1110,48 @@ local void compress_block(s, ltree, dtree) } while (lx < s->last_lit); send_code(s, END_BLOCK, ltree); - s->last_eob_len = ltree[END_BLOCK].Len; } /* =========================================================================== - * Set the data type to BINARY or TEXT, using a crude approximation: - * set it to Z_TEXT if all symbols are either printable characters (33 to 255) - * or white spaces (9 to 13, or 32); or set it to Z_BINARY otherwise. + * Check if the data type is TEXT or BINARY, using the following algorithm: + * - TEXT if the two conditions below are satisfied: + * a) There are no non-portable control characters belonging to the + * "black list" (0..6, 14..25, 28..31). + * b) There is at least one printable character belonging to the + * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). + * - BINARY otherwise. + * - The following partially-portable control characters form a + * "gray list" that is ignored in this detection algorithm: + * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). * IN assertion: the fields Freq of dyn_ltree are set. */ -local void set_data_type(s) +local int detect_data_type(s) deflate_state *s; { + /* black_mask is the bit mask of black-listed bytes + * set bits 0..6, 14..25, and 28..31 + * 0xf3ffc07f = binary 11110011111111111100000001111111 + */ + unsigned long black_mask = 0xf3ffc07fUL; int n; - for (n = 0; n < 9; n++) + /* Check for non-textual ("black-listed") bytes. */ + for (n = 0; n <= 31; n++, black_mask >>= 1) + if ((black_mask & 1) && (s->dyn_ltree[n].Freq != 0)) + return Z_BINARY; + + /* Check for textual ("white-listed") bytes. */ + if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 + || s->dyn_ltree[13].Freq != 0) + return Z_TEXT; + for (n = 32; n < LITERALS; n++) if (s->dyn_ltree[n].Freq != 0) - break; - if (n == 9) - for (n = 14; n < 32; n++) - if (s->dyn_ltree[n].Freq != 0) - break; - s->strm->data_type = (n == 32) ? Z_TEXT : Z_BINARY; + return Z_TEXT; + + /* There are no "black-listed" or "white-listed" bytes: + * this stream either is empty or has tolerated ("gray-listed") bytes only. + */ + return Z_BINARY; } /* =========================================================================== @@ -1185,35 +1201,7 @@ local void bi_windup(s) } s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } - -/* =========================================================================== - * Copy a stored block, storing first the length and its - * one's complement if requested. - */ -local void copy_block(s, buf, len, header) - deflate_state *s; - charf *buf; /* the input data */ - unsigned len; /* its length */ - int header; /* true if block header must be written */ -{ - bi_windup(s); /* align on byte boundary */ - s->last_eob_len = 8; /* enough lookahead for inflate */ - - if (header) { - put_short(s, (ush)len); - put_short(s, (ush)~len); -#ifdef DEBUG - s->bits_sent += 2*16; -#endif - } -#ifdef DEBUG - s->bits_sent += (ulg)len<<3; -#endif - while (len--) { - put_byte(s, *buf++); - } -} diff --git a/zlib/trees.h b/zlib/trees.h index 72facf900f7..d35639d82a2 100644 --- a/zlib/trees.h +++ b/zlib/trees.h @@ -70,7 +70,7 @@ local const ct_data static_dtree[D_CODES] = { {{19},{ 5}}, {{11},{ 5}}, {{27},{ 5}}, {{ 7},{ 5}}, {{23},{ 5}} }; -const uch _dist_code[DIST_CODE_LEN] = { +const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = { 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, @@ -99,7 +99,7 @@ const uch _dist_code[DIST_CODE_LEN] = { 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29 }; -const uch _length_code[MAX_MATCH-MIN_MATCH+1]= { +const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= { 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, diff --git a/zlib/uncompr.c b/zlib/uncompr.c index b59e3d0defb..f03a1a865e3 100644 --- a/zlib/uncompr.c +++ b/zlib/uncompr.c @@ -1,5 +1,5 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003 Jean-loup Gailly. + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,53 +9,85 @@ #include "zlib.h" /* =========================================================================== - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - This function can be used to decompress a whole file at once if the - input file is mmap'ed. + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. */ +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong *sourceLen; +{ + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = inflateInit(&stream); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + int ZEXPORT uncompress (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; uLong sourceLen; { - z_stream stream; - int err; - - stream.next_in = (Bytef*)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; - - stream.zalloc = (alloc_func)0; - stream.zfree = (free_func)0; - - err = inflateInit(&stream); - if (err != Z_OK) return err; - - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - *destLen = stream.total_out; - - err = inflateEnd(&stream); - return err; + return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/zlib/watcom/watcom_f.mak b/zlib/watcom/watcom_f.mak new file mode 100644 index 00000000000..37f4d74c19c --- /dev/null +++ b/zlib/watcom/watcom_f.mak @@ -0,0 +1,43 @@ +# Makefile for zlib +# OpenWatcom flat model +# Last updated: 28-Dec-2005 + +# To use, do "wmake -f watcom_f.mak" + +C_SOURCE = adler32.c compress.c crc32.c deflate.c & + gzclose.c gzlib.c gzread.c gzwrite.c & + infback.c inffast.c inflate.c inftrees.c & + trees.c uncompr.c zutil.c + +OBJS = adler32.obj compress.obj crc32.obj deflate.obj & + gzclose.obj gzlib.obj gzread.obj gzwrite.obj & + infback.obj inffast.obj inflate.obj inftrees.obj & + trees.obj uncompr.obj zutil.obj + +CC = wcc386 +LINKER = wcl386 +CFLAGS = -zq -mf -3r -fp3 -s -bt=dos -oilrtfm -fr=nul -wx +ZLIB_LIB = zlib_f.lib + +.C.OBJ: + $(CC) $(CFLAGS) $[@ + +all: $(ZLIB_LIB) example.exe minigzip.exe + +$(ZLIB_LIB): $(OBJS) + wlib -b -c $(ZLIB_LIB) -+adler32.obj -+compress.obj -+crc32.obj + wlib -b -c $(ZLIB_LIB) -+gzclose.obj -+gzlib.obj -+gzread.obj -+gzwrite.obj + wlib -b -c $(ZLIB_LIB) -+deflate.obj -+infback.obj + wlib -b -c $(ZLIB_LIB) -+inffast.obj -+inflate.obj -+inftrees.obj + wlib -b -c $(ZLIB_LIB) -+trees.obj -+uncompr.obj -+zutil.obj + +example.exe: $(ZLIB_LIB) example.obj + $(LINKER) -ldos32a -fe=example.exe example.obj $(ZLIB_LIB) + +minigzip.exe: $(ZLIB_LIB) minigzip.obj + $(LINKER) -ldos32a -fe=minigzip.exe minigzip.obj $(ZLIB_LIB) + +clean: .SYMBOLIC + del *.obj + del $(ZLIB_LIB) + @echo Cleaning done diff --git a/zlib/watcom/watcom_l.mak b/zlib/watcom/watcom_l.mak new file mode 100644 index 00000000000..193eed7b318 --- /dev/null +++ b/zlib/watcom/watcom_l.mak @@ -0,0 +1,43 @@ +# Makefile for zlib +# OpenWatcom large model +# Last updated: 28-Dec-2005 + +# To use, do "wmake -f watcom_l.mak" + +C_SOURCE = adler32.c compress.c crc32.c deflate.c & + gzclose.c gzlib.c gzread.c gzwrite.c & + infback.c inffast.c inflate.c inftrees.c & + trees.c uncompr.c zutil.c + +OBJS = adler32.obj compress.obj crc32.obj deflate.obj & + gzclose.obj gzlib.obj gzread.obj gzwrite.obj & + infback.obj inffast.obj inflate.obj inftrees.obj & + trees.obj uncompr.obj zutil.obj + +CC = wcc +LINKER = wcl +CFLAGS = -zq -ml -s -bt=dos -oilrtfm -fr=nul -wx +ZLIB_LIB = zlib_l.lib + +.C.OBJ: + $(CC) $(CFLAGS) $[@ + +all: $(ZLIB_LIB) example.exe minigzip.exe + +$(ZLIB_LIB): $(OBJS) + wlib -b -c $(ZLIB_LIB) -+adler32.obj -+compress.obj -+crc32.obj + wlib -b -c $(ZLIB_LIB) -+gzclose.obj -+gzlib.obj -+gzread.obj -+gzwrite.obj + wlib -b -c $(ZLIB_LIB) -+deflate.obj -+infback.obj + wlib -b -c $(ZLIB_LIB) -+inffast.obj -+inflate.obj -+inftrees.obj + wlib -b -c $(ZLIB_LIB) -+trees.obj -+uncompr.obj -+zutil.obj + +example.exe: $(ZLIB_LIB) example.obj + $(LINKER) -fe=example.exe example.obj $(ZLIB_LIB) + +minigzip.exe: $(ZLIB_LIB) minigzip.obj + $(LINKER) -fe=minigzip.exe minigzip.obj $(ZLIB_LIB) + +clean: .SYMBOLIC + del *.obj + del $(ZLIB_LIB) + @echo Cleaning done diff --git a/zlib/win32/DLL_FAQ.txt b/zlib/win32/DLL_FAQ.txt new file mode 100644 index 00000000000..12c009018c3 --- /dev/null +++ b/zlib/win32/DLL_FAQ.txt @@ -0,0 +1,397 @@ + + Frequently Asked Questions about ZLIB1.DLL + + +This document describes the design, the rationale, and the usage +of the official DLL build of zlib, named ZLIB1.DLL. If you have +general questions about zlib, you should see the file "FAQ" found +in the zlib distribution, or at the following location: + http://www.gzip.org/zlib/zlib_faq.html + + + 1. What is ZLIB1.DLL, and how can I get it? + + - ZLIB1.DLL is the official build of zlib as a DLL. + (Please remark the character '1' in the name.) + + Pointers to a precompiled ZLIB1.DLL can be found in the zlib + web site at: + http://www.zlib.net/ + + Applications that link to ZLIB1.DLL can rely on the following + specification: + + * The exported symbols are exclusively defined in the source + files "zlib.h" and "zlib.def", found in an official zlib + source distribution. + * The symbols are exported by name, not by ordinal. + * The exported names are undecorated. + * The calling convention of functions is "C" (CDECL). + * The ZLIB1.DLL binary is linked to MSVCRT.DLL. + + The archive in which ZLIB1.DLL is bundled contains compiled + test programs that must run with a valid build of ZLIB1.DLL. + It is recommended to download the prebuilt DLL from the zlib + web site, instead of building it yourself, to avoid potential + incompatibilities that could be introduced by your compiler + and build settings. If you do build the DLL yourself, please + make sure that it complies with all the above requirements, + and it runs with the precompiled test programs, bundled with + the original ZLIB1.DLL distribution. + + If, for any reason, you need to build an incompatible DLL, + please use a different file name. + + + 2. Why did you change the name of the DLL to ZLIB1.DLL? + What happened to the old ZLIB.DLL? + + - The old ZLIB.DLL, built from zlib-1.1.4 or earlier, required + compilation settings that were incompatible to those used by + a static build. The DLL settings were supposed to be enabled + by defining the macro ZLIB_DLL, before including "zlib.h". + Incorrect handling of this macro was silently accepted at + build time, resulting in two major problems: + + * ZLIB_DLL was missing from the old makefile. When building + the DLL, not all people added it to the build options. In + consequence, incompatible incarnations of ZLIB.DLL started + to circulate around the net. + + * When switching from using the static library to using the + DLL, applications had to define the ZLIB_DLL macro and + to recompile all the sources that contained calls to zlib + functions. Failure to do so resulted in creating binaries + that were unable to run with the official ZLIB.DLL build. + + The only possible solution that we could foresee was to make + a binary-incompatible change in the DLL interface, in order to + remove the dependency on the ZLIB_DLL macro, and to release + the new DLL under a different name. + + We chose the name ZLIB1.DLL, where '1' indicates the major + zlib version number. We hope that we will not have to break + the binary compatibility again, at least not as long as the + zlib-1.x series will last. + + There is still a ZLIB_DLL macro, that can trigger a more + efficient build and use of the DLL, but compatibility no + longer dependents on it. + + + 3. Can I build ZLIB.DLL from the new zlib sources, and replace + an old ZLIB.DLL, that was built from zlib-1.1.4 or earlier? + + - In principle, you can do it by assigning calling convention + keywords to the macros ZEXPORT and ZEXPORTVA. In practice, + it depends on what you mean by "an old ZLIB.DLL", because the + old DLL exists in several mutually-incompatible versions. + You have to find out first what kind of calling convention is + being used in your particular ZLIB.DLL build, and to use the + same one in the new build. If you don't know what this is all + about, you might be better off if you would just leave the old + DLL intact. + + + 4. Can I compile my application using the new zlib interface, and + link it to an old ZLIB.DLL, that was built from zlib-1.1.4 or + earlier? + + - The official answer is "no"; the real answer depends again on + what kind of ZLIB.DLL you have. Even if you are lucky, this + course of action is unreliable. + + If you rebuild your application and you intend to use a newer + version of zlib (post- 1.1.4), it is strongly recommended to + link it to the new ZLIB1.DLL. + + + 5. Why are the zlib symbols exported by name, and not by ordinal? + + - Although exporting symbols by ordinal is a little faster, it + is risky. Any single glitch in the maintenance or use of the + DEF file that contains the ordinals can result in incompatible + builds and frustrating crashes. Simply put, the benefits of + exporting symbols by ordinal do not justify the risks. + + Technically, it should be possible to maintain ordinals in + the DEF file, and still export the symbols by name. Ordinals + exist in every DLL, and even if the dynamic linking performed + at the DLL startup is searching for names, ordinals serve as + hints, for a faster name lookup. However, if the DEF file + contains ordinals, the Microsoft linker automatically builds + an implib that will cause the executables linked to it to use + those ordinals, and not the names. It is interesting to + notice that the GNU linker for Win32 does not suffer from this + problem. + + It is possible to avoid the DEF file if the exported symbols + are accompanied by a "__declspec(dllexport)" attribute in the + source files. You can do this in zlib by predefining the + ZLIB_DLL macro. + + + 6. I see that the ZLIB1.DLL functions use the "C" (CDECL) calling + convention. Why not use the STDCALL convention? + STDCALL is the standard convention in Win32, and I need it in + my Visual Basic project! + + (For readability, we use CDECL to refer to the convention + triggered by the "__cdecl" keyword, STDCALL to refer to + the convention triggered by "__stdcall", and FASTCALL to + refer to the convention triggered by "__fastcall".) + + - Most of the native Windows API functions (without varargs) use + indeed the WINAPI convention (which translates to STDCALL in + Win32), but the standard C functions use CDECL. If a user + application is intrinsically tied to the Windows API (e.g. + it calls native Windows API functions such as CreateFile()), + sometimes it makes sense to decorate its own functions with + WINAPI. But if ANSI C or POSIX portability is a goal (e.g. + it calls standard C functions such as fopen()), it is not a + sound decision to request the inclusion of , or to + use non-ANSI constructs, for the sole purpose to make the user + functions STDCALL-able. + + The functionality offered by zlib is not in the category of + "Windows functionality", but is more like "C functionality". + + Technically, STDCALL is not bad; in fact, it is slightly + faster than CDECL, and it works with variable-argument + functions, just like CDECL. It is unfortunate that, in spite + of using STDCALL in the Windows API, it is not the default + convention used by the C compilers that run under Windows. + The roots of the problem reside deep inside the unsafety of + the K&R-style function prototypes, where the argument types + are not specified; but that is another story for another day. + + The remaining fact is that CDECL is the default convention. + Even if an explicit convention is hard-coded into the function + prototypes inside C headers, problems may appear. The + necessity to expose the convention in users' callbacks is one + of these problems. + + The calling convention issues are also important when using + zlib in other programming languages. Some of them, like Ada + (GNAT) and Fortran (GNU G77), have C bindings implemented + initially on Unix, and relying on the C calling convention. + On the other hand, the pre- .NET versions of Microsoft Visual + Basic require STDCALL, while Borland Delphi prefers, although + it does not require, FASTCALL. + + In fairness to all possible uses of zlib outside the C + programming language, we choose the default "C" convention. + Anyone interested in different bindings or conventions is + encouraged to maintain specialized projects. The "contrib/" + directory from the zlib distribution already holds a couple + of foreign bindings, such as Ada, C++, and Delphi. + + + 7. I need a DLL for my Visual Basic project. What can I do? + + - Define the ZLIB_WINAPI macro before including "zlib.h", when + building both the DLL and the user application (except that + you don't need to define anything when using the DLL in Visual + Basic). The ZLIB_WINAPI macro will switch on the WINAPI + (STDCALL) convention. The name of this DLL must be different + than the official ZLIB1.DLL. + + Gilles Vollant has contributed a build named ZLIBWAPI.DLL, + with the ZLIB_WINAPI macro turned on, and with the minizip + functionality built in. For more information, please read + the notes inside "contrib/vstudio/readme.txt", found in the + zlib distribution. + + + 8. I need to use zlib in my Microsoft .NET project. What can I + do? + + - Henrik Ravn has contributed a .NET wrapper around zlib. Look + into contrib/dotzlib/, inside the zlib distribution. + + + 9. If my application uses ZLIB1.DLL, should I link it to + MSVCRT.DLL? Why? + + - It is not required, but it is recommended to link your + application to MSVCRT.DLL, if it uses ZLIB1.DLL. + + The executables (.EXE, .DLL, etc.) that are involved in the + same process and are using the C run-time library (i.e. they + are calling standard C functions), must link to the same + library. There are several libraries in the Win32 system: + CRTDLL.DLL, MSVCRT.DLL, the static C libraries, etc. + Since ZLIB1.DLL is linked to MSVCRT.DLL, the executables that + depend on it should also be linked to MSVCRT.DLL. + + +10. Why are you saying that ZLIB1.DLL and my application should + be linked to the same C run-time (CRT) library? I linked my + application and my DLLs to different C libraries (e.g. my + application to a static library, and my DLLs to MSVCRT.DLL), + and everything works fine. + + - If a user library invokes only pure Win32 API (accessible via + and the related headers), its DLL build will work + in any context. But if this library invokes standard C API, + things get more complicated. + + There is a single Win32 library in a Win32 system. Every + function in this library resides in a single DLL module, that + is safe to call from anywhere. On the other hand, there are + multiple versions of the C library, and each of them has its + own separate internal state. Standalone executables and user + DLLs that call standard C functions must link to a C run-time + (CRT) library, be it static or shared (DLL). Intermixing + occurs when an executable (not necessarily standalone) and a + DLL are linked to different CRTs, and both are running in the + same process. + + Intermixing multiple CRTs is possible, as long as their + internal states are kept intact. The Microsoft Knowledge Base + articles KB94248 "HOWTO: Use the C Run-Time" and KB140584 + "HOWTO: Link with the Correct C Run-Time (CRT) Library" + mention the potential problems raised by intermixing. + + If intermixing works for you, it's because your application + and DLLs are avoiding the corruption of each of the CRTs' + internal states, maybe by careful design, or maybe by fortune. + + Also note that linking ZLIB1.DLL to non-Microsoft CRTs, such + as those provided by Borland, raises similar problems. + + +11. Why are you linking ZLIB1.DLL to MSVCRT.DLL? + + - MSVCRT.DLL exists on every Windows 95 with a new service pack + installed, or with Microsoft Internet Explorer 4 or later, and + on all other Windows 4.x or later (Windows 98, Windows NT 4, + or later). It is freely distributable; if not present in the + system, it can be downloaded from Microsoft or from other + software provider for free. + + The fact that MSVCRT.DLL does not exist on a virgin Windows 95 + is not so problematic. Windows 95 is scarcely found nowadays, + Microsoft ended its support a long time ago, and many recent + applications from various vendors, including Microsoft, do not + even run on it. Furthermore, no serious user should run + Windows 95 without a proper update installed. + + +12. Why are you not linking ZLIB1.DLL to + <> ? + + - We considered and abandoned the following alternatives: + + * Linking ZLIB1.DLL to a static C library (LIBC.LIB, or + LIBCMT.LIB) is not a good option. People are using the DLL + mainly to save disk space. If you are linking your program + to a static C library, you may as well consider linking zlib + in statically, too. + + * Linking ZLIB1.DLL to CRTDLL.DLL looks appealing, because + CRTDLL.DLL is present on every Win32 installation. + Unfortunately, it has a series of problems: it does not + work properly with Microsoft's C++ libraries, it does not + provide support for 64-bit file offsets, (and so on...), + and Microsoft discontinued its support a long time ago. + + * Linking ZLIB1.DLL to MSVCR70.DLL or MSVCR71.DLL, supplied + with the Microsoft .NET platform, and Visual C++ 7.0/7.1, + raises problems related to the status of ZLIB1.DLL as a + system component. According to the Microsoft Knowledge Base + article KB326922 "INFO: Redistribution of the Shared C + Runtime Component in Visual C++ .NET", MSVCR70.DLL and + MSVCR71.DLL are not supposed to function as system DLLs, + because they may clash with MSVCRT.DLL. Instead, the + application's installer is supposed to put these DLLs + (if needed) in the application's private directory. + If ZLIB1.DLL depends on a non-system runtime, it cannot + function as a redistributable system component. + + * Linking ZLIB1.DLL to non-Microsoft runtimes, such as + Borland's, or Cygwin's, raises problems related to the + reliable presence of these runtimes on Win32 systems. + It's easier to let the DLL build of zlib up to the people + who distribute these runtimes, and who may proceed as + explained in the answer to Question 14. + + +13. If ZLIB1.DLL cannot be linked to MSVCR70.DLL or MSVCR71.DLL, + how can I build/use ZLIB1.DLL in Microsoft Visual C++ 7.0 + (Visual Studio .NET) or newer? + + - Due to the problems explained in the Microsoft Knowledge Base + article KB326922 (see the previous answer), the C runtime that + comes with the VC7 environment is no longer considered a + system component. That is, it should not be assumed that this + runtime exists, or may be installed in a system directory. + Since ZLIB1.DLL is supposed to be a system component, it may + not depend on a non-system component. + + In order to link ZLIB1.DLL and your application to MSVCRT.DLL + in VC7, you need the library of Visual C++ 6.0 or older. If + you don't have this library at hand, it's probably best not to + use ZLIB1.DLL. + + We are hoping that, in the future, Microsoft will provide a + way to build applications linked to a proper system runtime, + from the Visual C++ environment. Until then, you have a + couple of alternatives, such as linking zlib in statically. + If your application requires dynamic linking, you may proceed + as explained in the answer to Question 14. + + +14. I need to link my own DLL build to a CRT different than + MSVCRT.DLL. What can I do? + + - Feel free to rebuild the DLL from the zlib sources, and link + it the way you want. You should, however, clearly state that + your build is unofficial. You should give it a different file + name, and/or install it in a private directory that can be + accessed by your application only, and is not visible to the + others (i.e. it's neither in the PATH, nor in the SYSTEM or + SYSTEM32 directories). Otherwise, your build may clash with + applications that link to the official build. + + For example, in Cygwin, zlib is linked to the Cygwin runtime + CYGWIN1.DLL, and it is distributed under the name CYGZ.DLL. + + +15. May I include additional pieces of code that I find useful, + link them in ZLIB1.DLL, and export them? + + - No. A legitimate build of ZLIB1.DLL must not include code + that does not originate from the official zlib source code. + But you can make your own private DLL build, under a different + file name, as suggested in the previous answer. + + For example, zlib is a part of the VCL library, distributed + with Borland Delphi and C++ Builder. The DLL build of VCL + is a redistributable file, named VCLxx.DLL. + + +16. May I remove some functionality out of ZLIB1.DLL, by enabling + macros like NO_GZCOMPRESS or NO_GZIP at compile time? + + - No. A legitimate build of ZLIB1.DLL must provide the complete + zlib functionality, as implemented in the official zlib source + code. But you can make your own private DLL build, under a + different file name, as suggested in the previous answer. + + +17. I made my own ZLIB1.DLL build. Can I test it for compliance? + + - We prefer that you download the official DLL from the zlib + web site. If you need something peculiar from this DLL, you + can send your suggestion to the zlib mailing list. + + However, in case you do rebuild the DLL yourself, you can run + it with the test programs found in the DLL distribution. + Running these test programs is not a guarantee of compliance, + but a failure can imply a detected problem. + +** + +This document is written and maintained by +Cosmin Truta diff --git a/zlib/win32/Makefile.bor b/zlib/win32/Makefile.bor new file mode 100644 index 00000000000..d152bbb7ffd --- /dev/null +++ b/zlib/win32/Makefile.bor @@ -0,0 +1,110 @@ +# Makefile for zlib +# Borland C++ for Win32 +# +# Usage: +# make -f win32/Makefile.bor +# make -f win32/Makefile.bor LOCAL_ZLIB=-DASMV OBJA=match.obj OBJPA=+match.obj + +# ------------ Borland C++ ------------ + +# Optional nonstandard preprocessor flags (e.g. -DMAX_MEM_LEVEL=7) +# should be added to the environment via "set LOCAL_ZLIB=-DFOO" or +# added to the declaration of LOC here: +LOC = $(LOCAL_ZLIB) + +CC = bcc32 +AS = bcc32 +LD = bcc32 +AR = tlib +CFLAGS = -a -d -k- -O2 $(LOC) +ASFLAGS = $(LOC) +LDFLAGS = $(LOC) + + +# variables +ZLIB_LIB = zlib.lib + +OBJ1 = adler32.obj compress.obj crc32.obj deflate.obj gzclose.obj gzlib.obj gzread.obj +OBJ2 = gzwrite.obj infback.obj inffast.obj inflate.obj inftrees.obj trees.obj uncompr.obj zutil.obj +#OBJA = +OBJP1 = +adler32.obj+compress.obj+crc32.obj+deflate.obj+gzclose.obj+gzlib.obj+gzread.obj +OBJP2 = +gzwrite.obj+infback.obj+inffast.obj+inflate.obj+inftrees.obj+trees.obj+uncompr.obj+zutil.obj +#OBJPA= + + +# targets +all: $(ZLIB_LIB) example.exe minigzip.exe + +.c.obj: + $(CC) -c $(CFLAGS) $< + +.asm.obj: + $(AS) -c $(ASFLAGS) $< + +adler32.obj: adler32.c zlib.h zconf.h + +compress.obj: compress.c zlib.h zconf.h + +crc32.obj: crc32.c zlib.h zconf.h crc32.h + +deflate.obj: deflate.c deflate.h zutil.h zlib.h zconf.h + +gzclose.obj: gzclose.c zlib.h zconf.h gzguts.h + +gzlib.obj: gzlib.c zlib.h zconf.h gzguts.h + +gzread.obj: gzread.c zlib.h zconf.h gzguts.h + +gzwrite.obj: gzwrite.c zlib.h zconf.h gzguts.h + +infback.obj: infback.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inffast.obj: inffast.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h + +inflate.obj: inflate.c zutil.h zlib.h zconf.h inftrees.h inflate.h \ + inffast.h inffixed.h + +inftrees.obj: inftrees.c zutil.h zlib.h zconf.h inftrees.h + +trees.obj: trees.c zutil.h zlib.h zconf.h deflate.h trees.h + +uncompr.obj: uncompr.c zlib.h zconf.h + +zutil.obj: zutil.c zutil.h zlib.h zconf.h + +example.obj: test/example.c zlib.h zconf.h + +minigzip.obj: test/minigzip.c zlib.h zconf.h + + +# For the sake of the old Borland make, +# the command line is cut to fit in the MS-DOS 128 byte limit: +$(ZLIB_LIB): $(OBJ1) $(OBJ2) $(OBJA) + -del $(ZLIB_LIB) + $(AR) $(ZLIB_LIB) $(OBJP1) + $(AR) $(ZLIB_LIB) $(OBJP2) + $(AR) $(ZLIB_LIB) $(OBJPA) + + +# testing +test: example.exe minigzip.exe + example + echo hello world | minigzip | minigzip -d + +example.exe: example.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) example.obj $(ZLIB_LIB) + +minigzip.exe: minigzip.obj $(ZLIB_LIB) + $(LD) $(LDFLAGS) minigzip.obj $(ZLIB_LIB) + + +# cleanup +clean: + -del $(ZLIB_LIB) + -del *.obj + -del *.exe + -del *.tds + -del zlib.bak + -del foo.gz diff --git a/zlib/win32/Makefile.gcc b/zlib/win32/Makefile.gcc new file mode 100644 index 00000000000..305be50afef --- /dev/null +++ b/zlib/win32/Makefile.gcc @@ -0,0 +1,182 @@ +# Makefile for zlib, derived from Makefile.dj2. +# Modified for mingw32 by C. Spieler, 6/16/98. +# Updated for zlib 1.2.x by Christian Spieler and Cosmin Truta, Mar-2003. +# Last updated: Mar 2012. +# Tested under Cygwin and MinGW. + +# Copyright (C) 1995-2003 Jean-loup Gailly. +# For conditions of distribution and use, see copyright notice in zlib.h + +# To compile, or to compile and test, type from the top level zlib directory: +# +# make -fwin32/Makefile.gcc; make test testdll -fwin32/Makefile.gcc +# +# To use the asm code, type: +# cp contrib/asm?86/match.S ./match.S +# make LOC=-DASMV OBJA=match.o -fwin32/Makefile.gcc +# +# To install libz.a, zconf.h and zlib.h in the system directories, type: +# +# make install -fwin32/Makefile.gcc +# +# BINARY_PATH, INCLUDE_PATH and LIBRARY_PATH must be set. +# +# To install the shared lib, append SHARED_MODE=1 to the make command : +# +# make install -fwin32/Makefile.gcc SHARED_MODE=1 + +# Note: +# If the platform is *not* MinGW (e.g. it is Cygwin or UWIN), +# the DLL name should be changed from "zlib1.dll". + +STATICLIB = libz.a +SHAREDLIB = zlib1.dll +IMPLIB = libz.dll.a + +# +# Set to 1 if shared object needs to be installed +# +SHARED_MODE=0 + +#LOC = -DASMV +#LOC = -DZLIB_DEBUG -g + +PREFIX = +CC = $(PREFIX)gcc +CFLAGS = $(LOC) -O3 -Wall + +AS = $(CC) +ASFLAGS = $(LOC) -Wall + +LD = $(CC) +LDFLAGS = $(LOC) + +AR = $(PREFIX)ar +ARFLAGS = rcs + +RC = $(PREFIX)windres +RCFLAGS = --define GCC_WINDRES + +STRIP = $(PREFIX)strip + +CP = cp -fp +# If GNU install is available, replace $(CP) with install. +INSTALL = $(CP) +RM = rm -f + +prefix ?= /usr/local +exec_prefix = $(prefix) + +OBJS = adler32.o compress.o crc32.o deflate.o gzclose.o gzlib.o gzread.o \ + gzwrite.o infback.o inffast.o inflate.o inftrees.o trees.o uncompr.o zutil.o +OBJA = + +all: $(STATICLIB) $(SHAREDLIB) $(IMPLIB) example.exe minigzip.exe example_d.exe minigzip_d.exe + +test: example.exe minigzip.exe + ./example + echo hello world | ./minigzip | ./minigzip -d + +testdll: example_d.exe minigzip_d.exe + ./example_d + echo hello world | ./minigzip_d | ./minigzip_d -d + +.c.o: + $(CC) $(CFLAGS) -c -o $@ $< + +.S.o: + $(AS) $(ASFLAGS) -c -o $@ $< + +$(STATICLIB): $(OBJS) $(OBJA) + $(AR) $(ARFLAGS) $@ $(OBJS) $(OBJA) + +$(IMPLIB): $(SHAREDLIB) + +$(SHAREDLIB): win32/zlib.def $(OBJS) $(OBJA) zlibrc.o + $(CC) -shared -Wl,--out-implib,$(IMPLIB) $(LDFLAGS) \ + -o $@ win32/zlib.def $(OBJS) $(OBJA) zlibrc.o + $(STRIP) $@ + +example.exe: example.o $(STATICLIB) + $(LD) $(LDFLAGS) -o $@ example.o $(STATICLIB) + $(STRIP) $@ + +minigzip.exe: minigzip.o $(STATICLIB) + $(LD) $(LDFLAGS) -o $@ minigzip.o $(STATICLIB) + $(STRIP) $@ + +example_d.exe: example.o $(IMPLIB) + $(LD) $(LDFLAGS) -o $@ example.o $(IMPLIB) + $(STRIP) $@ + +minigzip_d.exe: minigzip.o $(IMPLIB) + $(LD) $(LDFLAGS) -o $@ minigzip.o $(IMPLIB) + $(STRIP) $@ + +example.o: test/example.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -c -o $@ test/example.c + +minigzip.o: test/minigzip.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -c -o $@ test/minigzip.c + +zlibrc.o: win32/zlib1.rc + $(RC) $(RCFLAGS) -o $@ win32/zlib1.rc + +.PHONY: install uninstall clean + +install: zlib.h zconf.h $(STATICLIB) $(IMPLIB) + @if test -z "$(DESTDIR)$(INCLUDE_PATH)" -o -z "$(DESTDIR)$(LIBRARY_PATH)" -o -z "$(DESTDIR)$(BINARY_PATH)"; then \ + echo INCLUDE_PATH, LIBRARY_PATH, and BINARY_PATH must be specified; \ + exit 1; \ + fi + -@mkdir -p '$(DESTDIR)$(INCLUDE_PATH)' + -@mkdir -p '$(DESTDIR)$(LIBRARY_PATH)' '$(DESTDIR)$(LIBRARY_PATH)'/pkgconfig + -if [ "$(SHARED_MODE)" = "1" ]; then \ + mkdir -p '$(DESTDIR)$(BINARY_PATH)'; \ + $(INSTALL) $(SHAREDLIB) '$(DESTDIR)$(BINARY_PATH)'; \ + $(INSTALL) $(IMPLIB) '$(DESTDIR)$(LIBRARY_PATH)'; \ + fi + -$(INSTALL) zlib.h '$(DESTDIR)$(INCLUDE_PATH)' + -$(INSTALL) zconf.h '$(DESTDIR)$(INCLUDE_PATH)' + -$(INSTALL) $(STATICLIB) '$(DESTDIR)$(LIBRARY_PATH)' + sed \ + -e 's|@prefix@|${prefix}|g' \ + -e 's|@exec_prefix@|${exec_prefix}|g' \ + -e 's|@libdir@|$(LIBRARY_PATH)|g' \ + -e 's|@sharedlibdir@|$(LIBRARY_PATH)|g' \ + -e 's|@includedir@|$(INCLUDE_PATH)|g' \ + -e 's|@VERSION@|'`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' zlib.h`'|g' \ + zlib.pc.in > '$(DESTDIR)$(LIBRARY_PATH)'/pkgconfig/zlib.pc + +uninstall: + -if [ "$(SHARED_MODE)" = "1" ]; then \ + $(RM) '$(DESTDIR)$(BINARY_PATH)'/$(SHAREDLIB); \ + $(RM) '$(DESTDIR)$(LIBRARY_PATH)'/$(IMPLIB); \ + fi + -$(RM) '$(DESTDIR)$(INCLUDE_PATH)'/zlib.h + -$(RM) '$(DESTDIR)$(INCLUDE_PATH)'/zconf.h + -$(RM) '$(DESTDIR)$(LIBRARY_PATH)'/$(STATICLIB) + +clean: + -$(RM) $(STATICLIB) + -$(RM) $(SHAREDLIB) + -$(RM) $(IMPLIB) + -$(RM) *.o + -$(RM) *.exe + -$(RM) foo.gz + +adler32.o: zlib.h zconf.h +compress.o: zlib.h zconf.h +crc32.o: crc32.h zlib.h zconf.h +deflate.o: deflate.h zutil.h zlib.h zconf.h +gzclose.o: zlib.h zconf.h gzguts.h +gzlib.o: zlib.h zconf.h gzguts.h +gzread.o: zlib.h zconf.h gzguts.h +gzwrite.o: zlib.h zconf.h gzguts.h +inffast.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +inflate.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +infback.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +inftrees.o: zutil.h zlib.h zconf.h inftrees.h +trees.o: deflate.h zutil.h zlib.h zconf.h trees.h +uncompr.o: zlib.h zconf.h +zutil.o: zutil.h zlib.h zconf.h diff --git a/zlib/win32/Makefile.msc b/zlib/win32/Makefile.msc new file mode 100644 index 00000000000..6831882de43 --- /dev/null +++ b/zlib/win32/Makefile.msc @@ -0,0 +1,163 @@ +# Makefile for zlib using Microsoft (Visual) C +# zlib is copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler +# +# Usage: +# nmake -f win32/Makefile.msc (standard build) +# nmake -f win32/Makefile.msc LOC=-DFOO (nonstandard build) +# nmake -f win32/Makefile.msc LOC="-DASMV -DASMINF" \ +# OBJA="inffas32.obj match686.obj" (use ASM code, x86) +# nmake -f win32/Makefile.msc AS=ml64 LOC="-DASMV -DASMINF -I." \ +# OBJA="inffasx64.obj gvmat64.obj inffas8664.obj" (use ASM code, x64) + +# The toplevel directory of the source tree. +# +TOP = . + +# optional build flags +LOC = + +# variables +STATICLIB = zlib.lib +SHAREDLIB = zlib1.dll +IMPLIB = zdll.lib + +CC = cl +AS = ml +LD = link +AR = lib +RC = rc +CFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi -Fd"zlib" $(LOC) +WFLAGS = -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE +ASFLAGS = -coff -Zi $(LOC) +LDFLAGS = -nologo -debug -incremental:no -opt:ref +ARFLAGS = -nologo +RCFLAGS = /dWIN32 /r + +OBJS = adler32.obj compress.obj crc32.obj deflate.obj gzclose.obj gzlib.obj gzread.obj \ + gzwrite.obj infback.obj inflate.obj inftrees.obj inffast.obj trees.obj uncompr.obj zutil.obj +OBJA = + + +# targets +all: $(STATICLIB) $(SHAREDLIB) $(IMPLIB) \ + example.exe minigzip.exe example_d.exe minigzip_d.exe + +$(STATICLIB): $(OBJS) $(OBJA) + $(AR) $(ARFLAGS) -out:$@ $(OBJS) $(OBJA) + +$(IMPLIB): $(SHAREDLIB) + +$(SHAREDLIB): $(TOP)/win32/zlib.def $(OBJS) $(OBJA) zlib1.res + $(LD) $(LDFLAGS) -def:$(TOP)/win32/zlib.def -dll -implib:$(IMPLIB) \ + -out:$@ -base:0x5A4C0000 $(OBJS) $(OBJA) zlib1.res + if exist $@.manifest \ + mt -nologo -manifest $@.manifest -outputresource:$@;2 + +example.exe: example.obj $(STATICLIB) + $(LD) $(LDFLAGS) example.obj $(STATICLIB) + if exist $@.manifest \ + mt -nologo -manifest $@.manifest -outputresource:$@;1 + +minigzip.exe: minigzip.obj $(STATICLIB) + $(LD) $(LDFLAGS) minigzip.obj $(STATICLIB) + if exist $@.manifest \ + mt -nologo -manifest $@.manifest -outputresource:$@;1 + +example_d.exe: example.obj $(IMPLIB) + $(LD) $(LDFLAGS) -out:$@ example.obj $(IMPLIB) + if exist $@.manifest \ + mt -nologo -manifest $@.manifest -outputresource:$@;1 + +minigzip_d.exe: minigzip.obj $(IMPLIB) + $(LD) $(LDFLAGS) -out:$@ minigzip.obj $(IMPLIB) + if exist $@.manifest \ + mt -nologo -manifest $@.manifest -outputresource:$@;1 + +{$(TOP)}.c.obj: + $(CC) -c $(WFLAGS) $(CFLAGS) $< + +{$(TOP)/test}.c.obj: + $(CC) -c -I$(TOP) $(WFLAGS) $(CFLAGS) $< + +{$(TOP)/contrib/masmx64}.c.obj: + $(CC) -c $(WFLAGS) $(CFLAGS) $< + +{$(TOP)/contrib/masmx64}.asm.obj: + $(AS) -c $(ASFLAGS) $< + +{$(TOP)/contrib/masmx86}.asm.obj: + $(AS) -c $(ASFLAGS) $< + +adler32.obj: $(TOP)/adler32.c $(TOP)/zlib.h $(TOP)/zconf.h + +compress.obj: $(TOP)/compress.c $(TOP)/zlib.h $(TOP)/zconf.h + +crc32.obj: $(TOP)/crc32.c $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/crc32.h + +deflate.obj: $(TOP)/deflate.c $(TOP)/deflate.h $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h + +gzclose.obj: $(TOP)/gzclose.c $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/gzguts.h + +gzlib.obj: $(TOP)/gzlib.c $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/gzguts.h + +gzread.obj: $(TOP)/gzread.c $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/gzguts.h + +gzwrite.obj: $(TOP)/gzwrite.c $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/gzguts.h + +infback.obj: $(TOP)/infback.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/inftrees.h $(TOP)/inflate.h \ + $(TOP)/inffast.h $(TOP)/inffixed.h + +inffast.obj: $(TOP)/inffast.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/inftrees.h $(TOP)/inflate.h \ + $(TOP)/inffast.h + +inflate.obj: $(TOP)/inflate.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/inftrees.h $(TOP)/inflate.h \ + $(TOP)/inffast.h $(TOP)/inffixed.h + +inftrees.obj: $(TOP)/inftrees.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/inftrees.h + +trees.obj: $(TOP)/trees.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h $(TOP)/deflate.h $(TOP)/trees.h + +uncompr.obj: $(TOP)/uncompr.c $(TOP)/zlib.h $(TOP)/zconf.h + +zutil.obj: $(TOP)/zutil.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h + +gvmat64.obj: $(TOP)/contrib\masmx64\gvmat64.asm + +inffasx64.obj: $(TOP)/contrib\masmx64\inffasx64.asm + +inffas8664.obj: $(TOP)/contrib\masmx64\inffas8664.c $(TOP)/zutil.h $(TOP)/zlib.h $(TOP)/zconf.h \ + $(TOP)/inftrees.h $(TOP)/inflate.h $(TOP)/inffast.h + +inffas32.obj: $(TOP)/contrib\masmx86\inffas32.asm + +match686.obj: $(TOP)/contrib\masmx86\match686.asm + +example.obj: $(TOP)/test/example.c $(TOP)/zlib.h $(TOP)/zconf.h + +minigzip.obj: $(TOP)/test/minigzip.c $(TOP)/zlib.h $(TOP)/zconf.h + +zlib1.res: $(TOP)/win32/zlib1.rc + $(RC) $(RCFLAGS) /fo$@ $(TOP)/win32/zlib1.rc + +# testing +test: example.exe minigzip.exe + example + echo hello world | minigzip | minigzip -d + +testdll: example_d.exe minigzip_d.exe + example_d + echo hello world | minigzip_d | minigzip_d -d + + +# cleanup +clean: + -del $(STATICLIB) + -del $(SHAREDLIB) + -del $(IMPLIB) + -del *.obj + -del *.res + -del *.exp + -del *.exe + -del *.pdb + -del *.manifest + -del foo.gz diff --git a/zlib/win32/README-WIN32.txt b/zlib/win32/README-WIN32.txt new file mode 100644 index 00000000000..df7ab7f4b34 --- /dev/null +++ b/zlib/win32/README-WIN32.txt @@ -0,0 +1,103 @@ +ZLIB DATA COMPRESSION LIBRARY + +zlib 1.2.11 is a general purpose data compression library. All the code is +thread safe. The data format used by the zlib library is described by RFCs +(Request for Comments) 1950 to 1952 in the files +http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) +and rfc1952.txt (gzip format). + +All functions of the compression library are documented in the file zlib.h +(volunteer to write man pages welcome, contact zlib@gzip.org). Two compiled +examples are distributed in this package, example and minigzip. The example_d +and minigzip_d flavors validate that the zlib1.dll file is working correctly. + +Questions about zlib should be sent to . The zlib home page +is http://zlib.net/ . Before reporting a problem, please check this site to +verify that you have the latest version of zlib; otherwise get the latest +version and check whether the problem still exists or not. + +PLEASE read DLL_FAQ.txt, and the the zlib FAQ http://zlib.net/zlib_faq.html +before asking for help. + + +Manifest: + +The package zlib-1.2.11-win32-x86.zip will contain the following files: + + README-WIN32.txt This document + ChangeLog Changes since previous zlib packages + DLL_FAQ.txt Frequently asked questions about zlib1.dll + zlib.3.pdf Documentation of this library in Adobe Acrobat format + + example.exe A statically-bound example (using zlib.lib, not the dll) + example.pdb Symbolic information for debugging example.exe + + example_d.exe A zlib1.dll bound example (using zdll.lib) + example_d.pdb Symbolic information for debugging example_d.exe + + minigzip.exe A statically-bound test program (using zlib.lib, not the dll) + minigzip.pdb Symbolic information for debugging minigzip.exe + + minigzip_d.exe A zlib1.dll bound test program (using zdll.lib) + minigzip_d.pdb Symbolic information for debugging minigzip_d.exe + + zlib.h Install these files into the compilers' INCLUDE path to + zconf.h compile programs which use zlib.lib or zdll.lib + + zdll.lib Install these files into the compilers' LIB path if linking + zdll.exp a compiled program to the zlib1.dll binary + + zlib.lib Install these files into the compilers' LIB path to link zlib + zlib.pdb into compiled programs, without zlib1.dll runtime dependency + (zlib.pdb provides debugging info to the compile time linker) + + zlib1.dll Install this binary shared library into the system PATH, or + the program's runtime directory (where the .exe resides) + zlib1.pdb Install in the same directory as zlib1.dll, in order to debug + an application crash using WinDbg or similar tools. + +All .pdb files above are entirely optional, but are very useful to a developer +attempting to diagnose program misbehavior or a crash. Many additional +important files for developers can be found in the zlib127.zip source package +available from http://zlib.net/ - review that package's README file for details. + + +Acknowledgments: + +The deflate format used by zlib was defined by Phil Katz. The deflate and +zlib specifications were written by L. Peter Deutsch. Thanks to all the +people who reported problems and suggested various improvements in zlib; they +are too numerous to cite here. + + +Copyright notice: + + (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +If you use the zlib library in a product, we would appreciate *not* receiving +lengthy legal documents to sign. The sources are provided for free but without +warranty of any kind. The library has been entirely written by Jean-loup +Gailly and Mark Adler; it does not include third-party code. + +If you redistribute modified sources, we would appreciate that you include in +the file ChangeLog history information documenting your changes. Please read +the FAQ for more information on the distribution of modified source versions. diff --git a/zlib/win32/VisualC.txt b/zlib/win32/VisualC.txt new file mode 100644 index 00000000000..1005b219410 --- /dev/null +++ b/zlib/win32/VisualC.txt @@ -0,0 +1,3 @@ + +To build zlib using the Microsoft Visual C++ environment, +use the appropriate project from the contrib/vstudio/ directory. diff --git a/zlib/win32/zlib.def b/zlib/win32/zlib.def new file mode 100644 index 00000000000..a2188b00062 --- /dev/null +++ b/zlib/win32/zlib.def @@ -0,0 +1,94 @@ +; zlib data compression library +EXPORTS +; basic functions + zlibVersion + deflate + deflateEnd + inflate + inflateEnd +; advanced functions + deflateSetDictionary + deflateGetDictionary + deflateCopy + deflateReset + deflateParams + deflateTune + deflateBound + deflatePending + deflatePrime + deflateSetHeader + inflateSetDictionary + inflateGetDictionary + inflateSync + inflateCopy + inflateReset + inflateReset2 + inflatePrime + inflateMark + inflateGetHeader + inflateBack + inflateBackEnd + zlibCompileFlags +; utility functions + compress + compress2 + compressBound + uncompress + uncompress2 + gzopen + gzdopen + gzbuffer + gzsetparams + gzread + gzfread + gzwrite + gzfwrite + gzprintf + gzvprintf + gzputs + gzgets + gzputc + gzgetc + gzungetc + gzflush + gzseek + gzrewind + gztell + gzoffset + gzeof + gzdirect + gzclose + gzclose_r + gzclose_w + gzerror + gzclearerr +; large file functions + gzopen64 + gzseek64 + gztell64 + gzoffset64 + adler32_combine64 + crc32_combine64 +; checksum functions + adler32 + adler32_z + crc32 + crc32_z + adler32_combine + crc32_combine +; various hacks, don't look :) + deflateInit_ + deflateInit2_ + inflateInit_ + inflateInit2_ + inflateBackInit_ + gzgetc_ + zError + inflateSyncPoint + get_crc_table + inflateUndermine + inflateValidate + inflateCodesUsed + inflateResetKeep + deflateResetKeep + gzopen_w diff --git a/zlib/win32/zlib1.rc b/zlib/win32/zlib1.rc new file mode 100644 index 00000000000..234e641c329 --- /dev/null +++ b/zlib/win32/zlib1.rc @@ -0,0 +1,40 @@ +#include +#include "../zlib.h" + +#ifdef GCC_WINDRES +VS_VERSION_INFO VERSIONINFO +#else +VS_VERSION_INFO VERSIONINFO MOVEABLE IMPURE LOADONCALL DISCARDABLE +#endif + FILEVERSION ZLIB_VER_MAJOR,ZLIB_VER_MINOR,ZLIB_VER_REVISION,0 + PRODUCTVERSION ZLIB_VER_MAJOR,ZLIB_VER_MINOR,ZLIB_VER_REVISION,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS 1 +#else + FILEFLAGS 0 +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0 // not used +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + //language ID = U.S. English, char set = Windows, Multilingual + BEGIN + VALUE "FileDescription", "zlib data compression library\0" + VALUE "FileVersion", ZLIB_VERSION "\0" + VALUE "InternalName", "zlib1.dll\0" + VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0" + VALUE "OriginalFilename", "zlib1.dll\0" + VALUE "ProductName", "zlib\0" + VALUE "ProductVersion", ZLIB_VERSION "\0" + VALUE "Comments", "For more information visit http://www.zlib.net/\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x0409, 1252 + END +END diff --git a/zlib/zconf.h.cmakein b/zlib/zconf.h.cmakein new file mode 100644 index 00000000000..a7f24cce60f --- /dev/null +++ b/zlib/zconf.h.cmakein @@ -0,0 +1,536 @@ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#ifndef ZCONF_H +#define ZCONF_H +#cmakedefine Z_PREFIX +#cmakedefine Z_HAVE_UNISTD_H + +/* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". + */ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z +# define deflate z_deflate +# define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending +# define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 +# define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep +# define inflateSetDictionary z_inflateSetDictionary +# define inflateSync z_inflateSync +# define inflateSyncPoint z_inflateSyncPoint +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif +# define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion + +/* all zlib typedefs in zlib.h and zconf.h */ +# define Byte z_Byte +# define Bytef z_Bytef +# define alloc_func z_alloc_func +# define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func +# define intf z_intf +# define out_func z_out_func +# define uInt z_uInt +# define uIntf z_uIntf +# define uLong z_uLong +# define uLongf z_uLongf +# define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + +#endif + +#if defined(__MSDOS__) && !defined(MSDOS) +# define MSDOS +#endif +#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) +# define OS2 +#endif +#if defined(_WINDOWS) && !defined(WINDOWS) +# define WINDOWS +#endif +#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) +# ifndef WIN32 +# define WIN32 +# endif +#endif +#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) +# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) +# ifndef SYS16BIT +# define SYS16BIT +# endif +# endif +#endif + +/* + * Compile with -DMAXSEG_64K if the alloc function cannot allocate more + * than 64k bytes at a time (needed on systems with 16-bit int). + */ +#ifdef SYS16BIT +# define MAXSEG_64K +#endif +#ifdef MSDOS +# define UNALIGNED_OK +#endif + +#ifdef __STDC_VERSION__ +# ifndef STDC +# define STDC +# endif +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +#endif +#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) +# define STDC +#endif +#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) +# define STDC +#endif +#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) +# define STDC +#endif +#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) +# define STDC +#endif + +#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ +# define STDC +#endif + +#ifndef STDC +# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ +# define const /* note: need a more gentle solution here */ +# endif +#endif + +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# ifdef MAXSEG_64K +# define MAX_MEM_LEVEL 8 +# else +# define MAX_MEM_LEVEL 9 +# endif +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + + /* Type declarations */ + +#ifndef OF /* function prototypes */ +# ifdef STDC +# define OF(args) args +# else +# define OF(args) () +# endif +#endif + +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + +/* The following definitions for FAR are needed only for MSDOS mixed + * model programming (small or medium model with some far allocations). + * This was tested only with MSC; for other MSDOS compilers you may have + * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, + * just define FAR to be empty. + */ +#ifdef SYS16BIT +# if defined(M_I86SM) || defined(M_I86MM) + /* MSC small or medium model */ +# define SMALL_MEDIUM +# ifdef _MSC_VER +# define FAR _far +# else +# define FAR far +# endif +# endif +# if (defined(__SMALL__) || defined(__MEDIUM__)) + /* Turbo C small or medium model */ +# define SMALL_MEDIUM +# ifdef __BORLANDC__ +# define FAR _far +# else +# define FAR far +# endif +# endif +#endif + +#if defined(WINDOWS) || defined(WIN32) + /* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +# ifdef ZLIB_DLL +# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) +# ifdef ZLIB_INTERNAL +# define ZEXTERN extern __declspec(dllexport) +# else +# define ZEXTERN extern __declspec(dllimport) +# endif +# endif +# endif /* ZLIB_DLL */ + /* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +# ifdef ZLIB_WINAPI +# ifdef FAR +# undef FAR +# endif +# include + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define ZEXPORT WINAPI +# ifdef WIN32 +# define ZEXPORTVA WINAPIV +# else +# define ZEXPORTVA FAR CDECL +# endif +# endif +#endif + +#if defined (__BEOS__) +# ifdef ZLIB_DLL +# ifdef ZLIB_INTERNAL +# define ZEXPORT __declspec(dllexport) +# define ZEXPORTVA __declspec(dllexport) +# else +# define ZEXPORT __declspec(dllimport) +# define ZEXPORTVA __declspec(dllimport) +# endif +# endif +#endif + +#ifndef ZEXTERN +# define ZEXTERN extern +#endif +#ifndef ZEXPORT +# define ZEXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA +#endif + +#ifndef FAR +# define FAR +#endif + +#if !defined(__MACTYPES__) +typedef unsigned char Byte; /* 8 bits */ +#endif +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +#ifdef SMALL_MEDIUM + /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ +# define Bytef Byte FAR +#else + typedef Byte FAR Bytef; +#endif +typedef char FAR charf; +typedef int FAR intf; +typedef uInt FAR uIntf; +typedef uLong FAR uLongf; + +#ifdef STDC + typedef void const *voidpc; + typedef void FAR *voidpf; + typedef void *voidp; +#else + typedef Byte const *voidpc; + typedef Byte FAR *voidpf; + typedef Byte *voidp; +#endif + +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short +# endif +#endif + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +/* MVS linker does not support external names larger than 8 bytes */ +#if defined(__MVS__) + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") +#endif + +#endif /* ZCONF_H */ diff --git a/zlib/zconf.h b/zlib/zconf.h.in similarity index 50% rename from zlib/zconf.h rename to zlib/zconf.h.in index d983fa3eed4..5e1d68a004e 100644 --- a/zlib/zconf.h +++ b/zlib/zconf.h.in @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,59 +8,161 @@ #ifndef ZCONF_H #define ZCONF_H -#ifdef HAVE_CONFIG_H -#include -#endif - /* * If you *really* need a unique prefix for all types and library functions, * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". */ -#ifdef Z_PREFIX -# define deflateInit_ z_deflateInit_ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate -# define deflateEnd z_deflateEnd -# define inflateInit_ z_inflateInit_ -# define inflate z_inflate -# define inflateEnd z_inflateEnd -# define deflateInit2_ z_deflateInit2_ -# define deflateSetDictionary z_deflateSetDictionary -# define deflateCopy z_deflateCopy -# define deflateReset z_deflateReset -# define deflateParams z_deflateParams # define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending # define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint -# define inflateCopy z_inflateCopy -# define inflateReset z_inflateReset -# define inflateBack z_inflateBack -# define inflateBackEnd z_inflateBackEnd -# define compress z_compress -# define compress2 z_compress2 -# define compressBound z_compressBound -# define uncompress z_uncompress -# define adler32 z_adler32 -# define crc32 z_crc32 -# define get_crc_table z_get_crc_table +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif # define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion -# define alloc_func z_alloc_func -# define free_func z_free_func -# define in_func z_in_func -# define out_func z_out_func +/* all zlib typedefs in zlib.h and zconf.h */ # define Byte z_Byte -# define uInt z_uInt -# define uLong z_uLong # define Bytef z_Bytef +# define alloc_func z_alloc_func # define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func # define intf z_intf +# define out_func z_out_func +# define uInt z_uInt # define uIntf z_uIntf +# define uLong z_uLong # define uLongf z_uLongf -# define voidpf z_voidpf # define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + #endif #if defined(__MSDOS__) && !defined(MSDOS) @@ -129,9 +231,25 @@ # endif #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -161,7 +279,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ @@ -175,6 +293,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -288,49 +414,121 @@ typedef uLong FAR uLongf; typedef Byte *voidp; #endif -#ifdef HAVE_UNISTD_H -# include /* for off_t */ -# include /* for SEEK_* and off_t */ -# ifdef VMS -# include /* for off_t */ +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short # endif -# define z_off_t off_t #endif -#ifndef SEEK_SET + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) # define SEEK_SET 0 /* Seek from beginning of file. */ # define SEEK_CUR 1 /* Seek from current position. */ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ #endif + #ifndef z_off_t # define z_off_t long #endif -#if defined(__OS400__) -# define NO_vsnprintf -#endif - -#if defined(__MVS__) -# define NO_vsnprintf -# ifdef FAR -# undef FAR +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t # endif #endif /* MVS linker does not support external names larger than 8 bytes */ #if defined(__MVS__) -# pragma map(deflateInit_,"DEIN") -# pragma map(deflateInit2_,"DEIN2") -# pragma map(deflateEnd,"DEEND") -# pragma map(deflateBound,"DEBND") -# pragma map(inflateInit_,"ININ") -# pragma map(inflateInit2_,"ININ2") -# pragma map(inflateEnd,"INEND") -# pragma map(inflateSync,"INSY") -# pragma map(inflateSetDictionary,"INSEDI") -# pragma map(compressBound,"CMBND") -# pragma map(inflate_table,"INTABL") -# pragma map(inflate_fast,"INFA") -# pragma map(inflate_copyright,"INCOPY") + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") #endif #endif /* ZCONF_H */ diff --git a/zlib/zlib.3 b/zlib/zlib.3 index 90b8162870f..bda4eb07370 100644 --- a/zlib/zlib.3 +++ b/zlib/zlib.3 @@ -1,4 +1,4 @@ -.TH ZLIB 3 "18 July 2005" +.TH ZLIB 3 "15 Jan 2017" .SH NAME zlib \- compression/decompression library .SH SYNOPSIS @@ -9,15 +9,15 @@ for full description] The .I zlib library is a general purpose data compression library. -The code is thread safe. +The code is thread safe, assuming that the standard library functions +used are thread safe, such as memory allocation routines. It provides in-memory compression and decompression functions, including integrity checks of the uncompressed data. This version of the library supports only one compression method (deflation) -but other algorithms will be added later -and will have the same stream interface. +but other algorithms may be added later +with the same stream interface. .LP Compression can be done in a single step if the buffers are large enough -(for example if an input file is mmap'ed), or can be done by repeated calls of the compression function. In the latter case, the application must provide more input and/or consume the output @@ -30,91 +30,63 @@ with an interface similar to that of stdio. .LP The library does not install any signal handler. The decoder checks the consistency of the compressed data, -so the library should never crash even in case of corrupted input. +so the library should never crash even in the case of corrupted input. .LP All functions of the compression library are documented in the file .IR zlib.h . The distribution source includes examples of use of the library in the files -.I example.c +.I test/example.c and -.IR minigzip.c . +.IR test/minigzip.c, +as well as other examples in the +.IR examples/ +directory. .LP Changes to this version are documented in the file .I ChangeLog -that accompanies the source, -and are concerned primarily with bug fixes and portability enhancements. +that accompanies the source. .LP -A Java implementation of .I zlib -is available in the Java Development Kit 1.1: -.IP -http://www.javasoft.com/products/JDK/1.1/docs/api/Package-java.util.zip.html +is built in to many languages and operating systems, including but not limited to +Java, Python, .NET, PHP, Perl, Ruby, Swift, and Go. .LP -A Perl interface to -.IR zlib , -written by Paul Marquess (pmqs@cpan.org), -is available at CPAN (Comprehensive Perl Archive Network) sites, -including: -.IP -http://www.cpan.org/modules/by-module/Compress/ -.LP -A Python interface to -.IR zlib , -written by A.M. Kuchling (amk@magnet.com), -is available in Python 1.5 and later versions: -.IP -http://www.python.org/doc/lib/module-zlib.html -.LP -A -.I zlib -binding for -.IR tcl (1), -written by Andreas Kupries (a.kupries@westend.com), -is availlable at: -.IP -http://www.westend.com/~kupries/doc/trf/man/man.html -.LP -An experimental package to read and write files in .zip format, +An experimental package to read and write files in the .zip format, written on top of .I zlib by Gilles Vollant (info@winimage.com), is available at: .IP -http://www.winimage.com/zLibDll/unzip.html +http://www.winimage.com/zLibDll/minizip.html and also in the .I contrib/minizip directory of the main .I zlib -web site. +source distribution. .SH "SEE ALSO" The .I zlib -web site can be found at either of these locations: +web site can be found at: .IP -http://www.zlib.org -.br -http://www.gzip.org/zlib/ +http://zlib.net/ .LP -The data format used by the zlib library is described by RFC +The data format used by the +.I zlib +library is described by RFC (Request for Comments) 1950 to 1952 in the files: .IP -http://www.ietf.org/rfc/rfc1950.txt (concerning zlib format) +http://tools.ietf.org/html/rfc1950 (for the zlib header and trailer format) .br -http://www.ietf.org/rfc/rfc1951.txt (concerning deflate format) +http://tools.ietf.org/html/rfc1951 (for the deflate compressed data format) .br -http://www.ietf.org/rfc/rfc1952.txt (concerning gzip format) +http://tools.ietf.org/html/rfc1952 (for the gzip header and trailer format) .LP -These documents are also available in other formats from: -.IP -ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html -.LP -Mark Nelson (markn@ieee.org) wrote an article about +Mark Nelson wrote an article about .I zlib for the Jan. 1997 issue of Dr. Dobb's Journal; a copy of the article is available at: .IP -http://dogma.net/markn/articles/zlibtool/zlibtool.htm +http://marknelson.us/1997/01/01/zlib-engine/ .SH "REPORTING PROBLEMS" Before reporting a problem, please check the @@ -127,22 +99,40 @@ Please read the .I zlib FAQ at: .IP -http://www.gzip.org/zlib/zlib_faq.html +http://zlib.net/zlib_faq.html .LP before asking for help. Send questions and/or comments to zlib@gzip.org, or (for the Windows DLL version) to Gilles Vollant (info@winimage.com). -.SH AUTHORS -Version 1.2.3 -Copyright (C) 1995-2005 Jean-loup Gailly (jloup@gzip.org) -and Mark Adler (madler@alumni.caltech.edu). +.SH AUTHORS AND LICENSE +Version 1.2.11 .LP -This software is provided "as-is," -without any express or implied warranty. -In no event will the authors be held liable for any damages +Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler +.LP +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages arising from the use of this software. -See the distribution directory with respect to requirements -governing redistribution. +.LP +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: +.LP +.nr step 1 1 +.IP \n[step]. 3 +The origin of this software must not be misrepresented; you must not +claim that you wrote the original software. If you use this software +in a product, an acknowledgment in the product documentation would be +appreciated but is not required. +.IP \n+[step]. +Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original software. +.IP \n+[step]. +This notice may not be removed or altered from any source distribution. +.LP +Jean-loup Gailly Mark Adler +.br +jloup@gzip.org madler@alumni.caltech.edu +.LP The deflate format used by .I zlib was defined by Phil Katz. diff --git a/zlib/zlib.h b/zlib/zlib.h index 022817927ce..08f491999c4 100644 --- a/zlib/zlib.h +++ b/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.3, July 18th, 2005 + version 1.2.11, January 15th, 2017 - Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -24,54 +24,58 @@ The data format used by the zlib library is described by RFCs (Request for - Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt - (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). + Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). */ #ifndef ZLIB_H #define ZLIB_H -#include "zconf.h" +#include #ifdef __cplusplus extern "C" { #endif -#define ZLIB_VERSION "1.2.3" -#define ZLIB_VERNUM 0x1230 +#define ZLIB_VERSION "1.2.11" +#define ZLIB_VERNUM 0x12b0 +#define ZLIB_VER_MAJOR 1 +#define ZLIB_VER_MINOR 2 +#define ZLIB_VER_REVISION 11 +#define ZLIB_VER_SUBREVISION 0 /* - The 'zlib' compression library provides in-memory compression and - decompression functions, including integrity checks of the uncompressed - data. This version of the library supports only one compression method - (deflation) but other algorithms will be added later and will have the same - stream interface. + The 'zlib' compression library provides in-memory compression and + decompression functions, including integrity checks of the uncompressed data. + This version of the library supports only one compression method (deflation) + but other algorithms will be added later and will have the same stream + interface. - Compression can be done in a single step if the buffers are large - enough (for example if an input file is mmap'ed), or can be done by - repeated calls of the compression function. In the latter case, the - application must provide more input and/or consume the output + Compression can be done in a single step if the buffers are large enough, + or can be done by repeated calls of the compression function. In the latter + case, the application must provide more input and/or consume the output (providing more output space) before each call. - The compressed data format used by default by the in-memory functions is + The compressed data format used by default by the in-memory functions is the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped around a deflate stream, which is itself documented in RFC 1951. - The library also supports reading and writing files in gzip (.gz) format + The library also supports reading and writing files in gzip (.gz) format with an interface similar to that of stdio using the functions that start with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. + This library can optionally read and write gzip and raw deflate streams in + memory as well. - The zlib format was designed to be compact and fast for use in memory + The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- file compression on file systems, has a larger header than zlib to maintain directory information, and uses a different, slower check method than zlib. - The library does not install any signal handler. The decoder checks - the consistency of the compressed data, so the library should never - crash even in case of corrupted input. + The library does not install any signal handler. The decoder checks + the consistency of the compressed data, so the library should never crash + even in the case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -80,23 +84,24 @@ typedef void (*free_func) OF((voidpf opaque, voidpf address)); struct internal_state; typedef struct z_stream_s { - Bytef *next_in; /* next input byte */ + z_const Bytef *next_in; /* next input byte */ uInt avail_in; /* number of bytes available at next_in */ - uLong total_in; /* total nb of input bytes read so far */ + uLong total_in; /* total number of input bytes read so far */ - Bytef *next_out; /* next output byte should be put there */ + Bytef *next_out; /* next output byte will go here */ uInt avail_out; /* remaining free space at next_out */ - uLong total_out; /* total nb of bytes output so far */ + uLong total_out; /* total number of bytes output so far */ - char *msg; /* last error message, NULL if no error */ + z_const char *msg; /* last error message, NULL if no error */ struct internal_state FAR *state; /* not visible by applications */ alloc_func zalloc; /* used to allocate the internal state */ free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ } z_stream; @@ -126,45 +131,47 @@ typedef struct gz_header_s { typedef gz_header FAR *gz_headerp; /* - The application must update next_in and avail_in when avail_in has - dropped to zero. It must update next_out and avail_out when avail_out - has dropped to zero. The application must initialize zalloc, zfree and - opaque before calling the init function. All other fields are set by the - compression library and must not be updated by the application. + The application must update next_in and avail_in when avail_in has dropped + to zero. It must update next_out and avail_out when avail_out has dropped + to zero. The application must initialize zalloc, zfree and opaque before + calling the init function. All other fields are set by the compression + library and must not be updated by the application. - The opaque value provided by the application will be passed as the first - parameter for calls of zalloc and zfree. This can be useful for custom - memory management. The compression library attaches no meaning to the + The opaque value provided by the application will be passed as the first + parameter for calls of zalloc and zfree. This can be useful for custom + memory management. The compression library attaches no meaning to the opaque value. - zalloc must return Z_NULL if there is not enough memory for the object. + zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be - thread safe. + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). - On 16-bit systems, the functions zalloc and zfree must be able to allocate - exactly 65536 bytes, but will not be required to allocate more than this - if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, - pointers returned by zalloc for objects of exactly 65536 bytes *must* - have their offset normalized to zero. The default allocation function - provided by this library ensures this (see zutil.c). To reduce memory - requirements and avoid any allocation of 64K objects, at the expense of - compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h). + On 16-bit systems, the functions zalloc and zfree must be able to allocate + exactly 65536 bytes, but will not be required to allocate more than this if + the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers + returned by zalloc for objects of exactly 65536 bytes *must* have their + offset normalized to zero. The default allocation function provided by this + library ensures this (see zutil.c). To reduce memory requirements and avoid + any allocation of 64K objects, at the expense of compression ratio, compile + the library with -DMAX_WBITS=14 (see zconf.h). - The fields total_in and total_out can be used for statistics or - progress reports. After compression, total_in holds the total size of - the uncompressed data and may be saved for use in the decompressor - (particularly if the decompressor wants to decompress everything in - a single step). + The fields total_in and total_out can be used for statistics or progress + reports. After compression, total_in holds the total size of the + uncompressed data and may be saved for use by the decompressor (particularly + if the decompressor wants to decompress everything in a single step). */ /* constants */ #define Z_NO_FLUSH 0 -#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */ +#define Z_PARTIAL_FLUSH 1 #define Z_SYNC_FLUSH 2 #define Z_FULL_FLUSH 3 #define Z_FINISH 4 #define Z_BLOCK 5 +#define Z_TREES 6 /* Allowed flush values; see deflate() and inflate() below for details */ #define Z_OK 0 @@ -176,8 +183,8 @@ typedef gz_header FAR *gz_headerp; #define Z_MEM_ERROR (-4) #define Z_BUF_ERROR (-5) #define Z_VERSION_ERROR (-6) -/* Return codes for the compression/decompression functions. Negative - * values are errors, positive values are used for special but normal events. +/* Return codes for the compression/decompression functions. Negative values + * are errors, positive values are used for special but normal events. */ #define Z_NO_COMPRESSION 0 @@ -197,7 +204,7 @@ typedef gz_header FAR *gz_headerp; #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 -/* Possible values of the data_type field (though see inflate()) */ +/* Possible values of the data_type field for deflate() */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ @@ -207,134 +214,162 @@ typedef gz_header FAR *gz_headerp; #define zlib_version zlibVersion() /* for compatibility with versions < 1.0.2 */ + /* basic functions */ ZEXTERN const char * ZEXPORT zlibVersion OF((void)); /* The application can compare zlibVersion and ZLIB_VERSION for consistency. - If the first character differs, the library code actually used is - not compatible with the zlib.h header file used by the application. - This check is automatically made by deflateInit and inflateInit. + If the first character differs, the library code actually used is not + compatible with the zlib.h header file used by the application. This check + is automatically made by deflateInit and inflateInit. */ /* ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); - Initializes the internal stream state for compression. The fields - zalloc, zfree and opaque must be initialized before by the caller. - If zalloc and zfree are set to Z_NULL, deflateInit updates them to - use default allocation functions. + Initializes the internal stream state for compression. The fields + zalloc, zfree and opaque must be initialized before by the caller. If + zalloc and zfree are set to Z_NULL, deflateInit updates them to use default + allocation functions. The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: - 1 gives best speed, 9 gives best compression, 0 gives no compression at - all (the input data is simply copied a block at a time). - Z_DEFAULT_COMPRESSION requests a default compromise between speed and - compression (currently equivalent to level 6). + 1 gives best speed, 9 gives best compression, 0 gives no compression at all + (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION + requests a default compromise between speed and compression (currently + equivalent to level 6). - deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_STREAM_ERROR if level is not a valid compression level, + deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if level is not a valid compression level, or Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible - with the version assumed by the caller (ZLIB_VERSION). - msg is set to null if there is no error message. deflateInit does not - perform any compression: this will be done by deflate(). + with the version assumed by the caller (ZLIB_VERSION). msg is set to null + if there is no error message. deflateInit does not perform any compression: + this will be done by deflate(). */ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); /* deflate compresses as much data as possible, and stops when the input - buffer becomes empty or the output buffer becomes full. It may introduce some - output latency (reading input without producing any output) except when + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when forced to flush. - The detailed semantics are as follows. deflate performs one or both of the + The detailed semantics are as follows. deflate performs one or both of the following actions: - Compress more input starting at next_in and update next_in and avail_in - accordingly. If not all input can be processed (because there is not + accordingly. If not all input can be processed (because there is not enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - - Provide more output starting at next_out and update next_out and avail_out - accordingly. This action is forced if the parameter flush is non zero. + - Generate more output starting at next_out and update next_out and avail_out + accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter - should be set only when necessary (in interactive applications). - Some output may be provided even if flush is not set. + should be set only when necessary. Some output may be provided even if + flush is zero. - Before the call of deflate(), the application should ensure that at least - one of the actions is possible, by providing more input and/or consuming - more output, and updating avail_in or avail_out accordingly; avail_out - should never be zero before the call. The application can consume the - compressed output when it wants, for example when the output buffer is full - (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK - and with zero avail_out, it must be called again after making room in the - output buffer because there might be more output pending. + Before the call of deflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating avail_in or avail_out accordingly; avail_out should + never be zero before the call. The application can consume the compressed + output when it wants, for example when the output buffer is full (avail_out + == 0), or after each call of deflate(). If deflate returns Z_OK and with + zero avail_out, it must be called again after making room in the output + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to - decide how much data to accumualte before producing output, in order to + decide how much data to accumulate before producing output, in order to maximize compression. If the parameter flush is set to Z_SYNC_FLUSH, all pending output is flushed to the output buffer and the output is aligned on a byte boundary, so - that the decompressor can get all input data available so far. (In particular - avail_in is zero after the call if enough output space has been provided - before the call.) Flushing may degrade compression for some compression - algorithms and so it should be used only when necessary. + that the decompressor can get all input data available so far. (In + particular avail_in is zero after the call if enough output space has been + provided before the call.) Flushing may degrade compression for some + compression algorithms and so it should be used only when necessary. This + completes the current deflate block and follows it with an empty stored block + that is three bits plus filler bits to the next byte, followed by four bytes + (00 00 ff ff). + + If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the + output buffer, but the output is not aligned to a byte boundary. All of the + input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. + This completes the current deflate block and follows it with an empty fixed + codes block that is 10 bits long. This assures that enough bytes are output + in order for the decompressor to finish the block before the empty fixed + codes block. + + If flush is set to Z_BLOCK, a deflate block is completed and emitted, as + for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to + seven bits of the current block are held to be written as the next byte after + the next deflate block is completed. In this case, the decompressor may not + be provided enough bits at this point in order to complete decompression of + the data provided so far to the compressor. It may need to wait for the next + block to be emitted. This is for advanced applications that need to control + the emission of deflate blocks. If flush is set to Z_FULL_FLUSH, all output is flushed as with Z_SYNC_FLUSH, and the compression state is reset so that decompression can restart from this point if previous compressed data has been damaged or if - random access is desired. Using Z_FULL_FLUSH too often can seriously degrade + random access is desired. Using Z_FULL_FLUSH too often can seriously degrade compression. If deflate returns with avail_out == 0, this function must be called again with the same value of the flush parameter and more output space (updated avail_out), until the flush is complete (deflate returns with non-zero - avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that + avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that avail_out is greater than six to avoid repeated flush markers due to avail_out == 0 on return. If the parameter flush is set to Z_FINISH, pending input is processed, - pending output is flushed and deflate returns with Z_STREAM_END if there - was enough output space; if deflate returns with Z_OK, this function must be - called again with Z_FINISH and more output space (updated avail_out) but no - more input data, until it returns with Z_STREAM_END or an error. After - deflate has returned Z_STREAM_END, the only possible operations on the - stream are deflateReset or deflateEnd. + pending output is flushed and deflate returns with Z_STREAM_END if there was + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. - Z_FINISH can be used immediately after deflateInit if all the compression - is to be done in a single step. In this case, avail_out must be at least - the value returned by deflateBound (see below). If deflate does not return - Z_STREAM_END, then it must be called again as described above. + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. - deflate() sets strm->adler to the adler32 checksum of all input read - so far (that is, total_in bytes). + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) deflate() may update strm->data_type if it can make a good guess about - the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered - binary. This field is only for information purposes and does not affect - the compression algorithm in any manner. + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example - if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible - (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not - fatal, and deflate() can be called again with more input and more output - space to continue compressing. + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. */ ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. - This function discards any unprocessed input and does not flush any - pending output. + This function discards any unprocessed input and does not flush any pending + output. deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state was inconsistent, Z_DATA_ERROR if the stream was freed - prematurely (some input or output was discarded). In the error case, - msg may be set but then points to a static string (which must not be + prematurely (some input or output was discarded). In the error case, msg + may be set but then points to a static string (which must not be deallocated). */ @@ -342,134 +377,157 @@ ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); /* ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); - Initializes the internal stream state for decompression. The fields + Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by - the caller. If next_in is not Z_NULL and avail_in is large enough (the exact - value depends on the compression method), inflateInit determines the - compression method from the zlib header and allocates all data structures - accordingly; otherwise the allocation will be deferred to the first call of - inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to - use default allocation functions. + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the - version assumed by the caller. msg is set to null if there is no error - message. inflateInit does not perform any decompression apart from reading - the zlib header if present: this will be done by inflate(). (So next_in and - avail_in may be modified, but next_out and avail_out are unchanged.) + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. */ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); /* inflate decompresses as much data as possible, and stops when the input - buffer becomes empty or the output buffer becomes full. It may introduce + buffer becomes empty or the output buffer becomes full. It may introduce some output latency (reading input without producing any output) except when forced to flush. - The detailed semantics are as follows. inflate performs one or both of the + The detailed semantics are as follows. inflate performs one or both of the following actions: - Decompress more input starting at next_in and update next_in and avail_in - accordingly. If not all input can be processed (because there is not - enough room in the output buffer), next_in is updated and processing - will resume at this point for the next call of inflate(). + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). - - Provide more output starting at next_out and update next_out and avail_out - accordingly. inflate() provides as much output as possible, until there - is no more input data or no more space in the output buffer (see below - about the flush parameter). + - Generate more output starting at next_out and update next_out and avail_out + accordingly. inflate() provides as much output as possible, until there is + no more input data or no more space in the output buffer (see below about + the flush parameter). - Before the call of inflate(), the application should ensure that at least - one of the actions is possible, by providing more input and/or consuming - more output, and updating the next_* and avail_* values accordingly. - The application can consume the uncompressed output when it wants, for - example when the output buffer is full (avail_out == 0), or after each - call of inflate(). If inflate returns Z_OK and with zero avail_out, it - must be called again after making room in the output buffer because there - might be more output pending. + Before the call of inflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The + application can consume the uncompressed output when it wants, for example + when the output buffer is full (avail_out == 0), or after each call of + inflate(). If inflate returns Z_OK and with zero avail_out, it must be + called again after making room in the output buffer because there might be + more output pending. - The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, - Z_FINISH, or Z_BLOCK. Z_SYNC_FLUSH requests that inflate() flush as much - output as possible to the output buffer. Z_BLOCK requests that inflate() stop - if and when it gets to the next deflate block boundary. When decoding the - zlib or gzip format, this will cause inflate() to return immediately after - the header and before the first block. When doing a raw inflate, inflate() - will go ahead and process the first block, and will return when it gets to - the end of that block, or when it runs out of data. + The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH, + Z_BLOCK, or Z_TREES. Z_SYNC_FLUSH requests that inflate() flush as much + output as possible to the output buffer. Z_BLOCK requests that inflate() + stop if and when it gets to the next deflate block boundary. When decoding + the zlib or gzip format, this will cause inflate() to return immediately + after the header and before the first block. When doing a raw inflate, + inflate() will go ahead and process the first block, and will return when it + gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. - Also to assist in this, on return inflate() will set strm->data_type to the - number of unused bits in the last byte taken from strm->next_in, plus 64 - if inflate() is currently decoding the last block in the deflate stream, - plus 128 if inflate() returned immediately after decoding an end-of-block - code or decoding the complete header up to just before the first byte of the - deflate stream. The end-of-block will not be indicated until all of the - uncompressed data from that block has been written to strm->next_out. The - number of unused bits may in general be greater than seven, except when - bit 7 of data_type is set, in which case the number of unused bits will be - less than eight. + To assist in this, on return inflate() always sets strm->data_type to the + number of unused bits in the last byte taken from strm->next_in, plus 64 if + inflate() is currently decoding the last block in the deflate stream, plus + 128 if inflate() returned immediately after decoding an end-of-block code or + decoding the complete header up to just before the first byte of the deflate + stream. The end-of-block will not be indicated until all of the uncompressed + data from that block has been written to strm->next_out. The number of + unused bits may in general be greater than seven, except when bit 7 of + data_type is set, in which case the number of unused bits will be less than + eight. data_type is set as noted here every time inflate() returns for all + flush options, and so can be used to determine the amount of currently + consumed input in bits. + + The Z_TREES option behaves as Z_BLOCK does, but it also returns when the + end of each deflate block header is reached, before any actual data in that + block is decoded. This allows the caller to determine the length of the + deflate block header for later use in random access within a deflate block. + 256 is added to the value of strm->data_type when inflate() returns + immediately after reaching the end of the deflate block header. inflate() should normally be called until it returns Z_STREAM_END or an - error. However if all decompression is to be performed in a single step - (a single call of inflate), the parameter flush should be set to - Z_FINISH. In this case all pending input is processed and all pending - output is flushed; avail_out must be large enough to hold all the - uncompressed data. (The size of the uncompressed data may have been saved - by the compressor for this purpose.) The next operation on this stream must - be inflateEnd to deallocate the decompression state. The use of Z_FINISH - is never required, but can be used to inform inflate that a faster approach - may be used for the single inflate() call. + error. However if all decompression is to be performed in a single step (a + single call of inflate), the parameter flush should be set to Z_FINISH. In + this case all pending input is processed and all pending output is flushed; + avail_out must be large enough to hold all of the uncompressed data for the + operation to complete. (The size of the uncompressed data may have been + saved by the compressor for this purpose.) The use of Z_FINISH is not + required to perform an inflation in one step. However it may be used to + inform inflate that a faster approach can be used for the single inflate() + call. Z_FINISH also informs inflate to not maintain a sliding window if the + stream completes, which reduces inflate's memory footprint. If the stream + does not complete, either because not all of the stream is provided or not + enough output space is provided, then a sliding window will be allocated and + inflate() can be called again to continue the operation as if Z_NO_FLUSH had + been used. In this implementation, inflate() always flushes as much output as possible to the output buffer, and always uses the faster approach on the - first call. So the only effect of the flush parameter in this implementation - is on the return value of inflate(), as noted below, or when it returns early - because Z_BLOCK is used. + first call. So the effects of the flush parameter in this implementation are + on the return value of inflate() as noted below, when inflate() returns early + when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of + memory for a sliding window when Z_FINISH is used. If a preset dictionary is needed after this call (see inflateSetDictionary - below), inflate sets strm->adler to the adler32 checksum of the dictionary + below), inflate sets strm->adler to the Adler-32 checksum of the dictionary chosen by the compressor and returns Z_NEED_DICT; otherwise it sets - strm->adler to the adler32 checksum of all output produced so far (that is, + strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described - below. At the end of the stream, inflate() checks that its computed adler32 + below. At the end of the stream, inflate() checks that its computed Adler-32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. - inflate() will decompress and check either zlib-wrapped or gzip-wrapped - deflate data. The header type is detected automatically. Any information - contained in the gzip header is not retained, so applications that need that - information should instead use raw inflate, see inflateInit2() below, or - inflateBack() and perform their own processing of the gzip header and - trailer. + inflate() can decompress and check either zlib-wrapped or gzip-wrapped + deflate data. The header type is detected automatically, if requested when + initializing with inflateInit2(). Any information contained in the gzip + header is not retained unless inflateGetHeader() is used. When processing + gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check - value), Z_STREAM_ERROR if the stream structure was inconsistent (for example - if next_in or next_out was NULL), Z_MEM_ERROR if there was not enough memory, - Z_BUF_ERROR if no progress is possible or if there was not enough room in the - output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to - continue decompressing. If Z_DATA_ERROR is returned, the application may then - call inflateSync() to look for a good compression block if a partial recovery - of the data is desired. + continue decompressing. If Z_DATA_ERROR is returned, the application may + then call inflateSync() to look for a good compression block if a partial + recovery of the data is to be attempted. */ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. - This function discards any unprocessed input and does not flush any - pending output. + This function discards any unprocessed input and does not flush any pending + output. - inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state - was inconsistent. In the error case, msg may be set but then points to a - static string (which must not be deallocated). + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. */ + /* Advanced functions */ /* @@ -484,55 +542,70 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, int memLevel, int strategy)); - This is another version of deflateInit with more compression options. The - fields next_in, zalloc, zfree and opaque must be initialized before by - the caller. + This is another version of deflateInit with more compression options. The + fields next_in, zalloc, zfree and opaque must be initialized before by the + caller. - The method parameter is the compression method. It must be Z_DEFLATED in + The method parameter is the compression method. It must be Z_DEFLATED in this version of the library. The windowBits parameter is the base two logarithm of the window size - (the size of the history buffer). It should be in the range 8..15 for this - version of the library. Larger values of this parameter result in better - compression at the expense of memory usage. The default value is 15 if + (the size of the history buffer). It should be in the range 8..15 for this + version of the library. Larger values of this parameter result in better + compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. - windowBits can also be -8..-15 for raw deflate. In this case, -windowBits - determines the window size. deflate() will then generate raw deflate data - with no zlib header or trailer, and will not compute an adler32 check value. + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). - windowBits can also be greater than 15 for optional gzip encoding. Add + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits + determines the window size. deflate() will then generate raw deflate data + with no zlib header or trailer, and will not compute a check value. + + windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the - compressed data instead of a zlib wrapper. The gzip header will have no - file name, no extra data, no comment, no modification time (set to zero), - no header crc, and the operating system will be set to 255 (unknown). If a - gzip stream is being written, strm->adler is a crc32 instead of an adler32. + compressed data instead of a zlib wrapper. The gzip header will have no + file name, no extra data, no comment, no modification time (set to zero), no + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. The memLevel parameter specifies how much memory should be allocated - for the internal compression state. memLevel=1 uses minimum memory but - is slow and reduces compression ratio; memLevel=9 uses maximum memory - for optimal speed. The default value is 8. See zconf.h for total memory - usage as a function of windowBits and memLevel. + for the internal compression state. memLevel=1 uses minimum memory but is + slow and reduces compression ratio; memLevel=9 uses maximum memory for + optimal speed. The default value is 8. See zconf.h for total memory usage + as a function of windowBits and memLevel. - The strategy parameter is used to tune the compression algorithm. Use the + The strategy parameter is used to tune the compression algorithm. Use the value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no string match), or Z_RLE to limit match distances to one (run-length - encoding). Filtered data consists mostly of small values with a somewhat - random distribution. In this case, the compression algorithm is tuned to - compress them better. The effect of Z_FILTERED is to force more Huffman + encoding). Filtered data consists mostly of small values with a somewhat + random distribution. In this case, the compression algorithm is tuned to + compress them better. The effect of Z_FILTERED is to force more Huffman coding and less string matching; it is somewhat intermediate between - Z_DEFAULT and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as - Z_HUFFMAN_ONLY, but give better compression for PNG image data. The strategy - parameter only affects the compression ratio but not the correctness of the - compressed output even if it is not set appropriately. Z_FIXED prevents the - use of dynamic Huffman codes, allowing for a simpler decoder for special - applications. + Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as + fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The + strategy parameter only affects the compression ratio but not the + correctness of the compressed output even if it is not set appropriately. + Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler + decoder for special applications. - deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough - memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid - method). msg is set to null if there is no error message. deflateInit2 does - not perform any compression: this will be done by deflate(). + deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid + method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is + incompatible with the version assumed by the caller (ZLIB_VERSION). msg is + set to null if there is no error message. deflateInit2 does not perform any + compression: this will be done by deflate(). */ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, @@ -540,38 +613,65 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, uInt dictLength)); /* Initializes the compression dictionary from the given byte sequence - without producing any compressed output. This function must be called - immediately after deflateInit, deflateInit2 or deflateReset, before any - call of deflate. The compressor and decompressor must use exactly the same - dictionary (see inflateSetDictionary). + without producing any compressed output. When using the zlib format, this + function must be called immediately after deflateInit, deflateInit2 or + deflateReset, and before any call of deflate. When doing raw deflate, this + function must be called either before any call of deflate, or immediately + after the completion of a deflate block, i.e. after all input has been + consumed and all output has been delivered when using any of the flush + options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The + compressor and decompressor must use exactly the same dictionary (see + inflateSetDictionary). The dictionary should consist of strings (byte sequences) that are likely to be encountered later in the data to be compressed, with the most commonly - used strings preferably put towards the end of the dictionary. Using a + used strings preferably put towards the end of the dictionary. Using a dictionary is most useful when the data to be compressed is short and can be predicted with good accuracy; the data can then be compressed better than with the default empty dictionary. Depending on the size of the compression data structures selected by deflateInit or deflateInit2, a part of the dictionary may in effect be - discarded, for example if the dictionary is larger than the window size in - deflate or deflate2. Thus the strings most likely to be useful should be - put at the end of the dictionary, not at the front. In addition, the - current implementation of deflate will use at most the window size minus - 262 bytes of the provided dictionary. + discarded, for example if the dictionary is larger than the window size + provided in deflateInit or deflateInit2. Thus the strings most likely to be + useful should be put at the end of the dictionary, not at the front. In + addition, the current implementation of deflate will use at most the window + size minus 262 bytes of the provided dictionary. - Upon return of this function, strm->adler is set to the adler32 value + Upon return of this function, strm->adler is set to the Adler-32 value of the dictionary; the decompressor may later use this value to determine - which dictionary has been used by the compressor. (The adler32 value + which dictionary has been used by the compressor. (The Adler-32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the - adler32 value is not computed and strm->adler is not set. + Adler-32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a - parameter is invalid (such as NULL dictionary) or the stream state is + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent (for example if deflate has already been called for this stream - or if the compression method is bsort). deflateSetDictionary does not - perform any compression: this will be done by deflate(). + or if not at a block boundary for raw deflate). deflateSetDictionary does + not perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. */ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, @@ -581,26 +681,26 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, This function can be useful when several compression strategies will be tried, for example when there are several ways of pre-processing the input - data with a filter. The streams that will be discarded should then be freed + data with a filter. The streams that will be discarded should then be freed by calling deflateEnd. Note that deflateCopy duplicates the internal - compression state which can be quite large, so this strategy is slow and - can consume lots of memory. + compression state which can be quite large, so this strategy is slow and can + consume lots of memory. deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if the source stream state was inconsistent - (such as zalloc being NULL). msg is left unchanged in both source and + (such as zalloc being Z_NULL). msg is left unchanged in both source and destination. */ ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* - This function is equivalent to deflateEnd followed by deflateInit, - but does not free and reallocate all the internal compression state. - The stream will keep the same compression level and any other attributes - that may have been set by deflateInit2. + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. - deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source - stream state was inconsistent (such as zalloc or state being NULL). + deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). */ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, @@ -608,20 +708,36 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int strategy)); /* Dynamically update the compression level and compression strategy. The - interpretation of level and strategy is as in deflateInit2. This can be + interpretation of level and strategy is as in deflateInit2(). This can be used to switch between compression and straight copy of the input data, or - to switch to a different kind of input data requiring a different - strategy. If the compression level is changed, the input available so far - is compressed with the old level (and may be flushed); the new level will - take effect only at the next call of deflate(). + to switch to a different kind of input data requiring a different strategy. + If the compression approach (which is a function of the level) or the + strategy is changed, and if any input has been consumed in a previous + deflate() call, then the input available so far is compressed with the old + level and strategy using deflate(strm, Z_BLOCK). There are three approaches + for the compression levels 0, 1..3, and 4..9 respectively. The new level + and strategy will take effect at the next call of deflate(). - Before the call of deflateParams, the stream state must be set as for - a call of deflate(), since the currently available input may have to - be compressed and flushed. In particular, strm->avail_out must be non-zero. + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. - deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source - stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR - if strm->avail_out was zero. + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, @@ -645,31 +761,53 @@ ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, uLong sourceLen)); /* deflateBound() returns an upper bound on the compressed size after - deflation of sourceLen bytes. It must be called after deflateInit() - or deflateInit2(). This would be used to allocate an output buffer - for deflation in a single pass, and so would be called before deflate(). + deflation of sourceLen bytes. It must be called after deflateInit() or + deflateInit2(), and after deflateSetHeader(), if used. This would be used + to allocate an output buffer for deflation in a single pass, and so would be + called before deflate(). If that first deflate() call is provided the + sourceLen input bytes, an output buffer allocated to the size returned by + deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed + to return Z_STREAM_END. Note that it is possible for the compressed size to + be larger than the value returned by deflateBound() if flush options other + than Z_FINISH or Z_NO_FLUSH are used. */ +ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)); +/* + deflatePending() returns the number of bytes and bits of output that have + been generated, but not yet provided in the available output. The bytes not + provided would be due to the available output space having being consumed. + The number of bits of output not provided are between 0 and 7, where they + await more bits to join them in order to fill out a full byte. If pending + or bits are Z_NULL, then those values are not set. + + deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. + */ + ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, int bits, int value)); /* deflatePrime() inserts bits in the deflate output stream. The intent - is that this function is used to start off the deflate output with the - bits leftover from a previous deflate stream when appending to it. As such, - this function can only be used for raw deflate, and must be used before the - first deflate() call after a deflateInit2() or deflateReset(). bits must be - less than or equal to 16, and that many of the least significant bits of - value will be inserted in the output. + is that this function is used to start off the deflate output with the bits + leftover from a previous deflate stream when appending to it. As such, this + function can only be used for raw deflate, and must be used before the first + deflate() call after a deflateInit2() or deflateReset(). bits must be less + than or equal to 16, and that many of the least significant bits of value + will be inserted in the output. - deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source - stream state was inconsistent. + deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough + room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the + source stream state was inconsistent. */ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, gz_headerp head)); /* - deflateSetHeader() provides gzip header information for when a gzip + deflateSetHeader() provides gzip header information for when a gzip stream is requested by deflateInit2(). deflateSetHeader() may be called after deflateInit2() or deflateReset() and before the first call of deflate(). The text, time, os, extra field, name, and comment information @@ -682,11 +820,11 @@ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, 1.3.x) do not support header crc's, and will report that it is a "multi-part gzip file" and give up. - If deflateSetHeader is not used, the default gzip header has text false, + If deflateSetHeader is not used, the default gzip header has text false, the time set to zero, and os set to 255, with no extra, name, or comment fields. The gzip header is returned to the default state by deflateReset(). - deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ @@ -694,43 +832,53 @@ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, int windowBits)); - This is another version of inflateInit with an extra parameter. The + This is another version of inflateInit with an extra parameter. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by the caller. The windowBits parameter is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for - this version of the library. The default value is 15 if inflateInit is used - instead. windowBits must be greater than or equal to the windowBits value + this version of the library. The default value is 15 if inflateInit is used + instead. windowBits must be greater than or equal to the windowBits value provided to deflateInit2() while compressing, or it must be equal to 15 if - deflateInit2() was not used. If a compressed stream with a larger window + deflateInit2() was not used. If a compressed stream with a larger window size is given as input, inflate() will return with the error code Z_DATA_ERROR instead of trying to allocate a larger window. - windowBits can also be -8..-15 for raw inflate. In this case, -windowBits - determines the window size. inflate() will then process raw deflate data, + windowBits can also be zero to request that inflate use the window size in + the zlib header of the compressed stream. + + windowBits can also be -8..-15 for raw inflate. In this case, -windowBits + determines the window size. inflate() will then process raw deflate data, not looking for a zlib or gzip header, not generating a check value, and not - looking for any check values for comparison at the end of the stream. This + looking for any check values for comparison at the end of the stream. This is for use with other formats that use the deflate compressed data format - such as zip. Those formats provide their own check values. If a custom + such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is - recommended that a check value such as an adler32 or a crc32 be applied to + recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For - most applications, the zlib format should be used as is. Note that comments + most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. - windowBits can also be greater than 15 for optional gzip decoding. Add + windowBits can also be greater than 15 for optional gzip decoding. Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will - return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is - a crc32 instead of an adler32. + return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough - memory, Z_STREAM_ERROR if a parameter is invalid (such as a null strm). msg - is set to null if there is no error message. inflateInit2 does not perform - any decompression apart from reading the zlib header if present: this will - be done by inflate(). (So next_in and avail_in may be modified, but next_out - and avail_out are unchanged.) + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit2 does not perform any decompression + apart from possibly reading the zlib header if present: actual decompression + will be done by inflate(). (So next_in and avail_in may be modified, but + next_out and avail_out are unused and unchanged.) The current implementation + of inflateInit2() does not process any header information -- that is + deferred until inflate() is called. */ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, @@ -738,36 +886,56 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, uInt dictLength)); /* Initializes the decompression dictionary from the given uncompressed byte - sequence. This function must be called immediately after a call of inflate, - if that call returned Z_NEED_DICT. The dictionary chosen by the compressor - can be determined from the adler32 value returned by that call of inflate. + sequence. This function must be called immediately after a call of inflate, + if that call returned Z_NEED_DICT. The dictionary chosen by the compressor + can be determined from the Adler-32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see - deflateSetDictionary). For raw inflate, this function can be called - immediately after inflateInit2() or inflateReset() and before any call of - inflate() to set the dictionary. The application must insure that the - dictionary that was used for compression is provided. + deflateSetDictionary). For raw inflate, this function can be called at any + time to set the dictionary. If the provided dictionary is smaller than the + window and there is already data in the window, then the provided dictionary + will amend what's there. The application must insure that the dictionary + that was used for compression is provided. inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a - parameter is invalid (such as NULL dictionary) or the stream state is + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the - expected one (incorrect adler32 value). inflateSetDictionary does not + expected one (incorrect Adler-32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ +ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by inflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If inflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); /* - Skips invalid compressed data until a full flush point (see above the - description of deflate with Z_FULL_FLUSH) can be found, or until all - available input is skipped. No output is provided. + Skips invalid compressed data until a possible full flush point (see above + for the description of deflate with Z_FULL_FLUSH) can be found, or until all + available input is skipped. No output is provided. - inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR - if no more input was provided, Z_DATA_ERROR if no flush point has been found, - or Z_STREAM_ERROR if the stream structure was inconsistent. In the success - case, the application may save the current current value of total_in which - indicates where valid compressed data was found. In the error case, the - application may repeatedly call inflateSync, providing more input each time, - until success or end of the input data. + inflateSync searches for a 00 00 FF FF pattern in the compressed data. + All full flush points have this pattern, but not all occurrences of this + pattern are full flush points. + + inflateSync returns Z_OK if a possible full flush point has been found, + Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point + has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. */ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, @@ -782,18 +950,32 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if the source stream state was inconsistent - (such as zalloc being NULL). msg is left unchanged in both source and + (such as zalloc being Z_NULL). msg is left unchanged in both source and destination. */ ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, - but does not free and reallocate all the internal decompression state. - The stream will keep attributes that may have been set by inflateInit2. + but does not free and reallocate the internal decompression state. The + stream will keep attributes that may have been set by inflateInit2. - inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source - stream state was inconsistent (such as zalloc or state being NULL). + inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, + int windowBits)); +/* + This function is the same as inflateReset, but it also permits changing + the wrap and window size requests. The windowBits parameter is interpreted + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. + + inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL), or if + the windowBits parameter is invalid. */ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, @@ -801,54 +983,87 @@ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, int value)); /* This function inserts bits in the inflate input stream. The intent is - that this function is used to start inflating at a bit position in the - middle of a byte. The provided bits will be used before any bytes are used - from next_in. This function should only be used with raw inflate, and - should be used before the first inflate() call after inflateInit2() or - inflateReset(). bits must be less than or equal to 16, and that many of the - least significant bits of value will be inserted in the input. + that this function is used to start inflating at a bit position in the + middle of a byte. The provided bits will be used before any bytes are used + from next_in. This function should only be used with raw inflate, and + should be used before the first inflate() call after inflateInit2() or + inflateReset(). bits must be less than or equal to 16, and that many of the + least significant bits of value will be inserted in the input. - inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + If bits is negative, then the input stream bit buffer is emptied. Then + inflatePrime() can be called again to put bits in the buffer. This is used + to clear out bits leftover after feeding inflate a block description prior + to feeding inflate codes. + + inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ +ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +/* + This function returns two values, one in the lower 16 bits of the return + value, and the other in the remaining upper bits, obtained by shifting the + return value down 16 bits. If the upper value is -1 and the lower value is + zero, then inflate() is currently decoding information outside of a block. + If the upper value is -1 and the lower value is non-zero, then inflate is in + the middle of a stored block, with the lower value equaling the number of + bytes from the input remaining to copy. If the upper value is not -1, then + it is the number of bits back from the current bit position in the input of + the code (literal or length/distance pair) currently being processed. In + that case the lower value is the number of bytes already emitted for that + code. + + A code is being processed if inflate is waiting for more input to complete + decoding of the code, or if it has completed decoding but is waiting for + more output space to write the literal or match data. + + inflateMark() is used to mark locations in the input data for random + access, which may be at bit positions, and to note those cases where the + output of a code may span boundaries of random access blocks. The current + location in the input stream can be determined from avail_in and data_type + as noted in the description for the Z_BLOCK flush parameter for inflate. + + inflateMark returns the value noted above, or -65536 if the provided + source stream state was inconsistent. +*/ + ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, gz_headerp head)); /* - inflateGetHeader() requests that gzip header information be stored in the + inflateGetHeader() requests that gzip header information be stored in the provided gz_header structure. inflateGetHeader() may be called after inflateInit2() or inflateReset(), and before the first call of inflate(). As inflate() processes the gzip stream, head->done is zero until the header is completed, at which time head->done is set to one. If a zlib stream is being decoded, then head->done is set to -1 to indicate that there will be - no gzip header information forthcoming. Note that Z_BLOCK can be used to - force inflate() to return immediately after header processing is complete - and before any actual data is decompressed. + no gzip header information forthcoming. Note that Z_BLOCK or Z_TREES can be + used to force inflate() to return immediately after header processing is + complete and before any actual data is decompressed. - The text, time, xflags, and os fields are filled in with the gzip header + The text, time, xflags, and os fields are filled in with the gzip header contents. hcrc is set to true if there is a header CRC. (The header CRC - was valid if done is set to one.) If extra is not Z_NULL, then extra_max + was valid if done is set to one.) If extra is not Z_NULL, then extra_max contains the maximum number of bytes to write to extra. Once done is true, extra_len contains the actual extra field length, and extra contains the extra field, or that field truncated if extra_max is less than extra_len. If name is not Z_NULL, then up to name_max characters are written there, terminated with a zero unless the length is greater than name_max. If comment is not Z_NULL, then up to comm_max characters are written there, - terminated with a zero unless the length is greater than comm_max. When - any of extra, name, or comment are not Z_NULL and the respective field is - not present in the header, then that field is set to Z_NULL to signal its + terminated with a zero unless the length is greater than comm_max. When any + of extra, name, or comment are not Z_NULL and the respective field is not + present in the header, then that field is set to Z_NULL to signal its absence. This allows the use of deflateSetHeader() with the returned structure to duplicate the header. However if those fields are set to allocated memory, then the application will need to save those pointers elsewhere so that they can be eventually freed. - If inflateGetHeader is not used, then the header information is simply + If inflateGetHeader is not used, then the header information is simply discarded. The header is always checked for validity, including the header CRC if present. inflateReset() will reset the process to discard the header information. The application would need to call inflateGetHeader() again to retrieve the header from the next gzip stream. - inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ @@ -869,12 +1084,13 @@ ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, See inflateBack() for the usage of these routines. inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of - the paramaters are invalid, Z_MEM_ERROR if the internal state could not - be allocated, or Z_VERSION_ERROR if the version of the library does not - match the version of the header file. + the parameters are invalid, Z_MEM_ERROR if the internal state could not be + allocated, or Z_VERSION_ERROR if the version of the library does not match + the version of the header file. */ -typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *)); +typedef unsigned (*in_func) OF((void FAR *, + z_const unsigned char FAR * FAR *)); typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, @@ -882,25 +1098,26 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, out_func out, void FAR *out_desc)); /* inflateBack() does a raw inflate with a single call using a call-back - interface for input and output. This is more efficient than inflate() for - file i/o applications in that it avoids copying between the output and the - sliding window by simply making the window itself the output buffer. This - function trusts the application to not change the output buffer passed by - the output function, at least until inflateBack() returns. + interface for input and output. This is potentially more efficient than + inflate() for file i/o applications, in that it avoids copying between the + output and the sliding window by simply making the window itself the output + buffer. inflate() can be faster on modern CPUs when used with large + buffers. inflateBack() trusts the application to not change the output + buffer passed by the output function, at least until inflateBack() returns. inflateBackInit() must be called first to allocate the internal state and to initialize the state with the user-provided window buffer. inflateBack() may then be used multiple times to inflate a complete, raw - deflate stream with each call. inflateBackEnd() is then called to free - the allocated state. + deflate stream with each call. inflateBackEnd() is then called to free the + allocated state. A raw deflate stream is one with no zlib or gzip header or trailer. This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the - header and process the trailer on its own, hence this routine expects - only the raw deflate stream to decompress. This is different from the - normal behavior of inflate(), which expects either a zlib or gzip header and - trailer around the deflate stream. + header and process the trailer on its own, hence this routine expects only + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those @@ -909,12 +1126,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If - there is no input available, in() must return zero--buf is ignored in that - case--and inflateBack() will return a buffer error. inflateBack() will call - out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() - should return zero on success, or non-zero on failure. If out() returns - non-zero, inflateBack() will return with an error. Neither in() nor out() - are permitted to change the contents of the window provided to + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). @@ -925,7 +1142,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in must also be initialized, and then if strm->avail_in is not zero, input will - initially be taken from strm->next_in[0 .. strm->avail_in - 1]. + initially be taken from strm->next_in[0 .. strm->avail_in - 1]. The in_desc and out_desc parameters of inflateBack() is passed as the first parameter of in() and out() respectively when they are called. These @@ -935,15 +1152,15 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, On return, inflateBack() will set strm->next_in and strm->avail_in to pass back any unused input that was provided by the last in() call. The return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR - if in() or out() returned an error, Z_DATA_ERROR if there was a format - error in the deflate stream (in which case strm->msg is set to indicate the - nature of the error), or Z_STREAM_ERROR if the stream was not properly - initialized. In the case of Z_BUF_ERROR, an input or output error can be - distinguished using strm->next_in which will be Z_NULL only if in() returned - an error. If strm->next is not Z_NULL, then the Z_BUF_ERROR was due to - out() returning non-zero. (in() will always be called before out(), so - strm->next_in is assured to be defined if out() returns non-zero.) Note - that inflateBack() cannot return Z_OK. + if in() or out() returned an error, Z_DATA_ERROR if there was a format error + in the deflate stream (in which case strm->msg is set to indicate the nature + of the error), or Z_STREAM_ERROR if the stream was not properly initialized. + In the case of Z_BUF_ERROR, an input or output error can be distinguished + using strm->next_in which will be Z_NULL only if in() returned an error. If + strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning + non-zero. (in() will always be called before out(), so strm->next_in is + assured to be defined if out() returns non-zero.) Note that inflateBack() + cannot return Z_OK. */ ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); @@ -964,7 +1181,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 7.6: size of z_off_t Compiler, assembler, and debug options: - 8: DEBUG + 8: ZLIB_DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) @@ -995,27 +1212,28 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 27-31: 0 (reserved) */ +#ifndef Z_SOLO /* utility functions */ /* - The following utility functions are implemented on top of the - basic stream-oriented functions. To simplify the interface, some - default options are assumed (compression level and memory usage, - standard memory allocation functions). The source code of these - utility functions can easily be modified if you need special options. + The following utility functions are implemented on top of the basic + stream-oriented functions. To simplify the interface, some default options + are assumed (compression level and memory usage, standard memory allocation + functions). The source code of these utility functions can be modified if + you need special options. */ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)); /* Compresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be at least the value returned - by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. - This function can be used to compress a whole file at once if the - input file is mmap'ed. + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. + compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer. @@ -1025,12 +1243,12 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)); /* - Compresses the source buffer into the destination buffer. The level + Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte - length of the source buffer. Upon entry, destLen is the total size of the + length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by - compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, @@ -1040,159 +1258,306 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); /* compressBound() returns an upper bound on the compressed size after - compress() or compress2() on sourceLen bytes. It would be used before - a compress() or compress2() call to allocate the destination buffer. + compress() or compress2() on sourceLen bytes. It would be used before a + compress() or compress2() call to allocate the destination buffer. */ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)); /* Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - This function can be used to decompress a whole file at once if the - input file is mmap'ed. + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, destLen + is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In + the case where there is not enough room, uncompress() will fill the output + buffer with the uncompressed data up to that point. */ - -typedef voidp gzFile; - -ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); /* - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb") but can also include a compression level - ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for - Huffman only compression as in "wb1h", or 'R' for run-length encoding - as in "wb1R". (See the description of deflateInit2 for more information - about the strategy parameter.) + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + + /* gzip file access functions */ + +/* + This library supports reading and writing files in gzip (.gz) format with + an interface similar to that of stdio, using the functions that start with + "gz". The gzip format is different from the zlib format. gzip is a gzip + wrapper, documented in RFC 1952, wrapped around a deflate stream. +*/ + +typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ + +/* +ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); + + Opens a gzip (.gz) file for reading or writing. The mode parameter is as + in fopen ("rb" or "wb") but can also include a compression level ("wb9") or + a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only + compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F' + for fixed code compression as in "wb9F". (See the description of + deflateInit2 for more information about the strategy parameter.) 'T' will + request transparent writing or appending with no compression and not using + the gzip format. + + "a" can be used instead of "w" to request that the gzip stream that will + be written be appended to the file. "+" will result in an error, since + reading and writing to the same gzip file is not supported. The addition of + "x" when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of "e" when + reading or writing will set the flag to close the file on an execve() call. + + These functions, as well as gzip, will read and decode a sequence of gzip + streams in a file. The append function of gzopen() can be used to create + such a file. (Also see gzflush() for another way to do this.) When + appending, gzopen does not test whether the file begins with a gzip stream, + nor does it look for the end of the gzip streams to begin appending. gzopen + will simply append a gzip stream to the existing file. gzopen can be used to read a file which is not in gzip format; in this - case gzread will directly read from the file without decompression. + case gzread will directly read from the file without decompression. When + reading, this will be detected automatically by looking for the magic two- + byte gzip header. - gzopen returns NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). */ + gzopen returns NULL if the file could not be opened, if there was + insufficient memory to allocate the gzFile state, or if an invalid mode was + specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). + errno can be checked to determine if the reason gzopen failed was that the + file could not be opened. +*/ -ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); +ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); /* - gzdopen() associates a gzFile with the file descriptor fd. File - descriptors are obtained from calls like open, dup, creat, pipe or - fileno (in the file has been previously opened with fopen). - The mode parameter is as in gzopen. - The next call of gzclose on the returned gzFile will also close the - file descriptor fd, just like fclose(fdopen(fd), mode) closes the file - descriptor fd. If you want to keep fd open, use gzdopen(dup(fd), mode). - gzdopen returns NULL if there was insufficient memory to allocate - the (de)compression state. + gzdopen associates a gzFile with the file descriptor fd. File descriptors + are obtained from calls like open, dup, creat, pipe or fileno (if the file + has been previously opened with fopen). The mode parameter is as in gzopen. + + The next call of gzclose on the returned gzFile will also close the file + descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor + fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, + mode);. The duplicated descriptor should be saved to avoid a leak, since + gzdopen does not close fd if it fails. If you are using fileno() to get the + file descriptor from a FILE *, then you will have to use dup() to avoid + double-close()ing the file descriptor. Both gzclose() and fclose() will + close the associated file descriptor, so they need to have different file + descriptors. + + gzdopen returns NULL if there was insufficient memory to allocate the + gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not + provided, or '+' was provided), or if fd is -1. The file descriptor is not + used until the next gz* read, write, seek, or close operation, so gzdopen + will not detect if fd is invalid (unless fd is -1). +*/ + +ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); +/* + Set the internal buffer size used by this library's functions. The + default buffer size is 8192 bytes. This function must be called after + gzopen() or gzdopen(), and before any other calls that read or write the + file. The buffer memory allocation is always deferred to the first read or + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). + + The new buffer size also affects the maximum length for gzprintf(). + + gzbuffer() returns 0 on success, or -1 on failure, such as being called + too late. */ ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* - Dynamically update the compression level or strategy. See the description - of deflateInit2 for the meaning of these parameters. - gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not - opened for writing. + Dynamically update the compression level or strategy. See the description + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. + + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. */ -ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); +ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); /* - Reads the given number of uncompressed bytes from the compressed file. - If the input file was not in gzip format, gzread copies the given number - of bytes into the buffer. - gzread returns the number of uncompressed bytes actually read (0 for - end of file, -1 for error). */ + Reads the given number of uncompressed bytes from the compressed file. If + the input file is not in gzip format, gzread copies the given number of + bytes into the buffer directly from the file. -ZEXTERN int ZEXPORT gzwrite OF((gzFile file, - voidpc buf, unsigned len)); + After reaching the end of a gzip stream in the input, gzread will continue + to read, looking for another gzip stream. Any number of gzip streams may be + concatenated in the input file, and will all be decompressed by gzread(). + If something other than a gzip stream is encountered after a gzip stream, + that remaining trailing garbage is ignored (and no error is returned). + + gzread can be used to read a gzip file that is being concurrently written. + Upon reaching the end of the input, gzread will return with the available + data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then + gzclearerr can be used to clear the end of file indicator in order to permit + gzread to be tried again. Z_OK indicates that a gzip stream was completed + on the last gzread. Z_BUF_ERROR indicates that the input file ended in the + middle of a gzip stream. Note that gzread does not return -1 in the event + of an incomplete gzip stream. This error is deferred until gzclose(), which + will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip + stream. Alternatively, gzerror can be used before gzclose to detect this + case. + + gzread returns the number of uncompressed bytes actually read, less than + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. +*/ + +ZEXTERN int ZEXPORT gzwrite OF((gzFile file, + voidpc buf, unsigned len)); /* Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of uncompressed bytes actually written - (0 in case of error). + gzwrite returns the number of uncompressed bytes written or 0 in case of + error. */ -ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...)); +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); /* - Converts, formats, and writes the args to the compressed file under - control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written (0 in case of error). The number of - uncompressed bytes written is limited to 4095. The caller should assure that - this limit is not exceeded. If it is exceeded, then gzprintf() will return - return an error (0) with nothing written. In this case, there may also be a + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + +ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); +/* + Converts, formats, and writes the arguments to the compressed file under + control of the format string, as in fprintf. gzprintf returns the number of + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a buffer overflow with unpredictable consequences, which is possible only if zlib was compiled with the insecure functions sprintf() or vsprintf() because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); /* - Writes the given null-terminated string to the compressed file, excluding + Writes the given null-terminated string to the compressed file, excluding the terminating null character. - gzputs returns the number of characters written, or -1 in case of error. + + gzputs returns the number of characters written, or -1 in case of error. */ ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); /* - Reads bytes from the compressed file until len-1 characters are read, or - a newline character is read and transferred to buf, or an end-of-file - condition is encountered. The string is then terminated with a null - character. - gzgets returns buf, or Z_NULL in case of error. + Reads bytes from the compressed file until len-1 characters are read, or a + newline character is read and transferred to buf, or an end-of-file + condition is encountered. If any characters are read or if len == 1, the + string is terminated with a null character. If no characters are read due + to an end-of-file or len < 1, then the buffer is left untouched. + + gzgets returns buf which is a null-terminated string, or it returns NULL + for end-of-file or in case of error. If there was an error, the contents at + buf are indeterminate. */ -ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); +ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); /* - Writes c, converted to an unsigned char, into the compressed file. - gzputc returns the value that was written, or -1 in case of error. + Writes c, converted to an unsigned char, into the compressed file. gzputc + returns the value that was written, or -1 in case of error. */ -ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); /* - Reads one byte from the compressed file. gzgetc returns this byte - or -1 in case of end of file or error. + Reads one byte from the compressed file. gzgetc returns this byte or -1 + in case of end of file or error. This is implemented as a macro for speed. + As such, it does not do all of the checking the other functions do. I.e. + it does not check to see if file is NULL, nor whether the structure file + points to has been clobbered or not. */ -ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); +ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); /* - Push one character back onto the stream to be read again later. - Only one character of push-back is allowed. gzungetc() returns the - character pushed, or -1 on failure. gzungetc() will fail if a - character has been pushed but not read yet, or if c is -1. The pushed - character will be discarded if the stream is repositioned with gzseek() - or gzrewind(). + Push one character back onto the stream to be read as the first character + on the next read. At least one character of push-back is allowed. + gzungetc() returns the character pushed, or -1 on failure. gzungetc() will + fail if c is -1, and may fail if a character has been pushed but not read + yet. If gzungetc is used immediately after gzopen or gzdopen, at least the + output buffer size of pushed characters is allowed. (See gzbuffer above.) + The pushed character will be discarded if the stream is repositioned with + gzseek() or gzrewind(). */ -ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); +ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); /* - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. The return value is the zlib - error number (see function gzerror below). gzflush returns Z_OK if - the flush parameter is Z_FINISH and all output could be flushed. - gzflush should be called only when strictly necessary because it can - degrade compression. + Flushes all pending output into the compressed file. The parameter flush + is as in the deflate() function. The return value is the zlib error number + (see function gzerror below). gzflush is only permitted when writing. + + If the flush parameter is Z_FINISH, the remaining data is written and the + gzip stream is completed in the output. If gzwrite() is called again, a new + gzip stream will be started in the output. gzread() is able to read such + concatenated gzip streams. + + gzflush should be called only when strictly necessary because it will + degrade compression if called too often. */ -ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, - z_off_t offset, int whence)); /* - Sets the starting position for the next gzread or gzwrite on the - given compressed file. The offset represents a number of bytes in the - uncompressed data stream. The whence parameter is defined as in lseek(2); +ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, + z_off_t offset, int whence)); + + Sets the starting position for the next gzread or gzwrite on the given + compressed file. The offset represents a number of bytes in the + uncompressed data stream. The whence parameter is defined as in lseek(2); the value SEEK_END is not supported. + If the file is opened for reading, this function is emulated but can be - extremely slow. If the file is opened for writing, only forward seeks are + extremely slow. If the file is opened for writing, only forward seeks are supported; gzseek then compresses a sequence of zeroes up to the new starting position. - gzseek returns the resulting offset location as measured in bytes from + gzseek returns the resulting offset location as measured in bytes from the beginning of the uncompressed stream, or -1 in case of error, in particular if the file is opened for writing and the new starting position would be before the current position. @@ -1202,68 +1567,134 @@ ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); /* Rewinds the given file. This function is supported only for reading. - gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET) + gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET) */ -ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); /* - Returns the starting position for the next gzread or gzwrite on the - given compressed file. This position represents a number of bytes in the - uncompressed data stream. +ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); - gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) + Returns the starting position for the next gzread or gzwrite on the given + compressed file. This position represents a number of bytes in the + uncompressed data stream, and is zero when starting, even if appending or + reading a gzip stream from the middle of a file using gzdopen(). + + gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); + + Returns the current offset in the file being read or written. This offset + includes the count of bytes that precede the gzip stream, for example when + appending or when using gzdopen() for reading. When reading, the offset + does not include as yet unused buffered input. This information can be used + for a progress indicator. On error, gzoffset() returns -1. */ ZEXTERN int ZEXPORT gzeof OF((gzFile file)); /* - Returns 1 when EOF has previously been detected reading the given - input stream, otherwise zero. + Returns true (1) if the end-of-file indicator has been set while reading, + false (0) otherwise. Note that the end-of-file indicator is set only if the + read tried to go past the end of the input, but came up short. Therefore, + just like feof(), gzeof() may return false even if there is no more data to + read, in the event that the last read request was for the exact number of + bytes remaining in the input file. This will happen if the input file size + is an exact multiple of the buffer size. + + If gzeof() returns true, then the read functions will return no more data, + unless the end-of-file indicator is reset by gzclearerr() and the input file + has grown since the previous end of file was detected. */ ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); /* - Returns 1 if file is being read directly without decompression, otherwise - zero. + Returns true (1) if file is being copied directly while reading, or false + (0) if file is a gzip stream being decompressed. + + If the input file is empty, gzdirect() will return true, since the input + does not contain a gzip stream. + + If gzdirect() is used immediately after gzopen() or gzdopen() it will + cause buffers to be allocated to allow reading the file to determine if it + is a gzip file. Therefore if gzbuffer() is used, it should be called before + gzdirect(). + + When writing, gzdirect() returns true (1) if transparent writing was + requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: + gzdirect() is not needed when writing. Transparent writing must be + explicitly requested, so the application already knows the answer. When + linking statically, using gzdirect() will include all of the zlib code for + gzip file reading and decompression, which may not be desired.) */ ZEXTERN int ZEXPORT gzclose OF((gzFile file)); /* - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. The return value is the zlib - error number (see function gzerror below). + Flushes all pending output if necessary, closes the compressed file and + deallocates the (de)compression state. Note that once file is closed, you + cannot call gzerror with file, since its structures have been deallocated. + gzclose must not be called more than once on the same file, just as free + must not be called more than once on the same allocation. + + gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a + file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the + last read ended in the middle of a gzip stream, or Z_OK on success. +*/ + +ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); +/* + Same as gzclose(), but gzclose_r() is only for use when reading, and + gzclose_w() is only for use when writing or appending. The advantage to + using these instead of gzclose() is that they avoid linking in zlib + compression or decompression code that is not used when only reading or only + writing respectively. If gzclose() is used, then both compression and + decompression code will be included the application when linking to a static + zlib library. */ ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); /* - Returns the error message for the last error which occurred on the - given compressed file. errnum is set to zlib error number. If an - error occurred in the file system and not in the compression library, - errnum is set to Z_ERRNO and the application may consult errno - to get the exact error code. + Returns the error message for the last error which occurred on the given + compressed file. errnum is set to zlib error number. If an error occurred + in the file system and not in the compression library, errnum is set to + Z_ERRNO and the application may consult errno to get the exact error code. + + The application must not modify the returned string. Future calls to + this function may invalidate the previously returned string. If file is + closed, then the string previously returned by gzerror will no longer be + available. + + gzerror() should be used to distinguish errors from end-of-file for those + functions above that do not distinguish those cases in their return values. */ ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); /* - Clears the error and end-of-file flags for file. This is analogous to the - clearerr() function in stdio. This is useful for continuing to read a gzip + Clears the error and end-of-file flags for file. This is analogous to the + clearerr() function in stdio. This is useful for continuing to read a gzip file that is being written concurrently. */ +#endif /* !Z_SOLO */ + /* checksum functions */ /* These functions are not related to compression but are exported - anyway because they might be useful in applications using the - compression library. + anyway because they might be useful in applications using the compression + library. */ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); /* Update a running Adler-32 checksum with the bytes buf[0..len-1] and - return the updated checksum. If buf is NULL, this function returns - the required initial value for the checksum. - An Adler-32 checksum is almost as reliable as a CRC32 but can be computed - much faster. Usage example: + return the updated checksum. If buf is Z_NULL, this function returns the + required initial value for the checksum. + + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed + much faster. + + Usage example: uLong adler = adler32(0L, Z_NULL, 0); @@ -1273,21 +1704,31 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + +/* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); -/* + Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of - seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note + that the z_off_t type (like off_t) is a signed integer. If len2 is + negative, the result has no meaning or utility. */ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); /* Update a running CRC-32 with the bytes buf[0..len-1] and return the - updated CRC-32. If buf is NULL, this function returns the required initial - value for the for the crc. Pre- and post-conditioning (one's complement) is + updated CRC-32. If buf is Z_NULL, this function returns the required + initial value for the crc. Pre- and post-conditioning (one's complement) is performed within this function so it shouldn't be done by the application. + Usage example: uLong crc = crc32(0L, Z_NULL, 0); @@ -1298,9 +1739,15 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ -ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ /* +ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); + Combine two CRC-32 check values into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, CRC-32 check values were calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 @@ -1328,27 +1775,135 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); -#define deflateInit(strm, level) \ - deflateInit_((strm), (level), ZLIB_VERSION, sizeof(z_stream)) -#define inflateInit(strm) \ - inflateInit_((strm), ZLIB_VERSION, sizeof(z_stream)) -#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ - deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ - (strategy), ZLIB_VERSION, sizeof(z_stream)) -#define inflateInit2(strm, windowBits) \ - inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream)) -#define inflateBackInit(strm, windowBits, window) \ - inflateBackInit_((strm), (windowBits), (window), \ - ZLIB_VERSION, sizeof(z_stream)) - - -#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) - struct internal_state {int dummy;}; /* hack for buggy compilers */ +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) #endif +#ifndef Z_SOLO + +/* gzgetc() macro and its supporting function and exposed data structure. Note + * that the real internal state is much larger than the exposed structure. + * This abbreviated structure exposes just enough for the gzgetc() macro. The + * user should not mess with these exposed elements, since their names or + * behavior could change in the future, perhaps even capriciously. They can + * only be used by the gzgetc() macro. You have been warned. + */ +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +# define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#else +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#endif + +/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or + * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if + * both are true, the application gets the *64 functions, and the regular + * functions are changed to 64 bits) -- in case these are set on systems + * without large file support, _LFS64_LARGEFILE must also be true + */ +#ifdef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); +#endif + +#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) +# ifdef Z_PREFIX_SET +# define z_gzopen z_gzopen64 +# define z_gzseek z_gzseek64 +# define z_gztell z_gztell64 +# define z_gzoffset z_gzoffset64 +# define z_adler32_combine z_adler32_combine64 +# define z_crc32_combine z_crc32_combine64 +# else +# define gzopen gzopen64 +# define gzseek gzseek64 +# define gztell gztell64 +# define gzoffset gzoffset64 +# define adler32_combine adler32_combine64 +# define crc32_combine crc32_combine64 +# endif +# ifndef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); +# endif +#else + ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); +#endif + +#else /* Z_SOLO */ + + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + +#endif /* !Z_SOLO */ + +/* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); -ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z)); -ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void)); +ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); +ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); +ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); +#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + const char *format, + va_list va)); +# endif +#endif #ifdef __cplusplus } diff --git a/zlib/zlib.pc.cmakein b/zlib/zlib.pc.cmakein new file mode 100644 index 00000000000..a5e642938c6 --- /dev/null +++ b/zlib/zlib.pc.cmakein @@ -0,0 +1,13 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@INSTALL_LIB_DIR@ +sharedlibdir=@INSTALL_LIB_DIR@ +includedir=@INSTALL_INC_DIR@ + +Name: zlib +Description: zlib compression library +Version: @VERSION@ + +Requires: +Libs: -L${libdir} -L${sharedlibdir} -lz +Cflags: -I${includedir} diff --git a/zlib/zlib.pc.in b/zlib/zlib.pc.in new file mode 100644 index 00000000000..7e5acf9c77e --- /dev/null +++ b/zlib/zlib.pc.in @@ -0,0 +1,13 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +sharedlibdir=@sharedlibdir@ +includedir=@includedir@ + +Name: zlib +Description: zlib compression library +Version: @VERSION@ + +Requires: +Libs: -L${libdir} -L${sharedlibdir} -lz +Cflags: -I${includedir} diff --git a/zlib/zlib2ansi b/zlib/zlib2ansi new file mode 100755 index 00000000000..15e3e165f37 --- /dev/null +++ b/zlib/zlib2ansi @@ -0,0 +1,152 @@ +#!/usr/bin/perl + +# Transform K&R C function definitions into ANSI equivalent. +# +# Author: Paul Marquess +# Version: 1.0 +# Date: 3 October 2006 + +# TODO +# +# Asumes no function pointer parameters. unless they are typedefed. +# Assumes no literal strings that look like function definitions +# Assumes functions start at the beginning of a line + +use strict; +use warnings; + +local $/; +$_ = <>; + +my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments + +my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; +my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; +my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; + + +while (s/^ + ( # Start $1 + ( # Start $2 + .*? # Minimal eat content + ( ^ \w [\w\s\*]+ ) # $3 -- function name + \s* # optional whitespace + ) # $2 - Matched up to before parameter list + + \( \s* # Literal "(" + optional whitespace + ( [^\)]+ ) # $4 - one or more anythings except ")" + \s* \) # optional whitespace surrounding a Literal ")" + + ( (?: $dList )+ ) # $5 + + $sp ^ { # literal "{" at start of line + ) # Remember to $1 + //xsom + ) +{ + my $all = $1 ; + my $prefix = $2; + my $param_list = $4 ; + my $params = $5; + + StripComments($params); + StripComments($param_list); + $param_list =~ s/^\s+//; + $param_list =~ s/\s+$//; + + my $i = 0 ; + my %pList = map { $_ => $i++ } + split /\s*,\s*/, $param_list; + my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; + + my @params = split /\s*;\s*/, $params; + my @outParams = (); + foreach my $p (@params) + { + if ($p =~ /,/) + { + my @bits = split /\s*,\s*/, $p; + my $first = shift @bits; + $first =~ s/^\s*//; + push @outParams, $first; + $first =~ /^(\w+\s*)/; + my $type = $1 ; + push @outParams, map { $type . $_ } @bits; + } + else + { + $p =~ s/^\s+//; + push @outParams, $p; + } + } + + + my %tmp = map { /$pMatch/; $_ => $pList{$1} } + @outParams ; + + @outParams = map { " $_" } + sort { $tmp{$a} <=> $tmp{$b} } + @outParams ; + + print $prefix ; + print "(\n" . join(",\n", @outParams) . ")\n"; + print "{" ; + +} + +# Output any trailing code. +print ; +exit 0; + + +sub StripComments +{ + + no warnings; + + # Strip C & C++ coments + # From the perlfaq + $_[0] =~ + + s{ + /\* ## Start of /* ... */ comment + [^*]*\*+ ## Non-* followed by 1-or-more *'s + ( + [^/*][^*]*\*+ + )* ## 0-or-more things which don't start with / + ## but do end with '*' + / ## End of /* ... */ comment + + | ## OR C++ Comment + // ## Start of C++ comment // + [^\n]* ## followed by 0-or-more non end of line characters + + | ## OR various things which aren't comments: + + ( + " ## Start of " ... " string + ( + \\. ## Escaped char + | ## OR + [^"\\] ## Non "\ + )* + " ## End of " ... " string + + | ## OR + + ' ## Start of ' ... ' string + ( + \\. ## Escaped char + | ## OR + [^'\\] ## Non '\ + )* + ' ## End of ' ... ' string + + | ## OR + + . ## Anything other char + [^/"'\\]* ## Chars which doesn't start a comment, string or escape + ) + }{$2}gxs; + +} diff --git a/zlib/zutil.c b/zlib/zutil.c index d55f5948a37..a76c6b0c7e5 100644 --- a/zlib/zutil.c +++ b/zlib/zutil.c @@ -1,27 +1,27 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #include "zutil.h" - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ +#ifndef Z_SOLO +# include "gzguts.h" #endif -const char * const z_errmsg[10] = { -"need dictionary", /* Z_NEED_DICT 2 */ -"stream end", /* Z_STREAM_END 1 */ -"", /* Z_OK 0 */ -"file error", /* Z_ERRNO (-1) */ -"stream error", /* Z_STREAM_ERROR (-2) */ -"data error", /* Z_DATA_ERROR (-3) */ -"insufficient memory", /* Z_MEM_ERROR (-4) */ -"buffer error", /* Z_BUF_ERROR (-5) */ -"incompatible version",/* Z_VERSION_ERROR (-6) */ -""}; +z_const char * const z_errmsg[10] = { + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; const char * ZEXPORT zlibVersion() @@ -34,31 +34,31 @@ uLong ZEXPORT zlibCompileFlags() uLong flags; flags = 0; - switch (sizeof(uInt)) { + switch ((int)(sizeof(uInt))) { case 2: break; case 4: flags += 1; break; case 8: flags += 2; break; default: flags += 3; } - switch (sizeof(uLong)) { + switch ((int)(sizeof(uLong))) { case 2: break; case 4: flags += 1 << 2; break; case 8: flags += 2 << 2; break; default: flags += 3 << 2; } - switch (sizeof(voidpf)) { + switch ((int)(sizeof(voidpf))) { case 2: break; case 4: flags += 1 << 4; break; case 8: flags += 2 << 4; break; default: flags += 3 << 4; } - switch (sizeof(z_off_t)) { + switch ((int)(sizeof(z_off_t))) { case 2: break; case 4: flags += 1 << 6; break; case 8: flags += 2 << 6; break; default: flags += 3 << 6; } -#ifdef DEBUG +#ifdef ZLIB_DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) @@ -85,41 +85,41 @@ uLong ZEXPORT zlibCompileFlags() #ifdef FASTEST flags += 1L << 21; #endif -#ifdef STDC +#if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifdef NO_vsnprintf - flags += 1L << 25; + flags += 1L << 25; # ifdef HAS_vsprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # else # ifdef HAS_vsnprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # endif #else - flags += 1L << 24; + flags += 1L << 24; # ifdef NO_snprintf - flags += 1L << 25; + flags += 1L << 25; # ifdef HAS_sprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # else # ifdef HAS_snprintf_void - flags += 1L << 26; + flags += 1L << 26; # endif # endif #endif return flags; } -#ifdef DEBUG - +#ifdef ZLIB_DEBUG +#include # ifndef verbose # define verbose 0 # endif -int z_verbose = verbose; +int ZLIB_INTERNAL z_verbose = verbose; -void z_error (m) +void ZLIB_INTERNAL z_error (m) char *m; { fprintf(stderr, "%s\n", m); @@ -146,7 +146,7 @@ const char * ZEXPORT zError(err) #ifndef HAVE_MEMCPY -void zmemcpy(dest, source, len) +void ZLIB_INTERNAL zmemcpy(dest, source, len) Bytef* dest; const Bytef* source; uInt len; @@ -157,7 +157,7 @@ void zmemcpy(dest, source, len) } while (--len != 0); } -int zmemcmp(s1, s2, len) +int ZLIB_INTERNAL zmemcmp(s1, s2, len) const Bytef* s1; const Bytef* s2; uInt len; @@ -170,7 +170,7 @@ int zmemcmp(s1, s2, len) return 0; } -void zmemzero(dest, len) +void ZLIB_INTERNAL zmemzero(dest, len) Bytef* dest; uInt len; { @@ -181,6 +181,7 @@ void zmemzero(dest, len) } #endif +#ifndef Z_SOLO #ifdef SYS16BIT @@ -213,11 +214,13 @@ local ptr_table table[MAX_PTR]; * a protected system like OS/2. Use Microsoft C instead. */ -voidpf zcalloc (voidpf opaque, unsigned items, unsigned size) +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { - voidpf buf = opaque; /* just to make some compilers happy */ + voidpf buf; ulg bsize = (ulg)items*size; + (void)opaque; + /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ @@ -237,9 +240,12 @@ voidpf zcalloc (voidpf opaque, unsigned items, unsigned size) return buf; } -void zcfree (voidpf opaque, voidpf ptr) +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; + + (void)opaque; + if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; @@ -255,7 +261,6 @@ void zcfree (voidpf opaque, voidpf ptr) next_ptr--; return; } - ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } @@ -272,15 +277,15 @@ void zcfree (voidpf opaque, voidpf ptr) # define _hfree hfree #endif -voidpf zcalloc (voidpf opaque, unsigned items, unsigned size) +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; return _halloc((long)items, size); } -void zcfree (voidpf opaque, voidpf ptr) +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; _hfree(ptr); } @@ -297,22 +302,24 @@ extern voidp calloc OF((uInt items, uInt size)); extern void free OF((voidpf ptr)); #endif -voidpf zcalloc (opaque, items, size) +voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) voidpf opaque; unsigned items; unsigned size; { - if (opaque) items += size - size; /* make compiler happy */ + (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } -void zcfree (opaque, ptr) +void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { + (void)opaque; free(ptr); - if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ + +#endif /* !Z_SOLO */ diff --git a/zlib/zutil.h b/zlib/zutil.h index 4482957d6ac..b079ea6a80f 100644 --- a/zlib/zutil.h +++ b/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -13,36 +13,32 @@ #ifndef ZUTIL_H #define ZUTIL_H -#define ZLIB_INTERNAL +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + #include "zlib.h" -#ifdef STDC -# ifndef _WIN32_WCE +#if defined(STDC) && !defined(Z_SOLO) +# if !(defined(_WIN32_WCE) && defined(_MSC_VER)) # include # endif # include # include #endif -#ifdef NO_ERRNO_H -# ifdef _WIN32_WCE - /* The Microsoft C Run-Time Library for Windows CE doesn't have - * errno. We define it as a global variable to simplify porting. - * Its value is always 0 and should not be used. We rename it to - * avoid conflict with other libraries that use the same workaround. - */ -# define errno z_errno -# endif - extern int errno; -#else -# ifndef _WIN32_WCE -# include -# endif + +#ifdef Z_SOLO + typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ #endif #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ typedef unsigned char uch; typedef uch FAR uchf; @@ -50,13 +46,13 @@ typedef unsigned short ush; typedef ush FAR ushf; typedef unsigned long ulg; -extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ +extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] #define ERR_RETURN(strm,err) \ - return (strm->msg = (char*)ERR_MSG(err), (err)) + return (strm->msg = ERR_MSG(err), (err)) /* To be used only when the state is known to be valid */ /* common constants */ @@ -88,70 +84,90 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) # define OS_CODE 0x00 -# if defined(__TURBOC__) || defined(__BORLANDC__) -# if(__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) - /* Allow compilation with ANSI keywords only enabled */ - void _Cdecl farfree( void *block ); - void *_Cdecl farmalloc( unsigned long nbytes ); -# else -# include +# ifndef Z_SOLO +# if defined(__TURBOC__) || defined(__BORLANDC__) +# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) + /* Allow compilation with ANSI keywords only enabled */ + void _Cdecl farfree( void *block ); + void *_Cdecl farmalloc( unsigned long nbytes ); +# else +# include +# endif +# else /* MSC or DJGPP */ +# include # endif -# else /* MSC or DJGPP */ -# include # endif #endif #ifdef AMIGA -# define OS_CODE 0x01 +# define OS_CODE 1 #endif #if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 +# define OS_CODE 2 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + #if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 +# define OS_CODE 5 #endif #ifdef OS2 -# define OS_CODE 0x06 -# ifdef M_I86 - #include +# define OS_CODE 6 +# if defined(M_I86) && !defined(Z_SOLO) +# include # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) -# define OS_CODE 0x07 -# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os -# include /* for fdopen */ -# else -# ifndef fdopen -# define fdopen(fd,mode) NULL /* No fdopen() */ +# define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif # endif # endif #endif -#ifdef TOPS20 -# define OS_CODE 0x0a +#ifdef __acorn +# define OS_CODE 13 #endif -#ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ -# define OS_CODE 0x0b -# endif +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 #endif -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0f +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 +#endif + +#ifdef __APPLE__ +# define OS_CODE 19 #endif #if defined(_BEOS_) || defined(RISCOS) # define fdopen(fd,mode) NULL /* No fdopen() */ #endif -#if (defined(_MSC_VER) && (_MSC_VER > 600)) +#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX # if defined(_WIN32_WCE) # define fdopen(fd,mode) NULL /* No fdopen() */ # ifndef _PTRDIFF_T_DEFINED @@ -163,10 +179,23 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif +#if defined(__BORLANDC__) && !defined(MSDOS) + #pragma warn -8004 + #pragma warn -8008 + #pragma warn -8066 +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_WIN32) && \ + (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); +#endif + /* common defaults */ #ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ +# define OS_CODE 3 /* assume Unix */ #endif #ifndef F_OPEN @@ -175,40 +204,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* functions */ -#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) -# ifndef HAVE_VSNPRINTF -# define HAVE_VSNPRINTF -# endif -#endif -#if defined(__CYGWIN__) -# ifndef HAVE_VSNPRINTF -# define HAVE_VSNPRINTF -# endif -#endif -#ifndef HAVE_VSNPRINTF -# ifdef MSDOS - /* vsnprintf may exist on some MS-DOS compilers (DJGPP?), - but for now we just assume it doesn't. */ -# define NO_vsnprintf -# endif -# ifdef __TURBOC__ -# define NO_vsnprintf -# endif -# if defined(WIN32) && (!defined(_MSC_VER) || _MSC_VER < 1500) - /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ -# if !defined(vsnprintf) && !defined(NO_vsnprintf) -# define vsnprintf _vsnprintf -# endif -# endif -# ifdef __SASC -# define NO_vsnprintf -# endif -#endif -#ifdef VMS -# define NO_vsnprintf -#endif - -#if defined(pyr) +#if defined(pyr) || defined(Z_SOLO) # define NO_MEMCPY #endif #if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) @@ -232,16 +228,16 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define zmemzero(dest, len) memset(dest, 0, len) # endif #else - extern void zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); - extern int zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); - extern void zmemzero OF((Bytef* dest, uInt len)); + void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); + int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); + void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); #endif /* Diagnostic functions */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include - extern int z_verbose; - extern void z_error OF((char *m)); + extern int ZLIB_INTERNAL z_verbose; + extern void ZLIB_INTERNAL z_error OF((char *m)); # define Assert(cond,msg) {if(!(cond)) z_error(msg);} # define Trace(x) {if (z_verbose>=0) fprintf x ;} # define Tracev(x) {if (z_verbose>0) fprintf x ;} @@ -257,13 +253,19 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define Tracecv(c,x) #endif - -voidpf zcalloc OF((voidpf opaque, unsigned items, unsigned size)); -void zcfree OF((voidpf opaque, voidpf ptr)); +#ifndef Z_SOLO + voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, + unsigned size)); + void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); +#endif #define ZALLOC(strm, items, size) \ (*((strm)->zalloc))((strm)->opaque, (items), (size)) #define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) #define TRY_FREE(s, p) {if (p) ZFREE(s, p);} +/* Reverse the bytes in a 32-bit value */ +#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + #endif /* ZUTIL_H */ From b6c038d4366d003ae00120c96e39a892e5d675c1 Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Wed, 18 Jul 2018 10:22:21 +0200 Subject: [PATCH 008/127] Fix build break with modern compilers: client/mysql.cc: In function void build_completion_hash(bool, bool): client/mysql.cc:2674:37: error: invalid conversion from char to char* [-fpermissive] field_names[i][num_fields*2]= '\0'; --- client/mysql.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysql.cc b/client/mysql.cc index 8510361a54e..bc32a58f37e 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -2671,7 +2671,7 @@ You can turn off this feature to get a quicker startup with -A\n\n"); mysql_free_result(fields); break; } - field_names[i][num_fields*2]= '\0'; + field_names[i][num_fields*2]= NULL; j=0; while ((sql_field=mysql_fetch_field(fields))) { From 394be4f1d53b2c246cc7b5c429c3b83ebb64da1c Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Wed, 18 Jul 2018 10:40:47 +0200 Subject: [PATCH 009/127] Bug#28366648 MTR FOR MYSQL5.5 DOES NOT WORK WITH MODERN PERL VERSIONS Problem: Can't locate lib/mtr_process.pl in @INC The fix is to push current directory to @INC. --- mysql-test/mysql-test-run.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 99d3203fb51..8086c641d11 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -100,6 +100,8 @@ use mtr_results; use IO::Socket::INET; use IO::Select; +push @INC, "."; + require "lib/mtr_process.pl"; require "lib/mtr_io.pl"; require "lib/mtr_gcov.pl"; From 8a7db4c3208057f04d6af9f28c7b1e542e899343 Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Tue, 3 Jul 2018 12:30:19 +0200 Subject: [PATCH 010/127] Bug#28200422 USE CTAGS RATHER THAN ETAGS FOR GENERATING TAGS FILE Switch to Exuberant Ctags when generating TAGS, since it is much better at parsing modern C++ Change-Id: I9652012708df7e7edf93161097a547f60fb0cf79 (cherry picked from commit 125b2804fbbb8662632f761f39aeef0a7f9cebb3) --- cmake/tags.cmake | 35 +++++++++++++++++++++++++--------- support-files/build-tags | 41 ++++++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/cmake/tags.cmake b/cmake/tags.cmake index 07c1411a1d6..fdbe61694a3 100644 --- a/cmake/tags.cmake +++ b/cmake/tags.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,12 +15,29 @@ # Generate tag files IF(UNIX) - ADD_CUSTOM_TARGET (tags - COMMAND support-files/build-tags - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - ) - ADD_CUSTOM_TARGET (ctags - COMMAND ctags -R -f CTAGS - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - ) + FIND_PROGRAM(CTAGS_EXECUTABLE ctags) + IF(NOT CTAGS_EXECUTABLE) + RETURN() + ENDIF() + EXEC_PROGRAM(${CTAGS_EXECUTABLE} ARGS --version OUTPUT_VARIABLE CTAGS_VERSION) + + IF(CTAGS_VERSION MATCHES "Exuberant") + ADD_CUSTOM_TARGET(tags + COMMAND support-files/build-tags + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + ADD_CUSTOM_TARGET(ctags + COMMAND support-files/build-tags ctags + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + ELSE() + ADD_CUSTOM_TARGET (tags + COMMAND exit 1 + COMMENT "Please install Exuberant Ctags" + ) + ADD_CUSTOM_TARGET (ctags + COMMAND exit 1 + COMMENT "Please install Exuberant Ctags" + ) + ENDIF() ENDIF() diff --git a/support-files/build-tags b/support-files/build-tags index c37485e32f9..14af22b3ef1 100755 --- a/support-files/build-tags +++ b/support-files/build-tags @@ -1,12 +1,41 @@ -#! /bin/sh +#! /bin/bash -rm -f TAGS +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +tagstyle=${1:-etags} + +common_opts="--langmap=C++:+.ic,YACC:+.yy -I MY_ATTRIBUTE+" +case $tagstyle in + "etags") tagfile=TAGS + tagopt="-e $common_opts" + ;; + "ctags") tagfile=tags + tagopt="--fields=+l $common_opts" + ;; + *) echo "$0 [etags|ctags]" + exit 1 + ;; +esac + +rm -f $tagfile filter='\.cpp$\|\.cc$\|\.c$\|\.h$\|sql_yacc\.yy$\|\.hpp$\|\.ic$' list="find . -type f" git rev-parse >/dev/null 2>/dev/null && list="git ls-files" -$list |grep $filter |while read f; -do - etags -o TAGS --append $f -done +ctags $tagopt -o $tagfile $( $list | grep $filter ) + +echo "wrote file `pwd`/$tagfile" From 15015579877476b38140e2828c23b3b716cac1e9 Mon Sep 17 00:00:00 2001 From: Karthik Kamath Date: Mon, 23 Jul 2018 10:16:58 +0530 Subject: [PATCH 011/127] BUG#27788685: NO WARNING WHEN TRUNCATING A STRING WITH DATA LOSS ANALYSIS: ========= When converting from a BLOB/TEXT type to a smaller BLOB/TEXT type, no warning/error is reported to the user informing about the truncation/data loss. FIX: ==== We are now reporting a warning in non-strict mode and an appropriate error in strict mode. --- mysql-test/r/alter_table.result | 50 +++++++++++++++++++++++++++++++++ mysql-test/r/myisam-blob.result | 2 ++ mysql-test/t/alter_table.test | 49 ++++++++++++++++++++++++++++++++ sql/field_conv.cc | 20 +++++++++++-- 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 725f2528c29..1c16693bfc4 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -1505,3 +1505,53 @@ SELECT * FROM t1; 10 DROP TABLE t1; +# +# BUG#27788685: NO WARNING WHEN TRUNCATING A STRING WITH DATA LOSS +# +SET GLOBAL max_allowed_packet=17825792; +CREATE TABLE t1 (t1_fld1 TEXT) ENGINE=InnoDB; +CREATE TABLE t2 (t2_fld1 MEDIUMTEXT) ENGINE=InnoDB; +CREATE TABLE t3 (t3_fld1 LONGTEXT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (REPEAT('a',300)); +INSERT INTO t2 VALUES (REPEAT('b',65680)); +INSERT INTO t3 VALUES (REPEAT('c',16777300)); +SELECT LENGTH(t1_fld1) FROM t1; +LENGTH(t1_fld1) +300 +SELECT LENGTH(t2_fld1) FROM t2; +LENGTH(t2_fld1) +65680 +SELECT LENGTH(t3_fld1) FROM t3; +LENGTH(t3_fld1) +16777300 +# With strict mode +SET SQL_MODE='STRICT_ALL_TABLES'; +ALTER TABLE t1 CHANGE `t1_fld1` `my_t1_fld1` TINYTEXT; +ERROR 22001: Data too long for column 'my_t1_fld1' at row 1 +ALTER TABLE t2 CHANGE `t2_fld1` `my_t2_fld1` TEXT; +ERROR 22001: Data too long for column 'my_t2_fld1' at row 1 +ALTER TABLE t3 CHANGE `t3_fld1` `my_t3_fld1` MEDIUMTEXT; +ERROR 22001: Data too long for column 'my_t3_fld1' at row 1 +# With non-strict mode +SET SQL_MODE=''; +ALTER TABLE t1 CHANGE `t1_fld1` `my_t1_fld1` TINYTEXT; +Warnings: +Warning 1265 Data truncated for column 'my_t1_fld1' at row 1 +ALTER TABLE t2 CHANGE `t2_fld1` `my_t2_fld1` TEXT; +Warnings: +Warning 1265 Data truncated for column 'my_t2_fld1' at row 1 +ALTER TABLE t3 CHANGE `t3_fld1` `my_t3_fld1` MEDIUMTEXT; +Warnings: +Warning 1265 Data truncated for column 'my_t3_fld1' at row 1 +SELECT LENGTH(my_t1_fld1) FROM t1; +LENGTH(my_t1_fld1) +44 +SELECT LENGTH(my_t2_fld1) FROM t2; +LENGTH(my_t2_fld1) +144 +SELECT LENGTH(my_t3_fld1) FROM t3; +LENGTH(my_t3_fld1) +84 +DROP TABLE t1, t2, t3; +SET SQL_MODE=default; +SET GLOBAL max_allowed_packet=default; diff --git a/mysql-test/r/myisam-blob.result b/mysql-test/r/myisam-blob.result index 43db7c8badd..971682b689a 100644 --- a/mysql-test/r/myisam-blob.result +++ b/mysql-test/r/myisam-blob.result @@ -29,6 +29,8 @@ select length(data) from t1; length(data) 18874368 alter table t1 modify data blob; +Warnings: +Warning 1265 Data truncated for column 'data' at row 1 select length(data) from t1; length(data) 0 diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 8768cd4975c..36824343749 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1280,3 +1280,52 @@ ALTER TABLE t1 ADD INDEX (` SELECT * FROM t1; DROP TABLE t1; +--echo # +--echo # BUG#27788685: NO WARNING WHEN TRUNCATING A STRING WITH DATA LOSS +--echo # + +SET GLOBAL max_allowed_packet=17825792; + +--connect(con1, localhost, root,,) +CREATE TABLE t1 (t1_fld1 TEXT) ENGINE=InnoDB; +CREATE TABLE t2 (t2_fld1 MEDIUMTEXT) ENGINE=InnoDB; +CREATE TABLE t3 (t3_fld1 LONGTEXT) ENGINE=InnoDB; + +INSERT INTO t1 VALUES (REPEAT('a',300)); +INSERT INTO t2 VALUES (REPEAT('b',65680)); +INSERT INTO t3 VALUES (REPEAT('c',16777300)); + +SELECT LENGTH(t1_fld1) FROM t1; +SELECT LENGTH(t2_fld1) FROM t2; +SELECT LENGTH(t3_fld1) FROM t3; + +--echo # With strict mode +SET SQL_MODE='STRICT_ALL_TABLES'; + +--error ER_DATA_TOO_LONG +ALTER TABLE t1 CHANGE `t1_fld1` `my_t1_fld1` TINYTEXT; +--error ER_DATA_TOO_LONG +ALTER TABLE t2 CHANGE `t2_fld1` `my_t2_fld1` TEXT; +--error ER_DATA_TOO_LONG +ALTER TABLE t3 CHANGE `t3_fld1` `my_t3_fld1` MEDIUMTEXT; + +--echo # With non-strict mode +SET SQL_MODE=''; + +ALTER TABLE t1 CHANGE `t1_fld1` `my_t1_fld1` TINYTEXT; +ALTER TABLE t2 CHANGE `t2_fld1` `my_t2_fld1` TEXT; +ALTER TABLE t3 CHANGE `t3_fld1` `my_t3_fld1` MEDIUMTEXT; + +SELECT LENGTH(my_t1_fld1) FROM t1; +SELECT LENGTH(my_t2_fld1) FROM t2; +SELECT LENGTH(my_t3_fld1) FROM t3; + +# Cleanup +--disconnect con1 +--source include/wait_until_disconnected.inc + +--connection default +DROP TABLE t1, t2, t3; + +SET SQL_MODE=default; +SET GLOBAL max_allowed_packet=default; diff --git a/sql/field_conv.cc b/sql/field_conv.cc index d98f19c3e01..d19eef8fdae 100644 --- a/sql/field_conv.cc +++ b/sql/field_conv.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -287,9 +287,23 @@ static void do_copy_next_number(Copy_field *copy) static void do_copy_blob(Copy_field *copy) { - ulong length=((Field_blob*) copy->from_field)->get_length(); - ((Field_blob*) copy->to_field)->store_length(length); + ulong from_length=((Field_blob*) copy->from_field)->get_length(); + ((Field_blob*) copy->to_field)->store_length(from_length); memcpy(copy->to_ptr, copy->from_ptr, sizeof(char*)); + ulong to_length=((Field_blob*) copy->to_field)->get_length(); + if (to_length < from_length) + { + if (copy->to_field->table->in_use->abort_on_warning) + { + copy->to_field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, + ER_DATA_TOO_LONG, 1); + } + else + { + copy->to_field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, + WARN_DATA_TRUNCATED, 1); + } + } } static void do_conv_blob(Copy_field *copy) From fe5df42a3753dd5b183395c304ae80891da0dda7 Mon Sep 17 00:00:00 2001 From: Balasubramanian Kandasamy Date: Fri, 3 Aug 2018 13:40:48 +0530 Subject: [PATCH 012/127] BUG#28144933 - MYSQL-SERVER RPM DOES NOT INSTALL PERL-DATA-DUMPER AS A DEPENDENCY - Added Perl Data dumper as a dependency for server and test sub packages. --- support-files/mysql.spec.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index cb1462c57e5..0a8d68ed1af 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -145,8 +145,8 @@ %if "%elver" == "6" || "%elver" == "7" %define distro_description Oracle Linux %elver %define distro_releasetag el%elver - %define distro_buildreq gcc-c++ ncurses-devel perl readline-devel time zlib-devel cmake libaio-devel - %define distro_requires chkconfig coreutils grep procps shadow-utils net-tools + %define distro_buildreq gcc-c++ ncurses-devel perl perl(Data::Dumper) readline-devel time zlib-devel cmake libaio-devel + %define distro_requires chkconfig coreutils grep perl(Data::Dumper) procps shadow-utils net-tools %else %{error:Oracle Linux %{elver} is unsupported} %endif @@ -168,8 +168,8 @@ %if "%rhelver" == "6" %define distro_description Red Hat Enterprise Linux 6 %define distro_releasetag rhel6 - %define distro_buildreq gcc-c++ ncurses-devel perl readline-devel time zlib-devel cmake libaio-devel - %define distro_requires chkconfig coreutils grep procps shadow-utils net-tools + %define distro_buildreq gcc-c++ ncurses-devel perl perl(Data::Dumper) readline-devel time zlib-devel cmake libaio-devel + %define distro_requires chkconfig coreutils grep perl(Data::Dumper) procps shadow-utils net-tools %else %{error:Red Hat Enterprise Linux %{rhelver} is unsupported} %endif @@ -203,8 +203,8 @@ %define generic_kernel %(uname -r | cut -d. -f1-2) %define distro_description Generic Linux (kernel %{generic_kernel}) %define distro_releasetag linux%{generic_kernel} - %define distro_buildreq gcc-c++ gperf ncurses-devel perl readline-devel time zlib-devel - %define distro_requires coreutils grep procps /sbin/chkconfig /usr/sbin/useradd /usr/sbin/groupadd + %define distro_buildreq gcc-c++ gperf ncurses-devel perl perl(Data::Dumper) readline-devel time zlib-devel + %define distro_requires coreutils grep perl(Data::Dumper) procps /sbin/chkconfig /usr/sbin/useradd /usr/sbin/groupadd %endif # Avoid debuginfo RPMs, leaves binaries unstripped @@ -362,6 +362,7 @@ Obsoletes: MySQL-test-classic MySQL-test-community MySQL-test-enterprise Obsoletes: MySQL-test-advanced-gpl MySQL-test-enterprise-gpl Provides: mysql-test = %{version}-%{release} Provides: mysql-test%{?_isa} = %{version}-%{release} +Requires: perl(Data::Dumper) AutoReqProv: no %description -n MySQL-test%{product_suffix} From a653fca99be15b6c3bd6483041578f0bfe5bf5b0 Mon Sep 17 00:00:00 2001 From: Sreeharsha Ramanavarapu Date: Sun, 19 Aug 2018 10:00:30 +0530 Subject: [PATCH 013/127] Bug #26791931: INCORRECT BEHAVIOR IN ALTER TABLE REORGANIZE PARTITION Issue: ------ ALTER TABLE REORGANIZE PARTITION .... can result in incorrect behavior if any partition other than the last one misses the "VALUES LESS THAN..." part of the syntax. Root cause: ----------- Currently ALTER TABLE with changes to partitions is handled incorrectly by the parser. Fix: ---- The if condition which handles partition management differently for ALTER TABLE in the parser should be removed. Change the code to handle the case in the parser. --- sql/sql_yacc.yy | 65 +++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 2ca36e23652..328fb138196 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4670,56 +4670,47 @@ opt_part_values: { LEX *lex= Lex; partition_info *part_info= lex->part_info; - if (! lex->is_partition_management()) - { - if (part_info->part_type == RANGE_PARTITION) - { - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), - "RANGE", "LESS THAN"); - MYSQL_YYABORT; - } - if (part_info->part_type == LIST_PARTITION) - { - my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), - "LIST", "IN"); - MYSQL_YYABORT; - } - } - else + if (part_info->part_type == NOT_A_PARTITION) part_info->part_type= HASH_PARTITION; + else if (part_info->part_type == RANGE_PARTITION) + { + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), + "RANGE", "LESS THAN"); + MYSQL_YYABORT; + } + else if (part_info->part_type == LIST_PARTITION) + { + my_error(ER_PARTITION_REQUIRES_VALUES_ERROR, MYF(0), + "LIST", "IN"); + MYSQL_YYABORT; + } } | VALUES LESS_SYM THAN_SYM { LEX *lex= Lex; partition_info *part_info= lex->part_info; - if (! lex->is_partition_management()) - { - if (part_info->part_type != RANGE_PARTITION) - { - my_error(ER_PARTITION_WRONG_VALUES_ERROR, MYF(0), - "RANGE", "LESS THAN"); - MYSQL_YYABORT; - } - } - else + if (part_info->part_type == NOT_A_PARTITION) part_info->part_type= RANGE_PARTITION; + else if (part_info->part_type != RANGE_PARTITION) + { + my_error(ER_PARTITION_WRONG_VALUES_ERROR, MYF(0), + "RANGE", "LESS THAN"); + MYSQL_YYABORT; + } } part_func_max {} | VALUES IN_SYM { LEX *lex= Lex; partition_info *part_info= lex->part_info; - if (! lex->is_partition_management()) - { - if (part_info->part_type != LIST_PARTITION) - { - my_error(ER_PARTITION_WRONG_VALUES_ERROR, MYF(0), - "LIST", "IN"); - MYSQL_YYABORT; - } - } - else + if (part_info->part_type == NOT_A_PARTITION) part_info->part_type= LIST_PARTITION; + else if (part_info->part_type != LIST_PARTITION) + { + my_error(ER_PARTITION_WRONG_VALUES_ERROR, MYF(0), + "LIST", "IN"); + MYSQL_YYABORT; + } } part_values_in {} ; From bac287c315b1792e7ae33f91add6a60292f9bae8 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Fri, 17 Aug 2018 14:20:25 +0200 Subject: [PATCH 014/127] Bug#27788907 SOME FILE OPERATIONS IN MF_IOCACHE2.C ARE NOT INSTRUMENTED MySQL bug number 90264 Contribution by Yura Sorokin. Problem: File mysys/mf_iocache2.c contains non instrumented file io operations. This causes inaccurate statistics in PERFORMANCE_SCHEMA. Solution: Use the instrumentation apis (mysql_file_tell instead of my_tell, etc). --- mysql-test/suite/perfschema/r/relaylog.result | 12 ++++++------ mysys/mf_iocache2.c | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mysql-test/suite/perfschema/r/relaylog.result b/mysql-test/suite/perfschema/r/relaylog.result index 0a7d0a5b2be..88cd2e35068 100644 --- a/mysql-test/suite/perfschema/r/relaylog.result +++ b/mysql-test/suite/perfschema/r/relaylog.result @@ -20,7 +20,7 @@ from performance_schema.file_summary_by_instance where file_name like "%master-%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE master-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -master-bin.index wait/io/file/sql/binlog_index NONE MANY NONE MANY +master-bin.index wait/io/file/sql/binlog_index MANY MANY MANY MANY select * from performance_schema.file_summary_by_instance where file_name like "%slave-%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE @@ -36,7 +36,7 @@ from performance_schema.file_summary_by_instance where event_name like "%binlog%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE master-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -master-bin.index wait/io/file/sql/binlog_index NONE MANY NONE MANY +master-bin.index wait/io/file/sql/binlog_index MANY MANY MANY MANY select EVENT_NAME, if (count_read > 0, "MANY", "NONE") as COUNT_READ, @@ -47,7 +47,7 @@ from performance_schema.file_summary_by_event_name where event_name like "%binlog%" order by event_name; EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE wait/io/file/sql/binlog MANY MANY MANY MANY -wait/io/file/sql/binlog_index NONE MANY NONE MANY +wait/io/file/sql/binlog_index MANY MANY MANY MANY select EVENT_NAME, if (count_star > 0, "MANY", "NONE") as COUNT_STAR @@ -93,7 +93,7 @@ where file_name like "%slave-%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE slave-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -slave-bin.index wait/io/file/sql/binlog_index NONE MANY NONE MANY +slave-bin.index wait/io/file/sql/binlog_index MANY MANY MANY MANY slave-relay-bin.000001 wait/io/file/sql/relaylog MANY MANY MANY MANY slave-relay-bin.000002 wait/io/file/sql/relaylog MANY MANY MANY MANY slave-relay-bin.index wait/io/file/sql/relaylog_index MANY MANY MANY MANY @@ -109,7 +109,7 @@ from performance_schema.file_summary_by_instance where event_name like "%binlog%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE slave-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -slave-bin.index wait/io/file/sql/binlog_index NONE MANY NONE MANY +slave-bin.index wait/io/file/sql/binlog_index MANY MANY MANY MANY select EVENT_NAME, if (count_read > 0, "MANY", "NONE") as COUNT_READ, @@ -120,7 +120,7 @@ from performance_schema.file_summary_by_event_name where event_name like "%binlog%" order by event_name; EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE wait/io/file/sql/binlog MANY MANY MANY MANY -wait/io/file/sql/binlog_index NONE MANY NONE MANY +wait/io/file/sql/binlog_index MANY MANY MANY MANY select EVENT_NAME, if (count_star > 0, "MANY", "NONE") as COUNT_STAR diff --git a/mysys/mf_iocache2.c b/mysys/mf_iocache2.c index 1b0375cb88e..8d7cb53041c 100644 --- a/mysys/mf_iocache2.c +++ b/mysys/mf_iocache2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -102,14 +102,14 @@ my_off_t my_b_append_tell(IO_CACHE* info) */ { volatile my_off_t save_pos; - save_pos = my_tell(info->file,MYF(0)); - my_seek(info->file,(my_off_t)0,MY_SEEK_END,MYF(0)); + save_pos = mysql_file_tell(info->file,MYF(0)); + mysql_file_seek(info->file,(my_off_t)0,MY_SEEK_END,MYF(0)); /* Save the value of my_tell in res so we can see it when studying coredump */ DBUG_ASSERT(info->end_of_file - (info->append_read_pos-info->write_buffer) - == (res=my_tell(info->file,MYF(0)))); - my_seek(info->file,save_pos,MY_SEEK_SET,MYF(0)); + == (res=mysql_file_tell(info->file,MYF(0)))); + mysql_file_seek(info->file,save_pos,MY_SEEK_SET,MYF(0)); } #endif res = info->end_of_file + (info->write_pos-info->append_read_pos); @@ -203,7 +203,7 @@ size_t my_b_fill(IO_CACHE *info) if (info->seek_not_done) { /* File touched, do seek */ - if (my_seek(info->file,pos_in_file,MY_SEEK_SET,MYF(0)) == + if (mysql_file_seek(info->file,pos_in_file,MY_SEEK_SET,MYF(0)) == MY_FILEPOS_ERROR) { info->error= 0; @@ -223,7 +223,7 @@ size_t my_b_fill(IO_CACHE *info) } DBUG_EXECUTE_IF ("simulate_my_b_fill_error", {DBUG_SET("+d,simulate_file_read_error");}); - if ((length= my_read(info->file,info->buffer,max_length, + if ((length= mysql_file_read(info->file,info->buffer,max_length, info->myflags)) == (size_t) -1) { info->error= -1; @@ -287,7 +287,7 @@ my_off_t my_b_filelength(IO_CACHE *info) return my_b_tell(info); info->seek_not_done= 1; - return my_seek(info->file, 0L, MY_SEEK_END, MYF(0)); + return mysql_file_seek(info->file, 0L, MY_SEEK_END, MYF(0)); } From 3d65d0db1611f3aea3e1bcde22949351f3b89661 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 4 Sep 2018 23:19:07 +0200 Subject: [PATCH 015/127] MDEV-9137 MariaDB Crash on Query Using Aria Engine Two bugs in Aria, related to 2-level fulltext indexes: * REPAIR calculated the key number incorrectly * CHECK copied the key into last_key too early and checking the second-level btree was overwriting it --- mysql-test/suite/maria/fulltext2.result | 21 ++++++++++++ mysql-test/suite/maria/fulltext2.test | 43 +++++++++++++++++++++++++ storage/maria/ma_check.c | 9 ++---- 3 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/maria/fulltext2.result create mode 100644 mysql-test/suite/maria/fulltext2.test diff --git a/mysql-test/suite/maria/fulltext2.result b/mysql-test/suite/maria/fulltext2.result new file mode 100644 index 00000000000..1af52b3cbf1 --- /dev/null +++ b/mysql-test/suite/maria/fulltext2.result @@ -0,0 +1,21 @@ +CREATE TABLE t1 ( +i int(10) unsigned not null auto_increment primary key, +a varchar(255) not null, +FULLTEXT KEY (a) +) ENGINE=Aria ROW_FORMAT=DYNAMIC MAX_ROWS=2000000000000; +repair table t1 quick; +Table Op Msg_type Msg_text +test.t1 repair status OK +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +repair table t1; +Table Op Msg_type Msg_text +test.t1 repair status OK +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +repair table t1; +Table Op Msg_type Msg_text +test.t1 repair status OK +drop table t1; diff --git a/mysql-test/suite/maria/fulltext2.test b/mysql-test/suite/maria/fulltext2.test new file mode 100644 index 00000000000..bbb5231d3c2 --- /dev/null +++ b/mysql-test/suite/maria/fulltext2.test @@ -0,0 +1,43 @@ +# +# test of new fulltext search features +# + +let collation=utf8_unicode_ci; +source include/have_collation.inc; + +# +# two-level tree +# + +CREATE TABLE t1 ( + i int(10) unsigned not null auto_increment primary key, + a varchar(255) not null, + FULLTEXT KEY (a) +) ENGINE=Aria ROW_FORMAT=DYNAMIC MAX_ROWS=2000000000000; + +# two-level entry, second-level tree with depth 2 +disable_query_log; +let $1=1024; +while ($1) +{ + eval insert t1 (a) values ('aaaxxx'); + dec $1; +} + +# one-level entry (entries) +let $1=150; +while ($1) +{ + eval insert t1 (a) values ('aaayyy'); + dec $1; +} +enable_query_log; + +repair table t1 quick; +check table t1; +repair table t1; +check table t1; +repair table t1; + + +drop table t1; diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index 198df72a1d6..d6379dc4d91 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -891,8 +891,7 @@ static int chk_index(HA_CHECK *param, MARIA_HA *info, MARIA_KEYDEF *keyinfo, if (level > param->max_level) param->max_level=level; - if (_ma_get_keynr(share, anc_page->buff) != - (uint) (keyinfo - share->keyinfo)) + if (_ma_get_keynr(share, anc_page->buff) != keyinfo->key_nr) _ma_check_print_error(param, "Page at %s is not marked for index %u", llstr(anc_page->pos, llbuff), (uint) (keyinfo - share->keyinfo)); @@ -916,7 +915,7 @@ static int chk_index(HA_CHECK *param, MARIA_HA *info, MARIA_KEYDEF *keyinfo, info->last_key.keyinfo= tmp_key.keyinfo= keyinfo; info->lastinx= ~0; /* Safety */ tmp_key.data= tmp_key_buff; - for ( ;; ) + for ( ;; _ma_copy_key(&info->last_key, &tmp_key)) { if (nod_flag) { @@ -998,7 +997,6 @@ static int chk_index(HA_CHECK *param, MARIA_HA *info, MARIA_KEYDEF *keyinfo, tmp_key.data); } } - _ma_copy_key(&info->last_key, &tmp_key); (*key_checksum)+= maria_byte_checksum(tmp_key.data, tmp_key.data_length); record= _ma_row_pos_from_key(&tmp_key); @@ -5728,8 +5726,7 @@ static int sort_insert_key(MARIA_SORT_PARAM *sort_param, a_length= share->keypage_header + nod_flag; key_block->end_pos= anc_buff + share->keypage_header; bzero(anc_buff, share->keypage_header); - _ma_store_keynr(share, anc_buff, (uint) (sort_param->keyinfo - - share->keyinfo)); + _ma_store_keynr(share, anc_buff, sort_param->keyinfo->key_nr); lastkey=0; /* No previous key in block */ } else From 7438667fa96433605078ada7874fc17eac925d9f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 5 Sep 2018 00:59:04 +0200 Subject: [PATCH 016/127] MDEV-9137 MariaDB Crash on Query Using Aria Engine update the code to match semantics of `key` - it's not a (char*) pointer to the buffer as in MyISAM. --- mysql-test/suite/maria/fulltext2.result | 1 + mysql-test/suite/maria/fulltext2.test | 2 ++ storage/maria/ma_write.c | 15 ++++++++++----- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/maria/fulltext2.result b/mysql-test/suite/maria/fulltext2.result index 1af52b3cbf1..f4e2e5dd5c9 100644 --- a/mysql-test/suite/maria/fulltext2.result +++ b/mysql-test/suite/maria/fulltext2.result @@ -18,4 +18,5 @@ test.t1 check status OK repair table t1; Table Op Msg_type Msg_text test.t1 repair status OK +insert t1 (a) values ('aaaxxx'),('aaayyy'); drop table t1; diff --git a/mysql-test/suite/maria/fulltext2.test b/mysql-test/suite/maria/fulltext2.test index bbb5231d3c2..352ca405e99 100644 --- a/mysql-test/suite/maria/fulltext2.test +++ b/mysql-test/suite/maria/fulltext2.test @@ -39,5 +39,7 @@ repair table t1; check table t1; repair table t1; +# mi_write: +insert t1 (a) values ('aaaxxx'),('aaayyy'); drop table t1; diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c index a9022417986..842f3a0aa8d 100644 --- a/storage/maria/ma_write.c +++ b/storage/maria/ma_write.c @@ -665,13 +665,18 @@ static int w_search(register MARIA_HA *info, uint32 comp_flag, MARIA_KEY *key, else { /* popular word. two-level tree. going down */ - my_off_t root=dup_key_pos; - keyinfo= &share->ft2_keyinfo; - get_key_full_length_rdonly(off, key); - key+=off; + my_off_t root= dup_key_pos; + MARIA_KEY subkey; + get_key_full_length_rdonly(off, key->data); + subkey.keyinfo= keyinfo= &share->ft2_keyinfo; + subkey.data= key->data + off; + subkey.data_length= key->data_length - off; + subkey.ref_length= key->ref_length; + subkey.flag= key->flag; + /* we'll modify key entry 'in vivo' */ keypos-= keyinfo->keylength + page.node; - error= _ma_ck_real_write_btree(info, key, &root, comp_flag); + error= _ma_ck_real_write_btree(info, &subkey, &root, comp_flag); _ma_dpointer(share, keypos+HA_FT_WLEN, root); subkeys--; /* should there be underflow protection ? */ DBUG_ASSERT(subkeys < 0); From fb324e3f8f7e81f60f19aa0840550acdcbbc429f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 5 Sep 2018 01:34:25 +0200 Subject: [PATCH 017/127] MDEV-9137 MariaDB Crash on Query Using Aria Engine fix for 2-level ft indexes and boolean search in Aria --- mysql-test/suite/maria/fulltext2.result | 3 +++ mysql-test/suite/maria/fulltext2.test | 2 ++ storage/maria/ma_ft_boolean_search.c | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/maria/fulltext2.result b/mysql-test/suite/maria/fulltext2.result index f4e2e5dd5c9..9fa82451b8f 100644 --- a/mysql-test/suite/maria/fulltext2.result +++ b/mysql-test/suite/maria/fulltext2.result @@ -18,5 +18,8 @@ test.t1 check status OK repair table t1; Table Op Msg_type Msg_text test.t1 repair status OK +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +count(*) +1024 insert t1 (a) values ('aaaxxx'),('aaayyy'); drop table t1; diff --git a/mysql-test/suite/maria/fulltext2.test b/mysql-test/suite/maria/fulltext2.test index 352ca405e99..8236eccd3af 100644 --- a/mysql-test/suite/maria/fulltext2.test +++ b/mysql-test/suite/maria/fulltext2.test @@ -39,6 +39,8 @@ repair table t1; check table t1; repair table t1; +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); + # mi_write: insert t1 (a) values ('aaaxxx'),('aaayyy'); diff --git a/storage/maria/ma_ft_boolean_search.c b/storage/maria/ma_ft_boolean_search.c index 83ae08553ef..d4578ad6bd7 100644 --- a/storage/maria/ma_ft_boolean_search.c +++ b/storage/maria/ma_ft_boolean_search.c @@ -457,7 +457,7 @@ static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search) */ ftbw->off=off; ftbw->key_root= info->cur_row.lastpos; - ftbw->keyinfo=& info->s->ft2_keyinfo; + ftbw->keyinfo= info->last_key.keyinfo= & info->s->ft2_keyinfo; r= _ma_search_first(info, ftbw->keyinfo, ftbw->key_root); DBUG_ASSERT(r==0); /* found something */ memcpy(lastkey_buf+off, info->last_key.data, From a265f0ff247a5ea1ab052a87c53be2d7ccf51300 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 5 Sep 2018 01:40:05 +0200 Subject: [PATCH 018/127] MDEV-9137 MariaDB Crash on Query Using Aria Engine more tests --- mysql-test/suite/maria/fulltext2.result | 61 +++++++++++++++++++++++++ mysql-test/suite/maria/fulltext2.test | 30 ++++++++++++ 2 files changed, 91 insertions(+) diff --git a/mysql-test/suite/maria/fulltext2.result b/mysql-test/suite/maria/fulltext2.result index 9fa82451b8f..1e4e6636ef6 100644 --- a/mysql-test/suite/maria/fulltext2.result +++ b/mysql-test/suite/maria/fulltext2.result @@ -18,8 +18,69 @@ test.t1 check status OK repair table t1; Table Op Msg_type Msg_text test.t1 repair status OK +select count(*) from t1 where match a against ('aaaxxx'); +count(*) +0 +select count(*) from t1 where match a against ('aaayyy'); +count(*) +150 select count(*) from t1 where match a against ('aaaxxx' in boolean mode); count(*) 1024 +select count(*) from t1 where match a against ('aaayyy' in boolean mode); +count(*) +150 +select count(*) from t1 where match a against ('aaax*' in boolean mode); +count(*) +1024 +select count(*) from t1 where match a against ('aaay*' in boolean mode); +count(*) +150 +select count(*) from t1 where match a against ('aaa*' in boolean mode); +count(*) +1174 insert t1 (a) values ('aaaxxx'),('aaayyy'); +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +count(*) +1025 +select count(*) from t1 where match a against ('aaayyy'); +count(*) +151 +insert t1 (a) values ('aaaxxx 000000'); +select count(*) from t1 where match a against ('000000'); +count(*) +1 +delete from t1 where match a against ('000000'); +select count(*) from t1 where match a against ('000000'); +count(*) +0 +select count(*) from t1 where match a against ('aaaxxx'); +count(*) +0 +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +count(*) +1025 +select count(*) from t1 where match a against ('aaayyy' in boolean mode); +count(*) +151 +select count(*) from t1 where a = 'aaaxxx'; +count(*) +1025 +select count(*) from t1 where a = 'aaayyy'; +count(*) +151 +insert t1 (a) values ('aaaxxx 000000'); +select count(*) from t1 where match a against ('000000'); +count(*) +1 +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +count(*) +1026 +update t1 set a='aaaxxx' where a = 'aaayyy'; +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +count(*) +1177 +select count(*) from t1 where match a against ('aaayyy' in boolean mode); +count(*) +0 drop table t1; diff --git a/mysql-test/suite/maria/fulltext2.test b/mysql-test/suite/maria/fulltext2.test index 8236eccd3af..060b748eb4f 100644 --- a/mysql-test/suite/maria/fulltext2.test +++ b/mysql-test/suite/maria/fulltext2.test @@ -39,9 +39,39 @@ repair table t1; check table t1; repair table t1; +select count(*) from t1 where match a against ('aaaxxx'); +select count(*) from t1 where match a against ('aaayyy'); select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +select count(*) from t1 where match a against ('aaayyy' in boolean mode); + +select count(*) from t1 where match a against ('aaax*' in boolean mode); +select count(*) from t1 where match a against ('aaay*' in boolean mode); +select count(*) from t1 where match a against ('aaa*' in boolean mode); # mi_write: insert t1 (a) values ('aaaxxx'),('aaayyy'); +# call to enlarge_root() below +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +select count(*) from t1 where match a against ('aaayyy'); + +# mi_delete +insert t1 (a) values ('aaaxxx 000000'); +select count(*) from t1 where match a against ('000000'); +delete from t1 where match a against ('000000'); +select count(*) from t1 where match a against ('000000'); +select count(*) from t1 where match a against ('aaaxxx'); +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +select count(*) from t1 where match a against ('aaayyy' in boolean mode); +# double-check without index +select count(*) from t1 where a = 'aaaxxx'; +select count(*) from t1 where a = 'aaayyy'; + +# update +insert t1 (a) values ('aaaxxx 000000'); +select count(*) from t1 where match a against ('000000'); +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +update t1 set a='aaaxxx' where a = 'aaayyy'; +select count(*) from t1 where match a against ('aaaxxx' in boolean mode); +select count(*) from t1 where match a against ('aaayyy' in boolean mode); drop table t1; From 27329aac33e6fd6d7a25d787d2e961c7095bc5bf Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 5 Sep 2018 13:09:01 +0200 Subject: [PATCH 019/127] mtr: no warning when an environment variable isn't set e.g. "No option named 'FILE_KEY_MANAGEMENT_SO' in group 'ENV' at lib/My/ConfigFactory.pm line 370." when a test has `plugin-load-add=@ENV.FILE_KEY_MANAGEMENT_SO` --- mysql-test/lib/My/Config.pm | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql-test/lib/My/Config.pm b/mysql-test/lib/My/Config.pm index 1d8d1617fce..d6924a3c866 100644 --- a/mysql-test/lib/My/Config.pm +++ b/mysql-test/lib/My/Config.pm @@ -195,14 +195,10 @@ sub value { my ($self, $option_name)= @_; my $option= $self->option($option_name); - if (! defined($option) and defined $ENV{$option_name}) { + if (! defined($option)) { my $value= $ENV{$option_name}; $option= My::Config::Option->new($option_name, $value); } - - croak "No option named '$option_name' in group '$self->{name}'" - if ! defined($option); - return $option->value(); } From 9905949b414740cdb15d5048b3289e4c5bb54f93 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 14 Sep 2018 21:25:37 +0200 Subject: [PATCH 020/127] cmake: fix usage of GET_TARGET_PROPERTY and followup fixes --- cmake/libutils.cmake | 12 ++++++++---- sql/CMakeLists.txt | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake index 41b7d9d1d86..67e222eefb0 100644 --- a/cmake/libutils.cmake +++ b/cmake/libutils.cmake @@ -136,6 +136,10 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) SET(SOURCE_FILE ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_depends.c) ADD_LIBRARY(${TARGET} STATIC ${SOURCE_FILE}) SET_TARGET_PROPERTIES(${TARGET} PROPERTIES OUTPUT_NAME ${OUTPUT_NAME}) + IF(NOT _SKIP_PIC) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES COMPILE_FLAGS + "${CMAKE_SHARED_LIBRARY_C_FLAGS}") + ENDIF() SET(OSLIBS) FOREACH(LIB ${LIBS_TO_MERGE}) @@ -240,11 +244,11 @@ MACRO(MERGE_LIBRARIES) # check for non-PIC libraries IF(NOT _SKIP_PIC) FOREACH(LIB ${LIBS}) - GET_TARGET_PROPERTY(${LIB} TYPE LIBTYPE) - IF(LIBTYPE STREQUAL "STATIC_LIBRARY") - GET_TARGET_PROPERTY(LIB COMPILE_FLAGS LIB_COMPILE_FLAGS) + GET_TARGET_PROPERTY(LTYPE ${LIB} TYPE) + IF(LTYPE STREQUAL "STATIC_LIBRARY") + GET_TARGET_PROPERTY(LIB_COMPILE_FLAGS ${LIB} COMPILE_FLAGS) STRING(REPLACE "${CMAKE_SHARED_LIBRARY_C_FLAGS}" - "" LIB_COMPILE_FLAGS ${LIB_COMPILE_FLAG}) + "" LIB_COMPILE_FLAGS "${LIB_COMPILE_FLAGS}") IF(NOT LIB_COMPILE_FLAGS MATCHES "") MESSAGE(FATAL_ERROR "Attempted to link non-PIC static library ${LIB} to shared library ${TARGET}\n" diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 9ce1794d1f1..36e08010793 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -181,7 +181,7 @@ DTRACE_INSTRUMENT_STATIC_LIBS(mysqld SET(WITH_MYSQLD_LDFLAGS "" CACHE STRING "Additional linker flags for mysqld") MARK_AS_ADVANCED(WITH_MYSQLD_LDFLAGS) IF(WITH_MYSQLD_LDFLAGS) - GET_TARGET_PROPERTY(mysqld LINK_FLAGS MYSQLD_LINK_FLAGS) + GET_TARGET_PROPERTY(MYSQLD_LINK_FLAGS mysqld LINK_FLAGS) IF(NOT MYSQLD_LINK_FLAGS) SET(MYSQLD_LINK_FLAGS) ENDIF() From a660a5ed42ace61284774b013d45dd4101f831bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Oct 2018 14:33:48 +0300 Subject: [PATCH 021/127] Correct a typo in a comment --- sql/sql_alter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_alter.h b/sql/sql_alter.h index 7114694124b..a4505f1d6c1 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -163,7 +163,7 @@ public: // Columns and keys to be dropped. List drop_list; - // Columns for ALTER_COLUMN_CHANGE_DEFAULT. + // Columns for ALTER_CHANGE_COLUMN_DEFAULT. List alter_list; // List of keys, used by both CREATE and ALTER TABLE. List key_list; From 0b7339eb451d5ffb141ac0a89b3596eec6dc948f Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sun, 7 Oct 2018 10:19:19 -0700 Subject: [PATCH 022/127] MDEV-17382 Hash join algorithm should not be used to join materialized derived table / view by equality Now rows of a materialized derived table are always put into a temporary table before join operation. If BNLH is used to join this table with the result of a partial join then both operands of the join are actually put into main memory. In most cases this is not efficient. We could avoid this by sending the rows of the derived table directly to the join operation. However this kind of data flow is not supported yet. Fixed by not allowing usage of hash join algorithm to join a materialized derived table if it's joined by an equality predicate of the form f=e where f is a field of the derived table. --- mysql-test/r/derived_opt.result | 35 ++++++++++++++++++++++++++- mysql-test/r/innodb_mrr_cpk.result | 2 +- mysql-test/t/derived_opt.test | 38 ++++++++++++++++++++++++++++++ sql/sql_select.cc | 8 +++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_opt.result b/mysql-test/r/derived_opt.result index dc9b5910af1..bbca7f2ac90 100644 --- a/mysql-test/r/derived_opt.result +++ b/mysql-test/r/derived_opt.result @@ -499,9 +499,42 @@ where D1.a= t1.a; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 10 Using where -1 PRIMARY hash_ALL key0 #hash#key0 5 test.t1.a 100 Using join buffer (flat, BNLH join) +1 PRIMARY ref key0 key0 5 test.t1.a 10 2 DERIVED t2 ALL NULL NULL NULL NULL 100 Using filesort set join_cache_level=@tmp_jcl; set optimizer_switch=@tmp_os; drop table t1, t2; +# +# Bug mdev-17382: equi-join of derived table with join_cache_level=4 +# +CREATE TABLE t1 ( +id int NOT NULL, +amount decimal DEFAULT NULL, +PRIMARY KEY (id) +); +CREATE TABLE t2 ( +id int NOT NULL, +name varchar(50) DEFAULT NULL, +PRIMARY KEY (id) +); +INSERT INTO t1 VALUES +(1, 10.0000), (2, 20.0000), (3, 30.0000), (4, 40.0000), +(5, NULL), (6, NULL), (7, 70.0000), (8, 80.0000); +INSERT INTO t2 VALUES +(1,'A'), (2,'B'), (3,'C'), (4,'D'), (5, NULL), (6, NULL), +(7,'E'), (8,'F'), (9,'G'), (10,'H'), (11, NULL), (12, NULL); +set join_cache_level=4; +EXPLAIN +SELECT t2.id,t2.name,t.total_amt +FROM t2 +LEFT JOIN +(SELECT id, sum(amount) total_amt FROM t1 GROUP BY id) AS t +ON t2.id=t.id +WHERE t2.id < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 range PRIMARY PRIMARY 4 NULL 3 Using index condition +1 PRIMARY ref key0 key0 5 test.t2.id 2 +2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using temporary; Using filesort +set join_cache_level=default; +DROP TABLE t1,t2; set optimizer_switch=@exit_optimizer_switch; diff --git a/mysql-test/r/innodb_mrr_cpk.result b/mysql-test/r/innodb_mrr_cpk.result index 99ed73a3e83..355eae70edb 100644 --- a/mysql-test/r/innodb_mrr_cpk.result +++ b/mysql-test/r/innodb_mrr_cpk.result @@ -226,7 +226,7 @@ set join_cache_level=3; explain SELECT 1 FROM (SELECT url, id FROM t2 LIMIT 1 OFFSET 20) derived RIGHT JOIN t1 ON t1.id = derived.id; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL # -1 PRIMARY hash_ALL key0 #hash#key0 25 test.t1.id # Using join buffer (flat, BNLH join) +1 PRIMARY ref key0 key0 25 test.t1.id # 2 DERIVED t2 ALL NULL NULL NULL NULL # set join_cache_level= @tmp_mdev5037; drop table t0,t1,t2; diff --git a/mysql-test/t/derived_opt.test b/mysql-test/t/derived_opt.test index 7f19553e4e5..aab95f69f26 100644 --- a/mysql-test/t/derived_opt.test +++ b/mysql-test/t/derived_opt.test @@ -363,5 +363,43 @@ set join_cache_level=@tmp_jcl; set optimizer_switch=@tmp_os; drop table t1, t2; +--echo # +--echo # Bug mdev-17382: equi-join of derived table with join_cache_level=4 +--echo # + +CREATE TABLE t1 ( + id int NOT NULL, + amount decimal DEFAULT NULL, +PRIMARY KEY (id) +); + +CREATE TABLE t2 ( + id int NOT NULL, + name varchar(50) DEFAULT NULL, +PRIMARY KEY (id) +); + +INSERT INTO t1 VALUES +(1, 10.0000), (2, 20.0000), (3, 30.0000), (4, 40.0000), +(5, NULL), (6, NULL), (7, 70.0000), (8, 80.0000); + +INSERT INTO t2 VALUES +(1,'A'), (2,'B'), (3,'C'), (4,'D'), (5, NULL), (6, NULL), +(7,'E'), (8,'F'), (9,'G'), (10,'H'), (11, NULL), (12, NULL); + +set join_cache_level=4; + +EXPLAIN +SELECT t2.id,t2.name,t.total_amt + FROM t2 + LEFT JOIN + (SELECT id, sum(amount) total_amt FROM t1 GROUP BY id) AS t + ON t2.id=t.id + WHERE t2.id < 3; + +set join_cache_level=default; + +DROP TABLE t1,t2; + # The following command must be the last one the file set optimizer_switch=@exit_optimizer_switch; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b2b2bcde80c..35937f0536f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -10054,7 +10054,15 @@ uint check_join_cache_usage(JOIN_TAB *tab, effort now. */ if (tab->table->pos_in_table_list->is_materialized_derived()) + { no_bka_cache= true; + /* + Don't use hash join algorithm if the temporary table for the rows + of the derived table will be created with an equi-join key. + */ + if (tab->table->s->keys) + no_hashed_cache= true; + } /* Don't use join buffering if we're dictated not to by no_jbuf_after From bd21904357d95631fbbb15defe4b023dce6a24a2 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sun, 7 Oct 2018 10:19:19 -0700 Subject: [PATCH 023/127] MDEV-17382 Hash join algorithm should not be used to join materialized derived table / view by equality Now rows of a materialized derived table are always put into a temporary table before join operation. If BNLH is used to join this table with the result of a partial join then both operands of the join are actually put into main memory. In most cases this is not efficient. We could avoid this by sending the rows of the derived table directly to the join operation. However this kind of data flow is not supported yet. Fixed by not allowing usage of hash join algorithm to join a materialized derived table if it's joined by an equality predicate of the form f=e where f is a field of the derived table. --- mysql-test/r/derived_opt.result | 35 ++++++++++++++++++++++++++- mysql-test/r/innodb_mrr_cpk.result | 2 +- mysql-test/t/derived_opt.test | 38 ++++++++++++++++++++++++++++++ sql/sql_select.cc | 8 +++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_opt.result b/mysql-test/r/derived_opt.result index 04a76c2cbc8..63d2c432768 100644 --- a/mysql-test/r/derived_opt.result +++ b/mysql-test/r/derived_opt.result @@ -499,9 +499,42 @@ where D1.a= t1.a; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 10 Using where -1 PRIMARY hash_ALL key0 #hash#key0 5 test.t1.a 100 Using join buffer (flat, BNLH join) +1 PRIMARY ref key0 key0 5 test.t1.a 10 2 DERIVED t2 ALL NULL NULL NULL NULL 100 Using filesort set join_cache_level=@tmp_jcl; set optimizer_switch=@tmp_os; drop table t1, t2; +# +# Bug mdev-17382: equi-join of derived table with join_cache_level=4 +# +CREATE TABLE t1 ( +id int NOT NULL, +amount decimal DEFAULT NULL, +PRIMARY KEY (id) +); +CREATE TABLE t2 ( +id int NOT NULL, +name varchar(50) DEFAULT NULL, +PRIMARY KEY (id) +); +INSERT INTO t1 VALUES +(1, 10.0000), (2, 20.0000), (3, 30.0000), (4, 40.0000), +(5, NULL), (6, NULL), (7, 70.0000), (8, 80.0000); +INSERT INTO t2 VALUES +(1,'A'), (2,'B'), (3,'C'), (4,'D'), (5, NULL), (6, NULL), +(7,'E'), (8,'F'), (9,'G'), (10,'H'), (11, NULL), (12, NULL); +set join_cache_level=4; +EXPLAIN +SELECT t2.id,t2.name,t.total_amt +FROM t2 +LEFT JOIN +(SELECT id, sum(amount) total_amt FROM t1 GROUP BY id) AS t +ON t2.id=t.id +WHERE t2.id < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 range PRIMARY PRIMARY 4 NULL 3 Using index condition +1 PRIMARY ref key0 key0 5 test.t2.id 2 +2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using temporary; Using filesort +set join_cache_level=default; +DROP TABLE t1,t2; set optimizer_switch=@exit_optimizer_switch; diff --git a/mysql-test/r/innodb_mrr_cpk.result b/mysql-test/r/innodb_mrr_cpk.result index 99ed73a3e83..355eae70edb 100644 --- a/mysql-test/r/innodb_mrr_cpk.result +++ b/mysql-test/r/innodb_mrr_cpk.result @@ -226,7 +226,7 @@ set join_cache_level=3; explain SELECT 1 FROM (SELECT url, id FROM t2 LIMIT 1 OFFSET 20) derived RIGHT JOIN t1 ON t1.id = derived.id; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL # -1 PRIMARY hash_ALL key0 #hash#key0 25 test.t1.id # Using join buffer (flat, BNLH join) +1 PRIMARY ref key0 key0 25 test.t1.id # 2 DERIVED t2 ALL NULL NULL NULL NULL # set join_cache_level= @tmp_mdev5037; drop table t0,t1,t2; diff --git a/mysql-test/t/derived_opt.test b/mysql-test/t/derived_opt.test index 7f19553e4e5..aab95f69f26 100644 --- a/mysql-test/t/derived_opt.test +++ b/mysql-test/t/derived_opt.test @@ -363,5 +363,43 @@ set join_cache_level=@tmp_jcl; set optimizer_switch=@tmp_os; drop table t1, t2; +--echo # +--echo # Bug mdev-17382: equi-join of derived table with join_cache_level=4 +--echo # + +CREATE TABLE t1 ( + id int NOT NULL, + amount decimal DEFAULT NULL, +PRIMARY KEY (id) +); + +CREATE TABLE t2 ( + id int NOT NULL, + name varchar(50) DEFAULT NULL, +PRIMARY KEY (id) +); + +INSERT INTO t1 VALUES +(1, 10.0000), (2, 20.0000), (3, 30.0000), (4, 40.0000), +(5, NULL), (6, NULL), (7, 70.0000), (8, 80.0000); + +INSERT INTO t2 VALUES +(1,'A'), (2,'B'), (3,'C'), (4,'D'), (5, NULL), (6, NULL), +(7,'E'), (8,'F'), (9,'G'), (10,'H'), (11, NULL), (12, NULL); + +set join_cache_level=4; + +EXPLAIN +SELECT t2.id,t2.name,t.total_amt + FROM t2 + LEFT JOIN + (SELECT id, sum(amount) total_amt FROM t1 GROUP BY id) AS t + ON t2.id=t.id + WHERE t2.id < 3; + +set join_cache_level=default; + +DROP TABLE t1,t2; + # The following command must be the last one the file set optimizer_switch=@exit_optimizer_switch; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b49a95e61a8..ca9a6a46fda 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -10785,7 +10785,15 @@ uint check_join_cache_usage(JOIN_TAB *tab, effort now. */ if (tab->table->pos_in_table_list->is_materialized_derived()) + { no_bka_cache= true; + /* + Don't use hash join algorithm if the temporary table for the rows + of the derived table will be created with an equi-join key. + */ + if (tab->table->s->keys) + no_hashed_cache= true; + } /* Don't use join buffering if we're dictated not to by no_jbuf_after From 00ddc8bc7c003122bc424929f2a85872dbd8b813 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 9 Oct 2018 18:06:22 +0100 Subject: [PATCH 024/127] MDEV-17413 Crash in my_malloc_size_cb_func() during shutdown with forceful connection close. Fix is to ensure that when close_connection() is called from shutdown thread, current_thd is set. This that allocation callback for THD specific memory won't assert(in debug version), or crash (in 10.1 and later) close_connection() allocates THD specific memory e.g when it writes the final error packet, and compression is ON for the connection. --- sql/mysqld.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9643f0699d1..6ac54d59ee4 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1699,7 +1699,14 @@ static void close_connections(void) tmp->thread_id, (tmp->main_security_ctx.user ? tmp->main_security_ctx.user : "")); + /* + close_connection() might need a valid current_thd + for memory allocation tracking. + */ + THD* save_thd= current_thd; + set_current_thd(tmp); close_connection(tmp,ER_SERVER_SHUTDOWN); + set_current_thd(save_thd); } #endif DBUG_PRINT("quit",("Unlocking LOCK_thread_count")); From acf8fc1ff8a7b2d49e25279670b04b8eb096ce0c Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Oct 2018 07:08:15 +0100 Subject: [PATCH 025/127] Fix cmake warning CMP0075 Include file check macros honor CMAKE_REQUIRED_LIBRARIES --- cmake/os/Windows.cmake | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 283f79b3b41..ea5084ebba6 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -132,9 +132,8 @@ IF(MSVC) ENDIF() # Always link with socket library -LINK_LIBRARIES(ws2_32) -# ..also for tests -SET(CMAKE_REQUIRED_LIBRARIES ws2_32) +STRING(APPEND CMAKE_C_STANDARD_LIBRARIES " ws2_32.lib") +STRING(APPEND CMAKE_CXX_STANDARD_LIBRARIES " ws2_32.lib") # System checks SET(SIGNAL_WITH_VIO_CLOSE 1) # Something that runtime team needs From 5936d43afb6ad5a75d9eed17eb39e8c00a08a684 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Feb 2018 14:24:36 +1100 Subject: [PATCH 026/127] threadpool_size can contribute to the wanted_files --- sql/mysqld.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 6ac54d59ee4..32253e60791 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4088,6 +4088,10 @@ static int init_common_variables() /* MyISAM requires two file handles per table. */ wanted_files= (10 + max_connections + extra_max_connections + tc_size * 2); +#if defined(HAVE_POOL_OF_THREADS) && !defined(__WIN__) + // add epoll or kevent fd for each threadpool group, in case pool of threads is used + wanted_file+= (thread_handling > SCHEDULER_NO_THREADS) ? 0 : threadpool_size; +#endif /* We are trying to allocate no less than max_connections*5 file handles (i.e. we are trying to set the limit so that they will From b7918a6d38a2b9286e4ffc65344eaf530e86ec4b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 11 Oct 2018 20:36:59 +0200 Subject: [PATCH 027/127] fix test suite after MDEV-15438 --- mysql-test/suite/sys_vars/r/thread_pool_size_high.result | 1 + mysql-test/suite/sys_vars/t/thread_pool_size_high.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/sys_vars/r/thread_pool_size_high.result b/mysql-test/suite/sys_vars/r/thread_pool_size_high.result index f581ae8e315..bc30104b79d 100644 --- a/mysql-test/suite/sys_vars/r/thread_pool_size_high.result +++ b/mysql-test/suite/sys_vars/r/thread_pool_size_high.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("Could not increase number of max_open_files to more than"); SELECT @@global.thread_pool_size; @@global.thread_pool_size 200 diff --git a/mysql-test/suite/sys_vars/t/thread_pool_size_high.test b/mysql-test/suite/sys_vars/t/thread_pool_size_high.test index 761aeee2b0a..d1e68f026f0 100644 --- a/mysql-test/suite/sys_vars/t/thread_pool_size_high.test +++ b/mysql-test/suite/sys_vars/t/thread_pool_size_high.test @@ -1,6 +1,7 @@ --source include/not_windows.inc --source include/not_embedded.inc --source include/have_pool_of_threads.inc +call mtr.add_suppression("Could not increase number of max_open_files to more than"); SELECT @@global.thread_pool_size; From 0f178e7cae4fed9a01d73a398c012fd0e62160f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 12 Oct 2018 13:51:37 +0300 Subject: [PATCH 028/127] Fix typo in 5936d43afb6ad5a75d9eed17eb39e8c00a08a684 --- sql/mysqld.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 32253e60791..21bfc3cead2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4090,7 +4090,7 @@ static int init_common_variables() tc_size * 2); #if defined(HAVE_POOL_OF_THREADS) && !defined(__WIN__) // add epoll or kevent fd for each threadpool group, in case pool of threads is used - wanted_file+= (thread_handling > SCHEDULER_NO_THREADS) ? 0 : threadpool_size; + wanted_files+= (thread_handling > SCHEDULER_NO_THREADS) ? 0 : threadpool_size; #endif /* We are trying to allocate no less than max_connections*5 file From 4d07e6c12da267758bed9e578922a9fc6260a543 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sat, 13 Oct 2018 18:47:16 +0300 Subject: [PATCH 029/127] Disabled storage engine tests using LOCK with MERGE engine Tests fail due to MDEV-17145 --- storage/myisammrg/mysql-test/storage_engine/disabled.def | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 storage/myisammrg/mysql-test/storage_engine/disabled.def diff --git a/storage/myisammrg/mysql-test/storage_engine/disabled.def b/storage/myisammrg/mysql-test/storage_engine/disabled.def new file mode 100644 index 00000000000..ca25a5d331b --- /dev/null +++ b/storage/myisammrg/mysql-test/storage_engine/disabled.def @@ -0,0 +1,2 @@ +lock : MDEV-17145 (Unexpected ER_LOCK_WAIT_TIMEOUT) +select_high_prio : MDEV-17145 (Unexpected ER_LOCK_WAIT_TIMEOUT) From cb4877a50371e02993bd766ac8d92f9c6f5ca8a8 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 15 Oct 2018 10:48:55 +0400 Subject: [PATCH 030/127] MDEV-14322 main.type_datetime failed in buildbot, results mismatch Changing the pause from 5 to 10 seconds. --- mysql-test/t/type_datetime.test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/t/type_datetime.test b/mysql-test/t/type_datetime.test index 3f96673c43f..cf6835d436e 100644 --- a/mysql-test/t/type_datetime.test +++ b/mysql-test/t/type_datetime.test @@ -175,12 +175,12 @@ set @@sql_mode= @org_mode; ## ( Bug#29290 type_datetime.test failure in 5.1 ) ## Therefore we sleep a bit if we are too close to midnight. ## The complete test itself needs around 1 second. -## Therefore a time_distance to midnight of 5 seconds should be sufficient. -if (`SELECT CURTIME() > SEC_TO_TIME(24 * 3600 - 5)`) +## Therefore a time_distance to midnight of 10 seconds should be sufficient. +if (`SELECT CURTIME() > SEC_TO_TIME(24 * 3600 - 10)`) { - # We are here when CURTIME() is between '23:59:56' and '23:59:59'. - # So a sleep time of 5 seconds brings us between '00:00:01' and '00:00:04'. - --real_sleep 5 + # We are here when CURTIME() is between '23:59:51' and '23:59:59'. + # So a sleep time of 10 seconds brings us between '00:00:01' and '00:00:09'. + --real_sleep 10 } create table t1 (f1 date, f2 datetime, f3 timestamp); insert into t1(f1) values(curdate()); From ae3fe14c17b4982fd576f65dccc5aba2e630cb31 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 15 Oct 2018 10:57:36 +0400 Subject: [PATCH 031/127] Test for MDEV-13119 and MDEV-13120 These bugs: MDEV-13119 Wrong results with CAST(AS CHAR) and subquery MDEV-13120 Wrong results with MAKE_SET() and subquery were previously fixed by: MDEV-13790 UNHEX() of a somewhat complicated CONCAT() returns NULL Adding tests only --- mysql-test/r/func_concat.result | 20 ++++++++++++++++++++ mysql-test/t/func_concat.test | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/mysql-test/r/func_concat.result b/mysql-test/r/func_concat.result index 9ab6f74653e..acde1be051b 100644 --- a/mysql-test/r/func_concat.result +++ b/mysql-test/r/func_concat.result @@ -268,3 +268,23 @@ SET optimizer_switch=@save_optimizer_switch; SELECT UNHEX(CONCAT('414C2', HEX(8 + ROUND(RAND()*7)), SUBSTR(SHA(UUID()),6,33),HEX(2+ROUND(RAND()*8)))) IS NULL AS c1; c1 0 +# +# MDEV-13119 Wrong results with CAST(AS CHAR) and subquery +# +SET optimizer_switch=_utf8'derived_merge=on'; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT CAST(t AS CHAR CHARACTER SET utf8) t2 FROM t1) sub; +c2 +abcdefghi-abcdefghi +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; +# +# MDEV-13120 Wrong results with MAKE_SET() and subquery +# +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT MAKE_SET(3,t,t) t2 FROM t1) sub; +c2 +abcdefghi,abcdefghi-abcdefghi,abcdefghi +DROP TABLE t1; diff --git a/mysql-test/t/func_concat.test b/mysql-test/t/func_concat.test index 69dd2c4063e..e1bda4be29e 100644 --- a/mysql-test/t/func_concat.test +++ b/mysql-test/t/func_concat.test @@ -242,3 +242,25 @@ SET optimizer_switch=@save_optimizer_switch; --echo # SELECT UNHEX(CONCAT('414C2', HEX(8 + ROUND(RAND()*7)), SUBSTR(SHA(UUID()),6,33),HEX(2+ROUND(RAND()*8)))) IS NULL AS c1; + + +--echo # +--echo # MDEV-13119 Wrong results with CAST(AS CHAR) and subquery +--echo # + +SET optimizer_switch=_utf8'derived_merge=on'; +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT CAST(t AS CHAR CHARACTER SET utf8) t2 FROM t1) sub; +DROP TABLE t1; +SET optimizer_switch=@save_optimizer_switch; + + +--echo # +--echo # MDEV-13120 Wrong results with MAKE_SET() and subquery +--echo # + +CREATE TABLE t1 (t VARCHAR(10) CHARSET latin1); +INSERT INTO t1 VALUES('abcdefghi'); +SELECT CONCAT(t2,'-',t2) c2 FROM (SELECT MAKE_SET(3,t,t) t2 FROM t1) sub; +DROP TABLE t1; From 34f8a4071e2a77e3263f0fbf2adf1c9e3f8464b1 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 15 Oct 2018 13:22:18 +0400 Subject: [PATCH 032/127] MDEV-17064 LIKE function has error behavior on the fields in which the collation is xxx_unicode_xx Synchronizing sources in: - my_wildcmp_uca_impl() handling utf8_unicode_ci - my_wildcmp_unicode_impl() handling utf8_general_ci The latter has already had a fix for a similar MySQL bug in utf8_general_ci: Bug#11754 SET NAMES utf8 followed by SELECT "A\\" LIKE "A\\" returns 0 So fix is now propagated to utf8_unicode_ci. --- mysql-test/r/ctype_uca.result | 21 ++++++ mysql-test/t/ctype_uca.test | 18 ++++++ strings/ctype-uca.c | 116 +++++++++++++++++----------------- strings/ctype-utf8.c | 46 ++++++-------- 4 files changed, 119 insertions(+), 82 deletions(-) diff --git a/mysql-test/r/ctype_uca.result b/mysql-test/r/ctype_uca.result index 02809f3be22..9d561ee0b11 100644 --- a/mysql-test/r/ctype_uca.result +++ b/mysql-test/r/ctype_uca.result @@ -13571,5 +13571,26 @@ Warnings: Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where ((`test`.`t1`.`a` = 'oe') and (`test`.`t1`.`a` = 'oe')) DROP TABLE t1; # +# MDEV-17064 LIKE function has error behavior on the fields in which the collation is xxx_unicode_xx +# +CREATE TABLE t1 (name VARCHAR(20) CHARACTER SET utf8 COLLATE utf8_unicode_ci); +INSERT INTO t1 VALUES ('radio! test'); +SELECT * FROM t1 WHERE name LIKE '%!!%' ESCAPE '!'; +name +radio! test +ALTER TABLE t1 CHANGE COLUMN name name VARCHAR(20) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci'; +SELECT * FROM t1 WHERE name LIKE '%!!%' ESCAPE '!'; +name +radio! test +DROP TABLE t1; +CREATE TABLE t1 (name VARCHAR(20) CHARACTER SET utf8 COLLATE utf8_unicode_ci); +INSERT INTO t1 VALUES ('radio! test'); +SELECT name LIKE '%!!%' ESCAPE '!' AS c1, +name LIKE '%!!%' COLLATE utf8_general_ci ESCAPE '!' AS c2 +FROM t1; +c1 c2 +1 1 +DROP TABLE t1; +# # End of MariaDB-10.0 tests # diff --git a/mysql-test/t/ctype_uca.test b/mysql-test/t/ctype_uca.test index 5e8195e4718..34a6c5637ed 100644 --- a/mysql-test/t/ctype_uca.test +++ b/mysql-test/t/ctype_uca.test @@ -617,6 +617,24 @@ EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='oe' AND a='oe' COLLATE utf8_german2_c EXPLAIN EXTENDED SELECT * FROM t1 WHERE a='oe' COLLATE utf8_german2_ci AND a='oe'; DROP TABLE t1; +--echo # +--echo # MDEV-17064 LIKE function has error behavior on the fields in which the collation is xxx_unicode_xx +--echo # + +CREATE TABLE t1 (name VARCHAR(20) CHARACTER SET utf8 COLLATE utf8_unicode_ci); +INSERT INTO t1 VALUES ('radio! test'); +SELECT * FROM t1 WHERE name LIKE '%!!%' ESCAPE '!'; +ALTER TABLE t1 CHANGE COLUMN name name VARCHAR(20) CHARACTER SET 'utf8' COLLATE 'utf8_general_ci'; +SELECT * FROM t1 WHERE name LIKE '%!!%' ESCAPE '!'; +DROP TABLE t1; + +CREATE TABLE t1 (name VARCHAR(20) CHARACTER SET utf8 COLLATE utf8_unicode_ci); +INSERT INTO t1 VALUES ('radio! test'); +SELECT name LIKE '%!!%' ESCAPE '!' AS c1, + name LIKE '%!!%' COLLATE utf8_general_ci ESCAPE '!' AS c2 +FROM t1; +DROP TABLE t1; + --echo # --echo # End of MariaDB-10.0 tests --echo # diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index 2351ee9d932..4ccf8170c3e 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -21069,11 +21069,11 @@ int my_wildcmp_uca_impl(CHARSET_INFO *cs, const char *wildstr,const char *wildend, int escape, int w_one, int w_many, int recurse_level) { - int result= -1; /* Not found, using wildcards */ + int result= -1; /* Not found, using wildcards */ my_wc_t s_wc, w_wc; int scan; my_charset_conv_mb_wc mb_wc= cs->cset->mb_wc; - + if (my_string_stack_guard && my_string_stack_guard(recurse_level)) return 1; while (wildstr != wildend) @@ -21082,119 +21082,121 @@ int my_wildcmp_uca_impl(CHARSET_INFO *cs, { my_bool escaped= 0; if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <= 0) - return 1; + (const uchar*)wildend)) <= 0) + return 1; - if (w_wc == (my_wc_t)w_many) + if (w_wc == (my_wc_t) w_many) { - result= 1; /* Found an anchor char */ + result= 1; /* Found an anchor char */ break; } wildstr+= scan; - if (w_wc == (my_wc_t)escape) + if (w_wc == (my_wc_t) escape && wildstr < wildend) { if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <= 0) + (const uchar*)wildend)) <= 0) return 1; wildstr+= scan; escaped= 1; } - + if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, - (const uchar*)str_end)) <= 0) + (const uchar*)str_end)) <= 0) return 1; str+= scan; - - if (!escaped && w_wc == (my_wc_t)w_one) + + if (!escaped && w_wc == (my_wc_t) w_one) { - result= 1; /* Found an anchor char */ + result= 1; /* Found an anchor char */ } else { if (my_uca_charcmp(cs,s_wc,w_wc)) - return 1; + return 1; /* No match */ } if (wildstr == wildend) - return (str != str_end); /* Match if both are at end */ + return (str != str_end); /* Match if both are at end */ } - - - if (w_wc == (my_wc_t)w_many) - { /* Found w_many */ - + + if (w_wc == (my_wc_t) w_many) + { /* Found w_many */ /* Remove any '%' and '_' from the wild search string */ for ( ; wildstr != wildend ; ) { if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <= 0) + (const uchar*)wildend)) <= 0) return 1; - - if (w_wc == (my_wc_t)w_many) - { - wildstr+= scan; - continue; - } - - if (w_wc == (my_wc_t)w_one) - { - wildstr+= scan; - if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, - (const uchar*)str_end)) <= 0) + + if (w_wc == (my_wc_t) w_many) + { + wildstr+= scan; + continue; + } + + if (w_wc == (my_wc_t) w_one) + { + wildstr+= scan; + if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, + (const uchar*)str_end)) <= 0) return 1; str+= scan; - continue; - } - break; /* Not a wild character */ + continue; + } + break; /* Not a wild character */ } - + if (wildstr == wildend) - return 0; /* Ok if w_many is last */ - + return 0; /* Ok if w_many is last */ + if (str == str_end) - return -1; - + return -1; + if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <= 0) + (const uchar*)wildend)) <= 0) return 1; - - if (w_wc == (my_wc_t)escape) + wildstr+= scan; + + if (w_wc == (my_wc_t) escape) { - wildstr+= scan; - if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <= 0) - return 1; + if (wildstr < wildend) + { + if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, + (const uchar*)wildend)) <= 0) + return 1; + wildstr+= scan; + } } - + while (1) { /* Skip until the first character from wildstr is found */ while (str != str_end) { if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, - (const uchar*)str_end)) <= 0) + (const uchar*)str_end)) <= 0) return 1; - + if (!my_uca_charcmp(cs,s_wc,w_wc)) break; str+= scan; } if (str == str_end) return -1; - + + str+= scan; result= my_wildcmp_uca_impl(cs, str, str_end, wildstr, wildend, - escape, w_one, w_many, recurse_level+1); - + escape, w_one, w_many, + recurse_level + 1); if (result <= 0) return result; - - str+= scan; - } + } } } return (str != str_end ? 1 : 0); } + int my_wildcmp_uca(CHARSET_INFO *cs, const char *str,const char *str_end, const char *wildstr,const char *wildend, diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c index 7d7e61ce511..3cb832c5414 100644 --- a/strings/ctype-utf8.c +++ b/strings/ctype-utf8.c @@ -4400,9 +4400,7 @@ int my_wildcmp_unicode_impl(CHARSET_INFO *cs, int result= -1; /* Not found, using wildcards */ my_wc_t s_wc, w_wc; int scan; - int (*mb_wc)(CHARSET_INFO *, my_wc_t *, - const uchar *, const uchar *); - mb_wc= cs->cset->mb_wc; + my_charset_conv_mb_wc mb_wc= cs->cset->mb_wc; if (my_string_stack_guard && my_string_stack_guard(recurse_level)) return 1; @@ -4430,12 +4428,12 @@ int my_wildcmp_unicode_impl(CHARSET_INFO *cs, wildstr+= scan; escaped= 1; } - + if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, (const uchar*)str_end)) <= 0) return 1; str+= scan; - + if (!escaped && w_wc == (my_wc_t) w_one) { result= 1; /* Found an anchor char */ @@ -4453,86 +4451,84 @@ int my_wildcmp_unicode_impl(CHARSET_INFO *cs, if (wildstr == wildend) return (str != str_end); /* Match if both are at end */ } - - + if (w_wc == (my_wc_t) w_many) { /* Found w_many */ - /* Remove any '%' and '_' from the wild search string */ for ( ; wildstr != wildend ; ) { if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, (const uchar*)wildend)) <= 0) return 1; - - if (w_wc == (my_wc_t)w_many) + + if (w_wc == (my_wc_t) w_many) { wildstr+= scan; continue; } - - if (w_wc == (my_wc_t)w_one) + + if (w_wc == (my_wc_t) w_one) { wildstr+= scan; if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, - (const uchar*)str_end)) <=0) + (const uchar*)str_end)) <= 0) return 1; str+= scan; continue; } break; /* Not a wild character */ } - + if (wildstr == wildend) return 0; /* Ok if w_many is last */ - + if (str == str_end) return -1; - + if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <=0) + (const uchar*)wildend)) <= 0) return 1; wildstr+= scan; - - if (w_wc == (my_wc_t)escape) + + if (w_wc == (my_wc_t) escape) { if (wildstr < wildend) { if ((scan= mb_wc(cs, &w_wc, (const uchar*)wildstr, - (const uchar*)wildend)) <=0) + (const uchar*)wildend)) <= 0) return 1; wildstr+= scan; } } - + while (1) { /* Skip until the first character from wildstr is found */ while (str != str_end) { if ((scan= mb_wc(cs, &s_wc, (const uchar*)str, - (const uchar*)str_end)) <=0) + (const uchar*)str_end)) <= 0) return 1; if (weights) { my_tosort_unicode(weights, &s_wc, cs->state); my_tosort_unicode(weights, &w_wc, cs->state); } - + if (s_wc == w_wc) break; str+= scan; } if (str == str_end) return -1; - + str+= scan; result= my_wildcmp_unicode_impl(cs, str, str_end, wildstr, wildend, escape, w_one, w_many, weights, recurse_level + 1); if (result <= 0) return result; - } + } } } return (str != str_end ? 1 : 0); From dc3a20b191362227a6f85fadfc3a8b6c76f69ec6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 15 Oct 2018 12:06:00 +0200 Subject: [PATCH 033/127] MDEV-14429 sql_safe_updates in my.cnf not work --- mysql-test/r/mysqld--help.result | 2 ++ sql/sys_vars.cc | 2 +- sql/sys_vars.h | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 825c7e786a3..0d8728a6768 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -790,6 +790,7 @@ The following specify which files/extra groups are read (specified before remain this size --sql-mode=name Syntax: sql-mode=mode[,mode[,mode...]]. See the manual for the complete list of valid sql modes + --sql-safe-updates sql_safe_updates --stack-trace Print a symbolic stack trace on failure (Defaults to on; use --skip-stack-trace to disable.) --stored-program-cache=# @@ -1081,6 +1082,7 @@ slow-launch-time 2 slow-query-log FALSE sort-buffer-size 2097152 sql-mode +sql-safe-updates FALSE stack-trace TRUE stored-program-cache 256 symbolic-links FALSE diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index ffe861ddfbc..080e74ce367 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2719,7 +2719,7 @@ static Sys_var_bit Sys_auto_is_null( static Sys_var_bit Sys_safe_updates( "sql_safe_updates", "sql_safe_updates", - SESSION_VAR(option_bits), NO_CMD_LINE, OPTION_SAFE_UPDATES, + SESSION_VAR(option_bits), CMD_LINE(REQUIRED_ARG), OPTION_SAFE_UPDATES, DEFAULT(FALSE)); static Sys_var_bit Sys_buffer_results( diff --git a/sql/sys_vars.h b/sql/sys_vars.h index dbe27ab107e..c21bf8244f0 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -1443,7 +1443,6 @@ public: bitmask= reverse_semantics ? ~bitmask_arg : bitmask_arg; set(global_var_ptr(), def_val); SYSVAR_ASSERT(def_val < 2); - SYSVAR_ASSERT(getopt.id == -1); // force NO_CMD_LINE SYSVAR_ASSERT(size == sizeof(ulonglong)); } bool session_update(THD *thd, set_var *var) From 3c5f6aa21c54e2f93c96162a23f2e32772cf50bf Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sun, 14 Oct 2018 10:30:23 -0700 Subject: [PATCH 034/127] MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion This assert is hit when we do filesort using the priority queue and try to insert elements in the queue. The compare function used for the priority queue should handle the case for zerolength sortkey. --- mysql-test/r/order_by_zerolength-4285.result | 20 ++++++++++++++++++++ mysql-test/t/order_by_zerolength-4285.test | 13 +++++++++++++ mysys/ptr_cmp.c | 7 +++++++ 3 files changed, 40 insertions(+) diff --git a/mysql-test/r/order_by_zerolength-4285.result b/mysql-test/r/order_by_zerolength-4285.result index f60ce7d90c7..e4c117b26af 100644 --- a/mysql-test/r/order_by_zerolength-4285.result +++ b/mysql-test/r/order_by_zerolength-4285.result @@ -24,3 +24,23 @@ Warning 1292 Truncated incorrect CHAR(0) value: '8' Warning 1292 Truncated incorrect CHAR(0) value: '9' Warning 1292 Truncated incorrect CHAR(0) value: '10' drop table t1; +# +# MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +# +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 4 NULL 2 Using index; Using filesort +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +pk +1 +2 +Warnings: +Warning 1292 Truncated incorrect CHAR(0) value: '1' +Warning 1292 Truncated incorrect CHAR(0) value: '2' +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysql-test/t/order_by_zerolength-4285.test b/mysql-test/t/order_by_zerolength-4285.test index 2fb58edd36d..9533f2cc62e 100644 --- a/mysql-test/t/order_by_zerolength-4285.test +++ b/mysql-test/t/order_by_zerolength-4285.test @@ -6,3 +6,16 @@ insert into t1 values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); select * from t1 order by now(), cast(pk as char(0)); drop table t1; +--echo # +--echo # MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +--echo # + +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysys/ptr_cmp.c b/mysys/ptr_cmp.c index 6e373e98972..c9306dff14f 100644 --- a/mysys/ptr_cmp.c +++ b/mysys/ptr_cmp.c @@ -52,6 +52,11 @@ static int ptr_compare_0(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_1(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_2(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_3(size_t *compare_length, uchar **a, uchar **b); +static int degenerate_compare_func(size_t *compare_length, uchar **a, uchar **b) +{ + DBUG_ASSERT(*compare_length == 0); + return 0; +} #endif /* __sun */ /* Get a pointer to a optimal byte-compare function for a given size */ @@ -64,6 +69,8 @@ qsort2_cmp get_ptr_compare (size_t size __attribute__((unused))) #else qsort2_cmp get_ptr_compare (size_t size) { + if (size == 0) + return (qsort2_cmp) degenerate_compare_func; if (size < 4) return (qsort2_cmp) ptr_compare; switch (size & 3) { From 311126758e5335b19d5c6311ff63580e20f24b88 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 15 Oct 2018 22:11:14 +0100 Subject: [PATCH 035/127] MDEV-17462 Heap corruption with auth_gssapi on Windows. use FreeContextAttributes() on individual members of SecPkgContext_NativeNames, not on the struct itself. --- plugin/auth_gssapi/sspi_server.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index d2c2ae7e4b9..c3ec8161567 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -103,7 +103,12 @@ static int get_client_name_from_context(CtxtHandle *ctxt, *p = 0; } strncpy(name, native_names.sClientName, name_len); - FreeContextBuffer(&native_names); + + if (native_names.sClientName) + FreeContextBuffer(native_names.sClientName); + if (native_names.sServerName) + FreeContextBuffer(native_names.sServerName); + return CR_OK; } From 64b48aebe411655d2524c30f0c509612b27eea71 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 15 Oct 2018 22:57:15 +0100 Subject: [PATCH 036/127] make auth_gssapi_basic work, also in domain environment. --- plugin/auth_gssapi/mysql-test/auth_gssapi/basic.result | 2 ++ plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test | 1 + plugin/auth_gssapi/sspi_server.cc | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.result b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.result index dc5155fac8c..3044f984ffa 100644 --- a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.result +++ b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.result @@ -1,4 +1,6 @@ INSTALL SONAME 'auth_gssapi'; +Warnings: +Note 1105 SSPI: using principal name 'localhost', mech 'Negotiate' CREATE USER 'GSSAPI_SHORTNAME' IDENTIFIED WITH gssapi; SELECT USER(),CURRENT_USER(); USER() CURRENT_USER() diff --git a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test index f47ad8c20e2..e6e8cde68ca 100644 --- a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test +++ b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test @@ -1,3 +1,4 @@ +--replace_regex /name '[a-zA-Z@.]+'/name 'localhost'/ INSTALL SONAME 'auth_gssapi'; # diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index c3ec8161567..a4223e3cf7f 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -289,7 +289,7 @@ int plugin_init() { srv_principal_name= get_default_principal_name(); } - my_printf_error(0, "SSPI: using principal name '%s', mech '%s'", + my_printf_error(ER_UNKNOWN_ERROR, "SSPI: using principal name '%s', mech '%s'", ME_ERROR_LOG | ME_NOTE, srv_principal_name, srv_mech_name); ret = AcquireCredentialsHandle( From ea9c407e0be92bb2ac1160d9b8081867b49d8f9b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 15 Oct 2018 23:07:30 +0100 Subject: [PATCH 037/127] Fix regular expression in replace_regex in auth_gssapi test. --- plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test index e6e8cde68ca..2307aa3934a 100644 --- a/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test +++ b/plugin/auth_gssapi/mysql-test/auth_gssapi/basic.test @@ -1,4 +1,4 @@ ---replace_regex /name '[a-zA-Z@.]+'/name 'localhost'/ +--replace_regex /name '[^']+'/name 'localhost'/ INSTALL SONAME 'auth_gssapi'; # From 1dacd5f299be63b8dea8f6ff802739abba96b4e1 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 16 Oct 2018 13:02:50 +0530 Subject: [PATCH 038/127] MDEV-12547: InnoDB FULLTEXT index has too strict innodb_ft_result_cache_limit max limit - Backported the MYSQL_SYSVAR_SIZE_T to 10.0 - The parameter innodb_ft_result_cache_limit was only 32 bits wide also on 64-bit systems. Make it size_t, so that it will be 64 bits on 64-bit systems. - Added a test case that show how innodb_ft_result_cache_limit variables behaves in 32bit and 64 bit system. --- include/mysql/plugin.h | 12 ++++++++++++ mysql-test/include/have_32bit.inc | 9 +++++++++ mysql-test/include/have_64bit.inc | 9 +++++++++ .../r/innodb_ft_result_cache_limit_32.result | 7 +++++++ .../r/innodb_ft_result_cache_limit_64.result | 5 +++++ .../sys_vars/t/innodb_ft_result_cache_limit_32.test | 9 +++++++++ .../sys_vars/t/innodb_ft_result_cache_limit_64.test | 9 +++++++++ storage/innobase/fts/fts0fts.cc | 4 ++-- storage/innobase/fts/fts0que.cc | 6 +++--- storage/innobase/handler/ha_innodb.cc | 4 ++-- storage/innobase/include/fts0fts.h | 2 +- storage/innobase/include/fts0types.h | 4 ++-- storage/xtradb/fts/fts0fts.cc | 4 ++-- storage/xtradb/fts/fts0que.cc | 6 +++--- storage/xtradb/handler/ha_innodb.cc | 4 ++-- storage/xtradb/include/fts0fts.h | 2 +- storage/xtradb/include/fts0types.h | 4 ++-- 17 files changed, 80 insertions(+), 20 deletions(-) create mode 100644 mysql-test/include/have_32bit.inc create mode 100644 mysql-test/include/have_64bit.inc create mode 100644 mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_32.result create mode 100644 mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_64.result create mode 100644 mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_32.test create mode 100644 mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_64.test diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index ceb6ac93ff5..307138ad4db 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -383,6 +383,18 @@ DECLARE_MYSQL_SYSVAR_SIMPLE(name, unsigned long long) = { \ PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \ #name, comment, check, update, &varname, def, min, max, blk } +#ifdef _WIN64 +#define MYSQL_SYSVAR_SIZE_T(name, varname, opt, comment, check, update, def, min, max, blk) \ +DECLARE_MYSQL_SYSVAR_SIMPLE(name, size_t) = { \ + PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \ + #name, comment, check, update, &varname, def, min, max, blk } +#else +#define MYSQL_SYSVAR_SIZE_T(name, varname, opt, comment, check, update, def, min, max, blk) \ +DECLARE_MYSQL_SYSVAR_SIMPLE(name, size_t) = { \ + PLUGIN_VAR_LONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \ + #name, comment, check, update, &varname, def, min, max, blk } +#endif + #define MYSQL_SYSVAR_ENUM(name, varname, opt, comment, check, update, def, typelib) \ DECLARE_MYSQL_SYSVAR_TYPELIB(name, unsigned long) = { \ PLUGIN_VAR_ENUM | ((opt) & PLUGIN_VAR_MASK), \ diff --git a/mysql-test/include/have_32bit.inc b/mysql-test/include/have_32bit.inc new file mode 100644 index 00000000000..d62093d8be4 --- /dev/null +++ b/mysql-test/include/have_32bit.inc @@ -0,0 +1,9 @@ +disable_query_log; +disable_warnings; +let $VERSION_COMPILE_64BIT= + `SELECT IF(@@version_compile_machine like '%64%', 1, 0)`; +enable_warnings; +enable_query_log; +if ($VERSION_COMPILE_64BIT) { + skip Need a 32 bit machine/binary; +} diff --git a/mysql-test/include/have_64bit.inc b/mysql-test/include/have_64bit.inc new file mode 100644 index 00000000000..38c11156a53 --- /dev/null +++ b/mysql-test/include/have_64bit.inc @@ -0,0 +1,9 @@ +disable_query_log; +disable_warnings; +let $VERSION_COMPILE_64BIT= + `SELECT IF(@@version_compile_machine like '%64%', 1, 0)`; +enable_warnings; +enable_query_log; +if (!$VERSION_COMPILE_64BIT) { + skip Need a 64 bit machine/binary; +} diff --git a/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_32.result b/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_32.result new file mode 100644 index 00000000000..b3bec1eecdd --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_32.result @@ -0,0 +1,7 @@ +set global innodb_ft_result_cache_limit=5000000000; +Warnings: +Warning 1292 Truncated incorrect innodb_ft_result_cache_limit value: '5000000000' +select @@innodb_ft_result_cache_limit; +@@innodb_ft_result_cache_limit +4294967295 +set global innodb_ft_result_cache_limit=2000000000; diff --git a/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_64.result b/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_64.result new file mode 100644 index 00000000000..c86331a8a1c --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_ft_result_cache_limit_64.result @@ -0,0 +1,5 @@ +set global innodb_ft_result_cache_limit=5000000000; +select @@innodb_ft_result_cache_limit; +@@innodb_ft_result_cache_limit +5000000000 +set global innodb_ft_result_cache_limit=2000000000; diff --git a/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_32.test b/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_32.test new file mode 100644 index 00000000000..d9defc7447b --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_32.test @@ -0,0 +1,9 @@ +--source include/have_32bit.inc +--source include/have_innodb.inc + +let $innodb_ft_result_cache_limit_orig=`select @@innodb_ft_result_cache_limit`; + +set global innodb_ft_result_cache_limit=5000000000; +select @@innodb_ft_result_cache_limit; + +eval set global innodb_ft_result_cache_limit=$innodb_ft_result_cache_limit_orig; diff --git a/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_64.test b/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_64.test new file mode 100644 index 00000000000..2606d2b5ca8 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_ft_result_cache_limit_64.test @@ -0,0 +1,9 @@ +--source include/have_64bit.inc +--source include/have_innodb.inc + +let $innodb_ft_result_cache_limit_orig=`select @@innodb_ft_result_cache_limit`; + +set global innodb_ft_result_cache_limit=5000000000; +select @@innodb_ft_result_cache_limit; + +eval set global innodb_ft_result_cache_limit=$innodb_ft_result_cache_limit_orig; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 6044abd8e79..4891e572741 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -67,7 +67,7 @@ UNIV_INTERN ulong fts_max_total_cache_size; /** This is FTS result cache limit for each query and would be a configurable variable */ -UNIV_INTERN ulong fts_result_cache_limit; +UNIV_INTERN size_t fts_result_cache_limit; /** Variable specifying the maximum FTS max token size */ UNIV_INTERN ulong fts_max_token_size; @@ -4308,7 +4308,7 @@ fts_sync_begin( if (fts_enable_diag_print) { ib_logf(IB_LOG_LEVEL_INFO, "FTS SYNC for table %s, deleted count: %ld size: " - "%lu bytes", + "%zu bytes", sync->table->name, ib_vector_size(cache->deleted_doc_ids), cache->total_size); diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 78521df75d9..7983181c23a 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -76,7 +76,7 @@ struct fts_query_t { fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ - ulint total_size; /*!< total memory size used by query */ + size_t total_size; /*!< total memory size used by query */ fts_doc_ids_t* deleted; /*!< Deleted doc ids that need to be filtered from the output */ @@ -4039,7 +4039,7 @@ fts_query( /* Log memory consumption & result size */ ib_logf(IB_LOG_LEVEL_INFO, "Full Search Memory: " - "%lu (bytes), Row: %lu .", + "%zu (bytes), Row: %lu .", query.total_size, (*result)->rankings_by_id ? rbt_size((*result)->rankings_by_id) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 58060cd2edb..2092cd113a5 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -16931,10 +16931,10 @@ static MYSQL_SYSVAR_ULONG(ft_total_cache_size, fts_max_total_cache_size, "Total memory allocated for InnoDB Fulltext Search cache", NULL, NULL, 640000000, 32000000, 1600000000, 0); -static MYSQL_SYSVAR_ULONG(ft_result_cache_limit, fts_result_cache_limit, +static MYSQL_SYSVAR_SIZE_T(ft_result_cache_limit, fts_result_cache_limit, PLUGIN_VAR_RQCMDARG, "InnoDB Fulltext search query result cache limit in bytes", - NULL, NULL, 2000000000L, 1000000L, 4294967295UL, 0); + NULL, NULL, 2000000000L, 1000000L, SIZE_T_MAX, 0); static MYSQL_SYSVAR_ULONG(ft_min_token_size, fts_min_token_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 4c2d247e0a6..ce30a17c4b4 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -355,7 +355,7 @@ extern ulong fts_max_cache_size; extern ulong fts_max_total_cache_size; /** Variable specifying the FTS result cache limit for each query */ -extern ulong fts_result_cache_limit; +extern size_t fts_result_cache_limit; /** Variable specifying the maximum FTS max token size */ extern ulong fts_max_token_size; diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 0dad75d8f1b..9ecd9080881 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -161,7 +161,7 @@ struct fts_cache_t { the document from the table. Each element is of type fts_doc_t */ - ulint total_size; /*!< total size consumed by the ilist + size_t total_size; /*!< total size consumed by the ilist field of all nodes. SYNC is run whenever this gets too big */ fts_sync_t* sync; /*!< sync structure to sync data to @@ -243,7 +243,7 @@ struct fts_fetch_t { fts_sql_callback read_record; /*!< Callback for reading index record */ - ulint total_memory; /*!< Total memory used */ + size_t total_memory; /*!< Total memory used */ }; /** For horizontally splitting an FTS auxiliary index */ diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 2f9e331219b..e2a479bf0ae 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -67,7 +67,7 @@ UNIV_INTERN ulong fts_max_total_cache_size; /** This is FTS result cache limit for each query and would be a configurable variable */ -UNIV_INTERN ulong fts_result_cache_limit; +UNIV_INTERN size_t fts_result_cache_limit; /** Variable specifying the maximum FTS max token size */ UNIV_INTERN ulong fts_max_token_size; @@ -4308,7 +4308,7 @@ fts_sync_begin( if (fts_enable_diag_print) { ib_logf(IB_LOG_LEVEL_INFO, "FTS SYNC for table %s, deleted count: %ld size: " - "%lu bytes", + "%zu bytes", sync->table->name, ib_vector_size(cache->deleted_doc_ids), cache->total_size); diff --git a/storage/xtradb/fts/fts0que.cc b/storage/xtradb/fts/fts0que.cc index 9966656e772..b9ad43c626a 100644 --- a/storage/xtradb/fts/fts0que.cc +++ b/storage/xtradb/fts/fts0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -76,7 +76,7 @@ struct fts_query_t { fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ - ulint total_size; /*!< total memory size used by query */ + size_t total_size; /*!< total memory size used by query */ fts_doc_ids_t* deleted; /*!< Deleted doc ids that need to be filtered from the output */ @@ -4058,7 +4058,7 @@ fts_query( /* Log memory consumption & result size */ ib_logf(IB_LOG_LEVEL_INFO, "Full Search Memory: " - "%lu (bytes), Row: %lu .", + "%zu (bytes), Row: %lu .", query.total_size, (*result)->rankings_by_id ? rbt_size((*result)->rankings_by_id) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index e32d928d45e..6bc4c76f88e 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -18460,10 +18460,10 @@ static MYSQL_SYSVAR_ULONG(ft_total_cache_size, fts_max_total_cache_size, "Total memory allocated for InnoDB Fulltext Search cache", NULL, NULL, 640000000, 32000000, 1600000000, 0); -static MYSQL_SYSVAR_ULONG(ft_result_cache_limit, fts_result_cache_limit, +static MYSQL_SYSVAR_SIZE_T(ft_result_cache_limit, fts_result_cache_limit, PLUGIN_VAR_RQCMDARG, "InnoDB Fulltext search query result cache limit in bytes", - NULL, NULL, 2000000000L, 1000000L, 4294967295UL, 0); + NULL, NULL, 2000000000L, 1000000L, SIZE_T_MAX, 0); static MYSQL_SYSVAR_ULONG(ft_min_token_size, fts_min_token_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 4c2d247e0a6..ce30a17c4b4 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -355,7 +355,7 @@ extern ulong fts_max_cache_size; extern ulong fts_max_total_cache_size; /** Variable specifying the FTS result cache limit for each query */ -extern ulong fts_result_cache_limit; +extern size_t fts_result_cache_limit; /** Variable specifying the maximum FTS max token size */ extern ulong fts_max_token_size; diff --git a/storage/xtradb/include/fts0types.h b/storage/xtradb/include/fts0types.h index 0dad75d8f1b..9ecd9080881 100644 --- a/storage/xtradb/include/fts0types.h +++ b/storage/xtradb/include/fts0types.h @@ -161,7 +161,7 @@ struct fts_cache_t { the document from the table. Each element is of type fts_doc_t */ - ulint total_size; /*!< total size consumed by the ilist + size_t total_size; /*!< total size consumed by the ilist field of all nodes. SYNC is run whenever this gets too big */ fts_sync_t* sync; /*!< sync structure to sync data to @@ -243,7 +243,7 @@ struct fts_fetch_t { fts_sql_callback read_record; /*!< Callback for reading index record */ - ulint total_memory; /*!< Total memory used */ + size_t total_memory; /*!< Total memory used */ }; /** For horizontally splitting an FTS auxiliary index */ From 952f394f8e936bb254062c7e6f4f62afd1c90f8e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 16 Oct 2018 09:17:03 +0100 Subject: [PATCH 039/127] remove MYF flags from plugin --- plugin/auth_gssapi/sspi_server.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index a4223e3cf7f..b3605ddb0d9 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -42,11 +42,11 @@ static void log_error(SECURITY_STATUS err, const char *msg) { char buf[1024]; sspi_errmsg(err, buf, sizeof(buf)); - my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error 0x%x - %s - %s", MYF(0), msg, buf); + my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error 0x%x - %s - %s", 0, msg, buf); } else { - my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error %s", MYF(0), msg); + my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error %s", 0, msg); } } From 5a5bc21a65eb28f22b2896d96bd5fad0557b5719 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 16 Oct 2018 09:19:03 +0100 Subject: [PATCH 040/127] auth_gssapi : Fix string formatting in my_printf_error() --- plugin/auth_gssapi/sspi_server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index b3605ddb0d9..fe90a195d88 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -42,7 +42,7 @@ static void log_error(SECURITY_STATUS err, const char *msg) { char buf[1024]; sspi_errmsg(err, buf, sizeof(buf)); - my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error 0x%x - %s - %s", 0, msg, buf); + my_printf_error(ER_UNKNOWN_ERROR, "SSPI server error 0x%x - %s - %s", 0, err, msg, buf); } else { From bebe24b03b859d3d01b0bd56356047531a26fac4 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Tue, 26 Dec 2017 14:38:45 +0400 Subject: [PATCH 041/127] MDEV-11071 - Assertion `thd->transaction.stmt.is_empty()' failed in Locked_tables_list::unlock_locked_tables Similarly to regular DROP TABLE, don't leave locked tables mode if CREATE OR REPLACE dropped temporary table but failed to cerate new one. The problem is that there's no track of which temporary table was "locked" by LOCK TABLES. --- mysql-test/r/create_or_replace.result | 20 ++++++++++++++++++++ mysql-test/t/create_or_replace.test | 24 ++++++++++++++++++++++++ sql/sql_table.cc | 2 +- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/create_or_replace.result b/mysql-test/r/create_or_replace.result index 646183ebcb1..85602462de5 100644 --- a/mysql-test/r/create_or_replace.result +++ b/mysql-test/r/create_or_replace.result @@ -488,3 +488,23 @@ CREATE OR REPLACE TABLE t1 (i INT); UNLOCK TABLES; INSERT INTO t2 VALUES (1); DROP TABLE t1, t2, t3; +# +# MDEV-11071 - Assertion `thd->transaction.stmt.is_empty()' failed in +# Locked_tables_list::unlock_locked_tables +# +CREATE TEMPORARY TABLE t1(a INT) ENGINE=InnoDB; +CREATE TEMPORARY TABLE t2(a INT); +CREATE TABLE t3(a INT); +LOCK TABLE t2 WRITE; +SELECT * FROM t2; +a +CREATE OR REPLACE TEMPORARY TABLE t1(c INT DEFAULT ''); +ERROR 42000: Invalid default value for 'c' +SELECT * FROM t3; +ERROR HY000: Table 't3' was not locked with LOCK TABLES +CREATE OR REPLACE TEMPORARY TABLE t2(c INT DEFAULT ''); +ERROR 42000: Invalid default value for 'c' +SELECT * FROM t3; +ERROR HY000: Table 't3' was not locked with LOCK TABLES +UNLOCK TABLES; +DROP TABLE t3; diff --git a/mysql-test/t/create_or_replace.test b/mysql-test/t/create_or_replace.test index 8c8ed1c9886..9a1c77f38ee 100644 --- a/mysql-test/t/create_or_replace.test +++ b/mysql-test/t/create_or_replace.test @@ -444,3 +444,27 @@ INSERT INTO t2 VALUES (1); # Cleanup DROP TABLE t1, t2, t3; + +--echo # +--echo # MDEV-11071 - Assertion `thd->transaction.stmt.is_empty()' failed in +--echo # Locked_tables_list::unlock_locked_tables +--echo # +CREATE TEMPORARY TABLE t1(a INT) ENGINE=InnoDB; +CREATE TEMPORARY TABLE t2(a INT); +CREATE TABLE t3(a INT); +LOCK TABLE t2 WRITE; +SELECT * FROM t2; +# drops t2 +--error ER_INVALID_DEFAULT +CREATE OR REPLACE TEMPORARY TABLE t1(c INT DEFAULT ''); +# make sure we didn't leave locked tables mode +--error ER_TABLE_NOT_LOCKED +SELECT * FROM t3; +# drops t1 +--error ER_INVALID_DEFAULT +CREATE OR REPLACE TEMPORARY TABLE t2(c INT DEFAULT ''); +# make sure we didn't leave locked tables mode +--error ER_TABLE_NOT_LOCKED +SELECT * FROM t3; +UNLOCK TABLES; +DROP TABLE t3; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 5d20ad3967c..2302026b18b 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5087,7 +5087,7 @@ err: /* Write log if no error or if we already deleted a table */ if (!result || thd->log_current_statement) { - if (result && create_info->table_was_deleted) + if (result && create_info->table_was_deleted && pos_in_locked_tables) { /* Possible locked table was dropped. We should remove meta data locks From d8974ebd677f97cf80bf0cd1446e60631895a5d7 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 8 Oct 2018 21:47:53 +0300 Subject: [PATCH 042/127] MDEV-14431 binlog.binlog_flush_binlogs_delete_domain failed in buildbot The test and also rpl_gtid_delete_domain failed on PPC64 platform due to an incorrectly specified actual key for searching in a gtid domain system hash. While the correct size is 32 bits the supplied value was 8 bytes of long int size on the platform. The problem became evident thanks to the big endiness which cut off the *least* significant part of the value field. Fixed with correcting a dynamic array initialization to hold now uint32 values as well as the values extraction for searching in the gtid domain system hash. A new added test ensures no overflowed values are accepted for deletion which prevents inadvertent action. Notice though MariaDB [test]> set @@session.gtid_domain_id=(1 << 32) + 1; MariaDB [test]> show warnings; +---------+------+--------------------------------------------------------+ | Level | Code | Message | +---------+------+--------------------------------------------------------+ | Warning | 1292 | Truncated incorrect gtid_domain_id value: '4294967297' | +---------+------+--------------------------------------------------------+ MariaDB [test]> select @@session.gtid_domain_id; +--------------------------+ | @@session.gtid_domain_id | +--------------------------+ | 4294967295 | +--------------------------+ --- .../binlog_flush_binlogs_delete_domain.result | 12 ++++++++++++ .../t/binlog_flush_binlogs_delete_domain.test | 19 +++++++++++++++++++ sql/rpl_gtid.cc | 6 +++--- sql/sql_lex.cc | 2 +- sql/sql_lex.h | 2 +- sql/sql_yacc.yy | 13 +++++++++++-- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result b/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result index 99f2a57835f..fdcfb4bfa01 100644 --- a/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result +++ b/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result @@ -17,6 +17,12 @@ master-bin.000002 # SET @@SESSION.gtid_domain_id=1; SET @@SESSION.server_id=1; CREATE TABLE t (a int); +SELECT @@GLOBAL.gtid_binlog_state, @@GLOBAL.gtid_binlog_pos; +@@GLOBAL.gtid_binlog_state @@GLOBAL.gtid_binlog_pos +1-1-1 1-1-1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Rotate # # master-bin.000002;pos=POS FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); ERROR HY000: Could not delete gtid domain. Reason: binlog files may contain gtids from the domain ('1') being deleted. Make sure to first purge those files. FLUSH BINARY LOGS; @@ -74,5 +80,11 @@ the following command succeeds with warnings FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); Warnings: Warning 1076 The current gtid binlog state is incompatible with a former one having a gtid '11-11-1' which is less than the '11-11-11' of the gtid list describing an earlier state. The state may have been affected by manually injecting a lower sequence number gtid or via replication. +RESET MASTER; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (4294967296); +ERROR HY000: The value of gtid domain being deleted ('4294967296') exceeds its maximum size of 32 bit unsigned integer +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (4294967295); +Warnings: +Warning 1076 The gtid domain being deleted ('4294967295') is not in the current binlog state DROP TABLE t; RESET MASTER; diff --git a/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test b/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test index 0faafa35a1b..8311f4bd800 100644 --- a/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test +++ b/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test @@ -28,6 +28,9 @@ FLUSH BINARY LOGS DELETE_DOMAIN_ID = (99); SET @@SESSION.gtid_domain_id=1; SET @@SESSION.server_id=1; CREATE TABLE t (a int); +SELECT @@GLOBAL.gtid_binlog_state, @@GLOBAL.gtid_binlog_pos; +--let $binlog_start= +--source include/show_binlog_events.inc --error ER_BINLOG_CANT_DELETE_GTID_DOMAIN FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); @@ -129,6 +132,22 @@ SELECT @gtid_binlog_state_saved "as original state", @@GLOBAL.gtid_binlog_state --echo the following command succeeds with warnings --eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($del_d_id) +# cleanup: forget the out-of-order +RESET MASTER; + +# +# MDEV-14431 +# Check rejection to delete a domain with value exceeding its range's maximum +# +--let $d_max_plus_1=`SELECT 1 << 32` +--error ER_BINLOG_CANT_DELETE_GTID_DOMAIN +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($d_max_plus_1) + +# accepted maximum: +--let $d_max=`SELECT (1 << 32) - 1` +--error 0 +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($d_max) + # # Cleanup # diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 955763fe195..2d4acde03fc 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -1846,10 +1846,10 @@ rpl_binlog_state::drop_domain(DYNAMIC_ARRAY *ids, for (ulong i= 0; i < ids->elements; i++) { rpl_binlog_state::element *elem= NULL; - ulong *ptr_domain_id; + uint32 *ptr_domain_id; bool not_match; - ptr_domain_id= (ulong*) dynamic_array_ptr(ids, i); + ptr_domain_id= (uint32*) dynamic_array_ptr(ids, i); elem= (rpl_binlog_state::element *) my_hash_search(&hash, (const uchar *) ptr_domain_id, 0); if (!elem) @@ -1870,7 +1870,7 @@ rpl_binlog_state::drop_domain(DYNAMIC_ARRAY *ids, if (not_match) { - sprintf(errbuf, "binlog files may contain gtids from the domain ('%lu') " + sprintf(errbuf, "binlog files may contain gtids from the domain ('%u') " "being deleted. Make sure to first purge those files", *ptr_domain_id); errmsg= errbuf; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 085ad1a4b3b..1e6abf74a85 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -2882,7 +2882,7 @@ LEX::LEX() INITIAL_LEX_PLUGIN_LIST_SIZE, 0); reset_query_tables_list(TRUE); mi.init(); - init_dynamic_array2(&delete_gtid_domain, sizeof(ulong*), + init_dynamic_array2(&delete_gtid_domain, sizeof(uint32), gtid_domain_static_buffer, initial_gtid_domain_buffer_size, initial_gtid_domain_buffer_size, 0); diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 05e31c28277..f1edc809579 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -2752,7 +2752,7 @@ public: */ DYNAMIC_ARRAY delete_gtid_domain; static const ulong initial_gtid_domain_buffer_size= 16; - ulong gtid_domain_static_buffer[initial_gtid_domain_buffer_size]; + uint32 gtid_domain_static_buffer[initial_gtid_domain_buffer_size]; inline void set_limit_rows_examined() { diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 6ae65e0c50f..af1d67fb2cf 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -12829,9 +12829,18 @@ delete_domain_id_list: ; delete_domain_id: - ulong_num + ulonglong_num { - insert_dynamic(&Lex->delete_gtid_domain, (uchar*) &($1)); + uint32 value= (uint32) $1; + if ($1 > UINT_MAX32) + { + my_printf_error(ER_BINLOG_CANT_DELETE_GTID_DOMAIN, + "The value of gtid domain being deleted ('%llu') " + "exceeds its maximum size " + "of 32 bit unsigned integer", MYF(0), $1); + MYSQL_YYABORT; + } + insert_dynamic(&Lex->delete_gtid_domain, (uchar*) &value); } ; From 853dee854c60d13dfac7f7b52bd376a1bc63a4c3 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 3 Oct 2018 12:22:03 +0300 Subject: [PATCH 043/127] MDEV-17358 my_reverse_bits() is incorrect due to UB my_reverse_bits(): add a cast to fix a bit shift --- include/my_bit.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/my_bit.h b/include/my_bit.h index 174e0f70083..494d899c54f 100644 --- a/include/my_bit.h +++ b/include/my_bit.h @@ -115,10 +115,10 @@ static inline uint32 my_clear_highest_bit(uint32 v) static inline uint32 my_reverse_bits(uint32 key) { return - (_my_bits_reverse_table[ key & 255] << 24) | - (_my_bits_reverse_table[(key>> 8) & 255] << 16) | - (_my_bits_reverse_table[(key>>16) & 255] << 8) | - _my_bits_reverse_table[(key>>24) ]; + ((uint32)_my_bits_reverse_table[ key & 255] << 24) | + ((uint32)_my_bits_reverse_table[(key>> 8) & 255] << 16) | + ((uint32)_my_bits_reverse_table[(key>>16) & 255] << 8) | + (uint32)_my_bits_reverse_table[(key>>24) ]; } C_MODE_END From 2a576f71c5d3c7aacef564e5b1251f83bde48f51 Mon Sep 17 00:00:00 2001 From: Bernd Kuhls Date: Sun, 7 Oct 2018 14:25:59 +0200 Subject: [PATCH 044/127] cmake: fix ucontext detection On some archs uclibc does not provide the ucontext structure despite providing ucontext.h, for details see https://git.buildroot.net/buildroot/commit/?id=f1cbfeea95e6287c7a666aafc182ffa318eff262 This patch improves the detection of ucontext by making sure that HAVE_UCONTEXT_H is only set when makecontext() was found. Signed-off-by: Bernd Kuhls --- configure.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.cmake b/configure.cmake index df180e72963..d37dccbea4a 100644 --- a/configure.cmake +++ b/configure.cmake @@ -1107,11 +1107,11 @@ SET(CMAKE_EXTRA_INCLUDE_FILES) CHECK_STRUCT_HAS_MEMBER("struct dirent" d_ino "dirent.h" STRUCT_DIRENT_HAS_D_INO) CHECK_STRUCT_HAS_MEMBER("struct dirent" d_namlen "dirent.h" STRUCT_DIRENT_HAS_D_NAMLEN) SET(SPRINTF_RETURNS_INT 1) -CHECK_INCLUDE_FILE(ucontext.h HAVE_UCONTEXT_H) -IF(NOT HAVE_UCONTEXT_H) - CHECK_INCLUDE_FILE(sys/ucontext.h HAVE_UCONTEXT_H) +CHECK_INCLUDE_FILE(ucontext.h HAVE_FILE_UCONTEXT_H) +IF(NOT HAVE_FILE_UCONTEXT_H) + CHECK_INCLUDE_FILE(sys/ucontext.h HAVE_FILE_UCONTEXT_H) ENDIF() -IF(HAVE_UCONTEXT_H) +IF(HAVE_FILE_UCONTEXT_H) CHECK_FUNCTION_EXISTS(makecontext HAVE_UCONTEXT_H) ENDIF() From 8e716138cef2d9be603cdf71701d82bcb72dfd69 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 17 Oct 2018 08:26:13 +0400 Subject: [PATCH 045/127] MDEV-17257 Server crashes in Item::field_type_for_temporal_comparison or in get_datetime_value on SELECT with YEAR field and IN --- mysql-test/r/type_year.result | 13 +++++++++++++ mysql-test/t/type_year.test | 16 ++++++++++++++++ sql/item_cmpfunc.cc | 15 ++++++++++++--- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/type_year.result b/mysql-test/r/type_year.result index 38bddd42cfa..e9be0798426 100644 --- a/mysql-test/r/type_year.result +++ b/mysql-test/r/type_year.result @@ -394,3 +394,16 @@ select a from t1 where a=b; a drop table t1; drop function y2k; +# +# Start of 10.0 tests +# +# +# MDEV-17257 Server crashes in Item::field_type_for_temporal_comparison or in get_datetime_value on SELECT with YEAR field and IN +# +CREATE TABLE t1 (y YEAR); +SELECT * FROM t1 WHERE y IN ( CAST( '1993-03-26 10:14:20' AS DATE ), NULL ); +y +DROP TABLE t1; +# +# End of 10.0 tests +# diff --git a/mysql-test/t/type_year.test b/mysql-test/t/type_year.test index 685587fe3c5..eb6290033f4 100644 --- a/mysql-test/t/type_year.test +++ b/mysql-test/t/type_year.test @@ -188,3 +188,19 @@ drop table t1; drop function y2k; +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-17257 Server crashes in Item::field_type_for_temporal_comparison or in get_datetime_value on SELECT with YEAR field and IN +--echo # + +CREATE TABLE t1 (y YEAR); +SELECT * FROM t1 WHERE y IN ( CAST( '1993-03-26 10:14:20' AS DATE ), NULL ); +DROP TABLE t1; + + +--echo # +--echo # End of 10.0 tests +--echo # diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 49bbee9edd2..bdef3c1a89f 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4172,11 +4172,20 @@ void Item_func_in::fix_length_and_dec() if (field_item->field_type() == MYSQL_TYPE_LONGLONG || field_item->field_type() == MYSQL_TYPE_YEAR) { - bool all_converted= TRUE; + bool all_converted= true; for (arg=args+1, arg_end=args+arg_count; arg != arg_end ; arg++) { - if (!convert_const_to_int(thd, field_item, &arg[0])) - all_converted= FALSE; + /* + Explicit NULLs should not affect data cmp_type resolution: + - we ignore NULLs when calling collect_cmp_type() + - we ignore NULLs here + So this expression: + year_column IN (DATE'2001-01-01', NULL) + switches from TIME_RESULT to INT_RESULT. + */ + if (arg[0]->type() != Item::NULL_ITEM && + !convert_const_to_int(thd, field_item, &arg[0])) + all_converted= false; } if (all_converted) cmp_type= INT_RESULT; From e31e697f17f79ffa6913499e7e2d29866f24b475 Mon Sep 17 00:00:00 2001 From: Sachin Date: Sun, 14 Oct 2018 23:16:53 +0530 Subject: [PATCH 046/127] MDEV-15919 lower_case_table_names does not behave as expected(nor... consistently) on Replication Slave lower_case_table_names 0 -> 1 replication works, it's safe as long as mixed case names mapping to the lower case ones is one-to-one --- .../rpl_tests/rpl_lower_case_table_names.test | 141 ++++++++++++++++++ mysql-test/suite/rpl/r/rpl_15919.result | 16 ++ .../r/rpl_lcase_tblnames_rewrite_db.result | 33 ++++ .../suite/rpl/r/rpl_row_lcase_tblnames.result | 47 ++++++ .../suite/rpl/r/rpl_stm_lcase_tblnames.result | 44 ++++++ mysql-test/suite/rpl/t/rpl_15919-master.opt | 1 + mysql-test/suite/rpl/t/rpl_15919-slave.opt | 1 + mysql-test/suite/rpl/t/rpl_15919.test | 16 ++ .../t/rpl_lcase_tblnames_rewrite_db-slave.opt | 1 + .../rpl/t/rpl_lcase_tblnames_rewrite_db.test | 60 ++++++++ .../rpl/t/rpl_row_lcase_tblnames-slave.opt | 1 + .../suite/rpl/t/rpl_row_lcase_tblnames.test | 12 ++ .../rpl/t/rpl_stm_lcase_tblnames-slave.opt | 1 + .../suite/rpl/t/rpl_stm_lcase_tblnames.test | 12 ++ sql/log_event.cc | 46 ++++-- 15 files changed, 422 insertions(+), 10 deletions(-) create mode 100644 mysql-test/extra/rpl_tests/rpl_lower_case_table_names.test create mode 100644 mysql-test/suite/rpl/r/rpl_15919.result create mode 100644 mysql-test/suite/rpl/r/rpl_lcase_tblnames_rewrite_db.result create mode 100644 mysql-test/suite/rpl/r/rpl_row_lcase_tblnames.result create mode 100644 mysql-test/suite/rpl/r/rpl_stm_lcase_tblnames.result create mode 100644 mysql-test/suite/rpl/t/rpl_15919-master.opt create mode 100644 mysql-test/suite/rpl/t/rpl_15919-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_15919.test create mode 100644 mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db.test create mode 100644 mysql-test/suite/rpl/t/rpl_row_lcase_tblnames-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_row_lcase_tblnames.test create mode 100644 mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames.test diff --git a/mysql-test/extra/rpl_tests/rpl_lower_case_table_names.test b/mysql-test/extra/rpl_tests/rpl_lower_case_table_names.test new file mode 100644 index 00000000000..fa48142ee91 --- /dev/null +++ b/mysql-test/extra/rpl_tests/rpl_lower_case_table_names.test @@ -0,0 +1,141 @@ +# BUG#37656 +# +# This test aims at checking whether lower_case_table_names=1 option works +# for database names and table names. +# +# This test checks the following (when lower_case_table_names=1 is set on slave): +# (i) creating a database on upper case on master results in lower case +# database name on slave +# (ii) creating tables with upper case names on master results in lower case +# table names on slave +# (iii) loading data infile into capitalized table name on master replicates to +# lower case table name on slave +# (iv) Propagating changes from upper case table names on into correspondent +# lower case table names on slave works. + + +# setup: create database and tables +-- echo ******** [ MASTER ] ******** +-- let $dbname_upper= BUG_37656 +-- let $dbname_lower= `SELECT LOWER('$dbname_upper')` +-- eval CREATE DATABASE $dbname_upper +-- eval use $dbname_upper + +# assert: database names are in upper case in master and lower +# case in slave +-- eval show databases like '$dbname_upper' +sync_slave_with_master; +-- echo ******** [ SLAVE ] ******** +--eval show databases like '$dbname_lower' + +-- connection master +-- echo ******** [ MASTER ] ******** +CREATE TABLE T1 (a int); +-- eval CREATE TABLE T2 (b int) ENGINE=$engine +CREATE TABLE T3 (txt TEXT); + +# assert: that tables exist on master with upper case names +show tables; + +# assert: that tables exist on slave but with lower case names +-- sync_slave_with_master +-- echo ******** [ SLAVE ] ******** +-- eval use $dbname_lower +show tables; + +# action: lets create t1 for asserting below that t1 does not get changes +# from master (slave configured with --replicate-ignore-db=$dbname_lower.t1) +CREATE TABLE t1 (a INT); + +# action: fill data into tables +-- connection master +-- echo ******** [ MASTER ] ******** +-- eval use $dbname_upper +INSERT INTO T1 VALUES (1); +INSERT INTO T2 VALUES (1); +if (`SELECT @@session.binlog_format != 'ROW'`) +{ + -- eval LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE $dbname_upper.T3 +} + +if (`SELECT @@session.binlog_format = 'ROW'`) +{ + use test; + -- eval INSERT INTO $dbname_upper.T1 VALUES (2) + -- eval INSERT INTO $dbname_upper.T2 VALUES (2) + -- eval LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE $dbname_upper.T3 +} +# assert: lower case tables on lower case database on slave +# get updates from upper case tables on upper case +# database on master +-- sync_slave_with_master +-- echo ******** [ SLAVE ] ******** + +# assert: changes for slave's t1 were filterd out +if (`SELECT count(*) != 0 FROM t1`) +{ + -- echo UNEXPECTED DATA on $dbname_lower.t1 as table is filtered by replicate-ignore-table rules +} + +-- let $diff_tables=master:$dbname_upper.T2, slave:$dbname_lower.t2 +-- source include/diff_tables.inc + +-- let $diff_tables=master:$dbname_upper.T3, slave:$dbname_lower.t3 +-- source include/diff_tables.inc + +# clean up +-- connection master +-- echo ******** [ MASTER ] ******** +-- eval DROP DATABASE $dbname_upper +-- sync_slave_with_master + + +# +# BUG#50653: drop procedure implicitely treats db name in a case sensitive way +# + +-- connection master + +-- let $dbname= B50653 +-- let $procname= b50653_proc + +-- eval CREATE DATABASE $dbname +-- eval USE $dbname +-- eval CREATE PROCEDURE $procname() BEGIN SELECT 1; END + +if (`SELECT count(*) = 1 FROM mysql.proc WHERE name like '$dbname'`) +{ + -- die Procedure not created on MASTER +} + +-- sync_slave_with_master +if (`SELECT count(*) = 1 FROM mysql.proc WHERE name like '$dbname'`) +{ + -- die Procedure not created on SLAVE +} + +-- connection master +-- eval DROP PROCEDURE $procname + +if (`SELECT count(*) FROM mysql.proc WHERE name like '$dbname'`) +{ + -- die Procedure not dropped on MASTER +} + +-- sync_slave_with_master +if (`SELECT count(*) FROM mysql.proc WHERE name like '$dbname'`) +{ + -- die Procedure not dropped on SLAVE +} + +-- let $last_error = query_get_value("SHOW SLAVE STATUS", Last_SQL_Errno, 1) +if ($last_error) +{ + -- die UNEXPECTED SLAVE SQL error: $last_error +} + +-- connection master +-- eval DROP DATABASE $dbname +-- sync_slave_with_master + +-- source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_15919.result b/mysql-test/suite/rpl/r/rpl_15919.result new file mode 100644 index 00000000000..0c176624cf7 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_15919.result @@ -0,0 +1,16 @@ +include/master-slave.inc +[connection master] +create table RPL(a int); +insert into RPL values(1); +select * from rpl; +a +1 +insert into RPL values(3); +insert into rpl values(4); +select * from rpl; +a +1 +3 +4 +drop table RPL; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_lcase_tblnames_rewrite_db.result b/mysql-test/suite/rpl/r/rpl_lcase_tblnames_rewrite_db.result new file mode 100644 index 00000000000..3feb01d92fc --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_lcase_tblnames_rewrite_db.result @@ -0,0 +1,33 @@ +include/master-slave.inc +[connection master] +SET SQL_LOG_BIN=0; +CREATE DATABASE B37656; +SET SQL_LOG_BIN=1; +CREATE DATABASE BUG37656; +### action: show that database on slave is created in lowercase +SHOW DATABASES LIKE '%37656'; +Database (%37656) +bug37656 +USE B37656; +CREATE TABLE T1 (a int); +INSERT INTO T1 VALUES (1); +### assertion: master contains capitalized case table +SHOW TABLES; +Tables_in_B37656 +T1 +use bug37656; +### assertion: slave contains lowered case table +SHOW TABLES; +Tables_in_bug37656 +t1 +### assertion: master and slave tables do not differ +include/diff_tables.inc [master:B37656.T1, slave:bug37656.t1] +SET SQL_LOG_BIN=0; +DROP DATABASE B37656; +SET SQL_LOG_BIN=1; +SHOW DATABASES LIKE '%37656'; +Database (%37656) +DROP DATABASE BUG37656; +SHOW DATABASES LIKE '%37656'; +Database (%37656) +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_row_lcase_tblnames.result b/mysql-test/suite/rpl/r/rpl_row_lcase_tblnames.result new file mode 100644 index 00000000000..d8d459f4dcc --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_lcase_tblnames.result @@ -0,0 +1,47 @@ +include/master-slave.inc +[connection master] +******** [ MASTER ] ******** +CREATE DATABASE BUG_37656; +use BUG_37656; +show databases like 'BUG_37656'; +Database (BUG_37656) +BUG_37656 +******** [ SLAVE ] ******** +show databases like 'bug_37656'; +Database (bug_37656) +bug_37656 +******** [ MASTER ] ******** +CREATE TABLE T1 (a int); +CREATE TABLE T2 (b int) ENGINE=InnoDB; +CREATE TABLE T3 (txt TEXT); +show tables; +Tables_in_BUG_37656 +T1 +T2 +T3 +******** [ SLAVE ] ******** +use bug_37656; +show tables; +Tables_in_bug_37656 +t2 +t3 +CREATE TABLE t1 (a INT); +******** [ MASTER ] ******** +use BUG_37656; +INSERT INTO T1 VALUES (1); +INSERT INTO T2 VALUES (1); +use test; +INSERT INTO BUG_37656.T1 VALUES (2); +INSERT INTO BUG_37656.T2 VALUES (2); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE BUG_37656.T3; +******** [ SLAVE ] ******** +include/diff_tables.inc [master:BUG_37656.T2, slave:bug_37656.t2] +include/diff_tables.inc [master:BUG_37656.T3, slave:bug_37656.t3] +******** [ MASTER ] ******** +DROP DATABASE BUG_37656; +CREATE DATABASE B50653; +USE B50653; +CREATE PROCEDURE b50653_proc() BEGIN SELECT 1; END; +DROP PROCEDURE b50653_proc; +DROP DATABASE B50653; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_stm_lcase_tblnames.result b/mysql-test/suite/rpl/r/rpl_stm_lcase_tblnames.result new file mode 100644 index 00000000000..0d60f52a983 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_stm_lcase_tblnames.result @@ -0,0 +1,44 @@ +include/master-slave.inc +[connection master] +******** [ MASTER ] ******** +CREATE DATABASE BUG_37656; +use BUG_37656; +show databases like 'BUG_37656'; +Database (BUG_37656) +BUG_37656 +******** [ SLAVE ] ******** +show databases like 'bug_37656'; +Database (bug_37656) +bug_37656 +******** [ MASTER ] ******** +CREATE TABLE T1 (a int); +CREATE TABLE T2 (b int) ENGINE=InnoDB; +CREATE TABLE T3 (txt TEXT); +show tables; +Tables_in_BUG_37656 +T1 +T2 +T3 +******** [ SLAVE ] ******** +use bug_37656; +show tables; +Tables_in_bug_37656 +t2 +t3 +CREATE TABLE t1 (a INT); +******** [ MASTER ] ******** +use BUG_37656; +INSERT INTO T1 VALUES (1); +INSERT INTO T2 VALUES (1); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE BUG_37656.T3; +******** [ SLAVE ] ******** +include/diff_tables.inc [master:BUG_37656.T2, slave:bug_37656.t2] +include/diff_tables.inc [master:BUG_37656.T3, slave:bug_37656.t3] +******** [ MASTER ] ******** +DROP DATABASE BUG_37656; +CREATE DATABASE B50653; +USE B50653; +CREATE PROCEDURE b50653_proc() BEGIN SELECT 1; END; +DROP PROCEDURE b50653_proc; +DROP DATABASE B50653; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_15919-master.opt b/mysql-test/suite/rpl/t/rpl_15919-master.opt new file mode 100644 index 00000000000..9b27aef9bf8 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_15919-master.opt @@ -0,0 +1 @@ +--lower_case_table_names=0 diff --git a/mysql-test/suite/rpl/t/rpl_15919-slave.opt b/mysql-test/suite/rpl/t/rpl_15919-slave.opt new file mode 100644 index 00000000000..62ab6dad1e0 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_15919-slave.opt @@ -0,0 +1 @@ +--lower_case_table_names=1 diff --git a/mysql-test/suite/rpl/t/rpl_15919.test b/mysql-test/suite/rpl/t/rpl_15919.test new file mode 100644 index 00000000000..0462f7fd067 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_15919.test @@ -0,0 +1,16 @@ +--source include/master-slave.inc +--source include/have_innodb.inc +--connection master +create table RPL(a int); +insert into RPL values(1); + +--sync_slave_with_master +select * from rpl; +insert into RPL values(3); +insert into rpl values(4); +select * from rpl; + +--connection master +drop table RPL; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db-slave.opt b/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db-slave.opt new file mode 100644 index 00000000000..0031a57a693 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db-slave.opt @@ -0,0 +1 @@ +--lower-case-table-names=1 "--replicate-rewrite-db=b37656->bug37656" diff --git a/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db.test b/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db.test new file mode 100644 index 00000000000..9c804d8206a --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_lcase_tblnames_rewrite_db.test @@ -0,0 +1,60 @@ +# BUG#37656 +# +# DESCRIPTION +# +# +# This test case is tests whether replication works properly when +# slave is configured with --lower-case-table-names=1 and replication +# rewrite rules are in effect. +# +# It checks four issues: +# +# (i) master contains capitalized table name +# +# (ii) slave contains lowered case table name +# +# (iii) master and slave tables do not differ +# +-- source include/master-slave.inc +-- source include/not_windows.inc + +SET SQL_LOG_BIN=0; +CREATE DATABASE B37656; +SET SQL_LOG_BIN=1; + +-- connection slave +CREATE DATABASE BUG37656; + +-- echo ### action: show that database on slave is created in lowercase +SHOW DATABASES LIKE '%37656'; + +-- connection master +USE B37656; +CREATE TABLE T1 (a int); +INSERT INTO T1 VALUES (1); + +-- echo ### assertion: master contains capitalized case table +SHOW TABLES; + +-- sync_slave_with_master + +use bug37656; + +-- echo ### assertion: slave contains lowered case table +SHOW TABLES; + +-- echo ### assertion: master and slave tables do not differ +let $diff_tables= master:B37656.T1, slave:bug37656.t1; + +-- source include/diff_tables.inc + +-- connection master +SET SQL_LOG_BIN=0; +DROP DATABASE B37656; +SET SQL_LOG_BIN=1; +SHOW DATABASES LIKE '%37656'; + +-- connection slave +DROP DATABASE BUG37656; +SHOW DATABASES LIKE '%37656'; +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames-slave.opt b/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames-slave.opt new file mode 100644 index 00000000000..7624b013dcd --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames-slave.opt @@ -0,0 +1 @@ +--replicate-do-db=bug_37656 --replicate-ignore-table=buG_37656.T1 --replicate-do-table=bUg_37656.T2 --replicate-do-table=bUg_37656.T3 --lower-case-table-names=1 diff --git a/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames.test b/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames.test new file mode 100644 index 00000000000..78a66a7df32 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_lcase_tblnames.test @@ -0,0 +1,12 @@ +# BUG#37656 +# +# For details look into extra/rpl_tests/rpl_lower_case_table_names.test +# + +-- source include/master-slave.inc +-- source include/have_innodb.inc +-- source include/not_windows.inc +-- source include/have_binlog_format_row.inc + +-- let $engine=InnoDB +-- source extra/rpl_tests/rpl_lower_case_table_names.test diff --git a/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames-slave.opt b/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames-slave.opt new file mode 100644 index 00000000000..8be29bbe976 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames-slave.opt @@ -0,0 +1 @@ +--replicate-do-db=bug_37656 --replicate-ignore-table=bug_37656.t1 --replicate-do-table=bug_37656.t2 --replicate-do-table=bug_37656.t3 --lower-case-table-names=1 diff --git a/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames.test b/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames.test new file mode 100644 index 00000000000..00df8e9d385 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_stm_lcase_tblnames.test @@ -0,0 +1,12 @@ +# BUG#37656 +# +# For details look into extra/rpl_tests/rpl_lower_case_table_names.test +# + +-- source include/master-slave.inc +-- source include/have_innodb.inc +-- source include/not_windows.inc +-- source include/have_binlog_format_mixed_or_statement.inc + +-- let $engine=InnoDB +-- source extra/rpl_tests/rpl_lower_case_table_names.test diff --git a/sql/log_event.cc b/sql/log_event.cc index a33704bdd03..f84cfc94f6f 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -245,6 +245,27 @@ static void inline slave_rows_error_report(enum loglevel level, int ha_error, } #endif +#if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) +static void set_thd_db(THD *thd,const char *db, uint32 db_len) +{ + char lcase_db_buf[NAME_LEN +1]; + LEX_STRING new_db; + new_db.length= db_len; + if (lower_case_table_names == 1) + { + strmov(lcase_db_buf, db); + my_casedn_str(system_charset_info, lcase_db_buf); + new_db.str= lcase_db_buf; + } + else + new_db.str= (char*) db; + /* TODO WARNING this makes rewrite_db respect lower_case_table_names values + * for more info look MDEV-17446 */ + new_db.str= (char*) rpl_filter->get_rewrite_db(new_db.str, + &new_db.length); + thd->set_db(new_db.str, new_db.length); +} +#endif /* Cache that will automatically be written to a dedicated file on destruction. @@ -3619,7 +3640,6 @@ bool test_if_equal_repl_errors(int expected_error, int actual_error) int Query_log_event::do_apply_event(Relay_log_info const *rli, const char *query_arg, uint32 q_len_arg) { - LEX_STRING new_db; int expected_error,actual_error= 0; HA_CREATE_INFO db_options; DBUG_ENTER("Query_log_event::do_apply_event"); @@ -3646,9 +3666,7 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, goto end; } - new_db.length= db_len; - new_db.str= (char *) rpl_filter->get_rewrite_db(db, &new_db.length); - thd->set_db(new_db.str, new_db.length); /* allocates a copy of 'db' */ + set_thd_db(thd, db, db_len); /* Setting the character set and collation of the current database thd->db. @@ -5424,13 +5442,10 @@ void Load_log_event::set_fields(const char* affected_db, int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli, bool use_rli_only_for_errors) { - LEX_STRING new_db; DBUG_ENTER("Load_log_event::do_apply_event"); - new_db.length= db_len; - new_db.str= (char *) rpl_filter->get_rewrite_db(db, &new_db.length); - thd->set_db(new_db.str, new_db.length); DBUG_ASSERT(thd->query() == 0); + set_thd_db(thd, db, db_len); thd->reset_query_inner(); // Should not be needed thd->is_slave_error= 0; clear_all_errors(thd, const_cast(rli)); @@ -5485,6 +5500,8 @@ int Load_log_event::do_apply_event(NET* net, Relay_log_info const *rli, thd->warning_info->opt_clear_warning_info(thd->query_id); TABLE_LIST tables; + if (lower_case_table_names) + my_casedn_str(system_charset_info, (char *)table_name); tables.init_one_table(thd->strmake(thd->db, thd->db_length), thd->db_length, table_name, strlen(table_name), @@ -9678,7 +9695,7 @@ check_table_map(Relay_log_info const *rli, RPL_TABLE_LIST *table_list) int Table_map_log_event::do_apply_event(Relay_log_info const *rli) { RPL_TABLE_LIST *table_list; - char *db_mem, *tname_mem; + char *db_mem, *tname_mem, *ptr; size_t dummy_len; void *memory; DBUG_ENTER("Table_map_log_event::do_apply_event(Relay_log_info*)"); @@ -9694,8 +9711,17 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) NullS))) DBUG_RETURN(HA_ERR_OUT_OF_MEM); - strmov(db_mem, rpl_filter->get_rewrite_db(m_dbnam, &dummy_len)); + strmov(db_mem, m_dbnam); strmov(tname_mem, m_tblnam); + if (lower_case_table_names) + { + my_casedn_str(files_charset_info, (char*)tname_mem); + my_casedn_str(files_charset_info, (char*)db_mem); + } + + /* rewrite rules changed the database */ + if (((ptr= (char*) rpl_filter->get_rewrite_db(db_mem, &dummy_len)) != db_mem)) + strmov(db_mem, ptr); table_list->init_one_table(db_mem, strlen(db_mem), tname_mem, strlen(tname_mem), From 4ac85d6fd7c43fd0c45c98c0b55810591bd7594b Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Thu, 18 Oct 2018 22:48:28 +0400 Subject: [PATCH 047/127] MDEV-14815 - Server crash or AddressSanitizer errors or valgrind warnings in thr_lock / has_old_lock upon FLUSH TABLES Explicit partition access of partitioned MEMORY table under LOCK TABLES may cause subsequent statements to crash the server, deadlock, trigger valgrind warnings or ASAN errors. Freed memory was being used due to incorrect cleanup. At least MyISAM and InnoDB don't seem to be affected, since their THR_LOCK structures don't survive FLUSH TABLES. MEMORY keeps table shared data (including THR_LOCK) even if there're no open instances. There's partition_info::lock_partitions bitmap, which holds bits of partitions allowed to be accessed after pruning. This bitmap is updated for each individual statement. This bitmap was abused in ha_partition::store_lock() such that when we need to unlock a table, locked by LOCK TABLES, only locks for partitions that were accessed by previous statement were released. Eventually FLUSH TABLES frees THR_LOCK_DATA objects, which are still linked into THR_LOCK lists. When such THR_LOCK gets reused we end up with freed memory access. Fixed by using ha_partition::m_locked_partitions bitmap similarly to ha_partition::external_lock(). --- mysql-test/r/partition_explicit_prune.result | 19 +++++++++++++++++++ mysql-test/t/partition_explicit_prune.test | 19 +++++++++++++++++++ sql/ha_partition.cc | 9 +++++++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/partition_explicit_prune.result b/mysql-test/r/partition_explicit_prune.result index 3ca1e688e8f..c8ab243c34c 100644 --- a/mysql-test/r/partition_explicit_prune.result +++ b/mysql-test/r/partition_explicit_prune.result @@ -1870,3 +1870,22 @@ CREATE TABLE t2 LIKE t1 PARTITION (p0, p2); ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'PARTITION (p0, p2)' at line 1 DROP TABLE t1; SET @@default_storage_engine = @old_default_storage_engine; +# +# MDEV-14815 - Server crash or AddressSanitizer errors or valgrind warnings in thr_lock / has_old_lock upon FLUSH TABLES +# +CREATE TABLE t1 (i INT) ENGINE=MEMORY PARTITION BY RANGE (i) (PARTITION p0 VALUES LESS THAN (4), PARTITION pm VALUES LESS THAN MAXVALUE); +CREATE TABLE t2 (i INT) ENGINE=MEMORY; +LOCK TABLE t1 WRITE, t2 WRITE; +SELECT * FROM t1 PARTITION (p0); +i +FLUSH TABLES; +SELECT * FROM t1 PARTITION (p0); +i +ALTER TABLE t1 TRUNCATE PARTITION p0; +SELECT * FROM t1 PARTITION (p0); +i +ALTER TABLE t1 EXCHANGE PARTITION p0 WITH TABLE t2; +SELECT * FROM t1 PARTITION (p0); +i +UNLOCK TABLES; +DROP TABLE t1, t2; diff --git a/mysql-test/t/partition_explicit_prune.test b/mysql-test/t/partition_explicit_prune.test index 68b829fbcc3..b8b6e480ce9 100644 --- a/mysql-test/t/partition_explicit_prune.test +++ b/mysql-test/t/partition_explicit_prune.test @@ -858,3 +858,22 @@ CREATE TABLE t2 LIKE t1 PARTITION (p0, p2); DROP TABLE t1; SET @@default_storage_engine = @old_default_storage_engine; + + +--echo # +--echo # MDEV-14815 - Server crash or AddressSanitizer errors or valgrind warnings in thr_lock / has_old_lock upon FLUSH TABLES +--echo # +CREATE TABLE t1 (i INT) ENGINE=MEMORY PARTITION BY RANGE (i) (PARTITION p0 VALUES LESS THAN (4), PARTITION pm VALUES LESS THAN MAXVALUE); +CREATE TABLE t2 (i INT) ENGINE=MEMORY; +LOCK TABLE t1 WRITE, t2 WRITE; +SELECT * FROM t1 PARTITION (p0); +FLUSH TABLES; +SELECT * FROM t1 PARTITION (p0); +ALTER TABLE t1 TRUNCATE PARTITION p0; +SELECT * FROM t1 PARTITION (p0); +ALTER TABLE t1 EXCHANGE PARTITION p0 WITH TABLE t2; +SELECT * FROM t1 PARTITION (p0); +UNLOCK TABLES; + +# Cleanup +DROP TABLE t1, t2; diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 0488ebfb60f..02de92aa17f 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -3907,9 +3907,14 @@ THR_LOCK_DATA **ha_partition::store_lock(THD *thd, } else { - for (i= bitmap_get_first_set(&(m_part_info->lock_partitions)); + MY_BITMAP *used_partitions= lock_type == TL_UNLOCK || + lock_type == TL_IGNORE ? + &m_locked_partitions : + &m_part_info->lock_partitions; + + for (i= bitmap_get_first_set(used_partitions); i < m_tot_parts; - i= bitmap_get_next_set(&m_part_info->lock_partitions, i)) + i= bitmap_get_next_set(used_partitions, i)) { DBUG_PRINT("info", ("store lock %d iteration", i)); to= m_file[i]->store_lock(thd, to, lock_type); From 642394197e397fa73d776ee1fecdbd07f54607ff Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Fri, 19 Oct 2018 19:10:45 +0400 Subject: [PATCH 048/127] Remove unused code. --- sql/ha_partition.h | 1 - sql/lock.cc | 17 ----------- sql/lock.h | 1 - sql/sql_base.cc | 70 ---------------------------------------------- sql/sql_insert.cc | 2 +- 5 files changed, 1 insertion(+), 90 deletions(-) diff --git a/sql/ha_partition.h b/sql/ha_partition.h index 71ae84b06a0..0e9f5c4b499 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -251,7 +251,6 @@ private: /* Variables for lock structures. */ - THR_LOCK_DATA lock; /* MySQL lock */ bool auto_increment_lock; /**< lock reading/updating auto_inc */ /** diff --git a/sql/lock.cc b/sql/lock.cc index 3354da2640b..09e8dbd6aa8 100644 --- a/sql/lock.cc +++ b/sql/lock.cc @@ -541,23 +541,6 @@ void mysql_lock_remove(THD *thd, MYSQL_LOCK *locked,TABLE *table) } -/** Abort all other threads waiting to get lock in table. */ - -void mysql_lock_abort(THD *thd, TABLE *table, bool upgrade_lock) -{ - MYSQL_LOCK *locked; - DBUG_ENTER("mysql_lock_abort"); - - if ((locked= get_lock_data(thd, &table, 1, GET_LOCK_UNLOCK))) - { - for (uint i=0; i < locked->lock_count; i++) - thr_abort_locks(locked->locks[i]->lock, upgrade_lock); - my_free(locked); - } - DBUG_VOID_RETURN; -} - - /** Abort one thread / table combination. diff --git a/sql/lock.h b/sql/lock.h index a4833cdc38e..a9183bf8580 100644 --- a/sql/lock.h +++ b/sql/lock.h @@ -32,7 +32,6 @@ void mysql_unlock_tables(THD *thd, MYSQL_LOCK *sql_lock, bool free_lock= 1); void mysql_unlock_read_tables(THD *thd, MYSQL_LOCK *sql_lock); void mysql_unlock_some_tables(THD *thd, TABLE **table,uint count); void mysql_lock_remove(THD *thd, MYSQL_LOCK *locked,TABLE *table); -void mysql_lock_abort(THD *thd, TABLE *table, bool upgrade_lock); bool mysql_lock_abort_for_thread(THD *thd, TABLE *table); MYSQL_LOCK *mysql_lock_merge(MYSQL_LOCK *a,MYSQL_LOCK *b); /* Lock based on name */ diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 1a17c8e39d5..403ba853222 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9100,76 +9100,6 @@ my_bool mysql_rm_tmp_tables(void) unireg support functions *****************************************************************************/ -/** - A callback to the server internals that is used to address - special cases of the locking protocol. - Invoked when acquiring an exclusive lock, for each thread that - has a conflicting shared metadata lock. - - This function: - - aborts waiting of the thread on a data lock, to make it notice - the pending exclusive lock and back off. - - if the thread is an INSERT DELAYED thread, sends it a KILL - signal to terminate it. - - @note This function does not wait for the thread to give away its - locks. Waiting is done outside for all threads at once. - - @param thd Current thread context - @param in_use The thread to wake up - @param needs_thr_lock_abort Indicates that to wake up thread - this call needs to abort its waiting - on table-level lock. - - @retval TRUE if the thread was woken up - @retval FALSE otherwise. - - @note It is one of two places where border between MDL and the - rest of the server is broken. -*/ - -bool mysql_notify_thread_having_shared_lock(THD *thd, THD *in_use, - bool needs_thr_lock_abort) -{ - bool signalled= FALSE; - if ((in_use->system_thread & SYSTEM_THREAD_DELAYED_INSERT) && - !in_use->killed) - { - in_use->killed= KILL_SYSTEM_THREAD; - mysql_mutex_lock(&in_use->mysys_var->mutex); - if (in_use->mysys_var->current_cond) - { - mysql_mutex_lock(in_use->mysys_var->current_mutex); - mysql_cond_broadcast(in_use->mysys_var->current_cond); - mysql_mutex_unlock(in_use->mysys_var->current_mutex); - } - mysql_mutex_unlock(&in_use->mysys_var->mutex); - signalled= TRUE; - } - - if (needs_thr_lock_abort) - { - mysql_mutex_lock(&in_use->LOCK_thd_data); - for (TABLE *thd_table= in_use->open_tables; - thd_table ; - thd_table= thd_table->next) - { - /* - Check for TABLE::needs_reopen() is needed since in some places we call - handler::close() for table instance (and set TABLE::db_stat to 0) - and do not remove such instances from the THD::open_tables - for some time, during which other thread can see those instances - (e.g. see partitioning code). - */ - if (!thd_table->needs_reopen()) - signalled|= mysql_lock_abort_for_thread(thd, thd_table); - } - mysql_mutex_unlock(&in_use->LOCK_thd_data); - } - return signalled; -} - - int setup_ftfuncs(SELECT_LEX *select_lex) { List_iterator li(*(select_lex->ftfunc_list)), diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 8b03c722898..5ecdd72541f 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -2353,7 +2353,7 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) The thread could be killed with an error message if di->handle_inserts() or di->open_and_lock_table() fails. The thread could be killed without an error message if - killed using mysql_notify_thread_having_shared_lock() or + killed using THD::notify_shared_lock() or kill_delayed_threads_for_table(). */ if (!thd.is_error()) From d851dd619f9e373c453355fd613088488e2c3d5c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 19 Oct 2018 20:18:34 +0200 Subject: [PATCH 049/127] MDEV-13912 mysql_upgrade: case (in)sensitivity for stored procedures mysql_upgrade used to convert all columns of mysql.db to utf8_general_ci and then back to utf8_bin. In two separate ALTER's. This failed if UNIQUE indexes in mysql.db contained entries that differ only in the letter case. --- mysql-test/r/lowercase_fs_off.result | 41 ++++++++++++++++++++++++++++ mysql-test/t/lowercase_fs_off.test | 10 +++++++ scripts/mysql_system_tables_fix.sql | 10 +++---- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/lowercase_fs_off.result b/mysql-test/r/lowercase_fs_off.result index c3284b225dd..362661d8911 100644 --- a/mysql-test/r/lowercase_fs_off.result +++ b/mysql-test/r/lowercase_fs_off.result @@ -63,3 +63,44 @@ CREATE TRIGGER t1_bi BEFORE INSERT ON t1 FOR EACH ROW SET new.a= 1; RENAME TABLE t1 TO T1; ALTER TABLE T1 RENAME t1; DROP TABLE t1; +create database TEST; +create procedure TEST.pr() begin end; +create procedure test.pr() begin end; +Phase 1/4: Fixing views +Phase 2/4: Fixing table and database names +Phase 3/4: Checking and upgrading tables +Processing databases +information_schema +TEST +mtr +mtr.global_suppressions OK +mtr.test_suppressions OK +mysql +mysql.columns_priv OK +mysql.db OK +mysql.event OK +mysql.func OK +mysql.help_category OK +mysql.help_keyword OK +mysql.help_relation OK +mysql.help_topic OK +mysql.host OK +mysql.ndb_binlog_index OK +mysql.plugin OK +mysql.proc OK +mysql.procs_priv OK +mysql.proxies_priv OK +mysql.servers OK +mysql.tables_priv OK +mysql.time_zone OK +mysql.time_zone_leap_second OK +mysql.time_zone_name OK +mysql.time_zone_transition OK +mysql.time_zone_transition_type OK +mysql.user OK +performance_schema +test +Phase 4/4: Running 'mysql_fix_privilege_tables' +OK +drop procedure test.pr; +drop database TEST; diff --git a/mysql-test/t/lowercase_fs_off.test b/mysql-test/t/lowercase_fs_off.test index 1be0351f9bc..8729efde02c 100644 --- a/mysql-test/t/lowercase_fs_off.test +++ b/mysql-test/t/lowercase_fs_off.test @@ -102,3 +102,13 @@ CREATE TRIGGER t1_bi BEFORE INSERT ON t1 FOR EACH ROW SET new.a= 1; RENAME TABLE t1 TO T1; ALTER TABLE T1 RENAME t1; DROP TABLE t1; + +# +# MDEV-13912 mysql_upgrade: case (in)sensitivity for stored procedures +# +create database TEST; +create procedure TEST.pr() begin end; +create procedure test.pr() begin end; +--exec $MYSQL_UPGRADE --force 2>&1 +drop procedure test.pr; +drop database TEST; diff --git a/scripts/mysql_system_tables_fix.sql b/scripts/mysql_system_tables_fix.sql index 9bc61f38cea..dd792ce9f2f 100644 --- a/scripts/mysql_system_tables_fix.sql +++ b/scripts/mysql_system_tables_fix.sql @@ -468,15 +468,15 @@ ALTER TABLE proc MODIFY name char(64) DEFAULT '' NOT NULL, DEFAULT CHARACTER SET utf8; # Correct the character set and collation -ALTER TABLE proc CONVERT TO CHARACTER SET utf8; # Reset some fields after the conversion SET @alter_statement = CONCAT(" -ALTER TABLE proc MODIFY db - char(64) collate utf8_bin DEFAULT '' NOT NULL, +ALTER TABLE proc CONVERT TO CHARACTER SET utf8, + MODIFY db + char(64) binary DEFAULT '' NOT NULL, MODIFY definer - char(", @definer_name_length, ") collate utf8_bin DEFAULT '' NOT NULL, + char(", @definer_name_length, ") binary DEFAULT '' NOT NULL, MODIFY comment - char(64) collate utf8_bin DEFAULT '' NOT NULL + char(64) binary DEFAULT '' NOT NULL "); PREPARE alter_stmt FROM @alter_statement; EXECUTE alter_stmt; From d6889f2b58c0caeb31dc3f117ae33c31ec74df8c Mon Sep 17 00:00:00 2001 From: wlad Date: Fri, 19 Oct 2018 21:56:15 +0100 Subject: [PATCH 050/127] MDEV-17504 : add diagnostics if creation of directories fail in mysql_install_db.exe In addition,don't remove datadir if it was not created or was not empty. --- sql/CMakeLists.txt | 2 +- sql/mysql_install_db.cc | 87 ++++++++++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 074dd89ec7a..f2c1eeb6566 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -485,7 +485,7 @@ IF(WIN32) COMPONENT Server ) SET_TARGET_PROPERTIES(mysql_install_db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR}) - TARGET_LINK_LIBRARIES(mysql_install_db mysys) + TARGET_LINK_LIBRARIES(mysql_install_db mysys shlwapi) ADD_LIBRARY(winservice STATIC winservice.c) TARGET_LINK_LIBRARIES(winservice shell32) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index ce40b5edc2c..737ed97d790 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -26,6 +26,7 @@ #include #include #include +#include #define USAGETEXT \ "mysql_install_db.exe Ver 1.00 for Windows\n" \ @@ -93,9 +94,7 @@ static struct my_option my_long_options[]= static my_bool -get_one_option(int optid, - const struct my_option *opt __attribute__ ((unused)), - char *argument __attribute__ ((unused))) +get_one_option(int optid, const struct my_option *, char *) { DBUG_ENTER("get_one_option"); switch (optid) { @@ -109,7 +108,7 @@ get_one_option(int optid, } -static void die(const char *fmt, ...) +ATTRIBUTE_NORETURN static void die(const char *fmt, ...) { va_list args; DBUG_ENTER("die"); @@ -290,7 +289,7 @@ static int create_myini() FILE *myini= fopen("my.ini","wt"); if (!myini) { - die("Cannot create my.ini in data directory"); + die("Can't create my.ini in data directory"); } /* Write out server settings. */ @@ -308,7 +307,7 @@ static int create_myini() if (enable_named_pipe) { - fprintf(myini,"enable-named-pipe\n"); + fprintf(myini,"named-pipe=ON\n"); } if (opt_socket && opt_socket[0]) @@ -523,6 +522,7 @@ static int set_directory_permissions(const char *dir, const char *os_user) } + /* Create database instance (including registering as service etc) .*/ static int create_db_instance() @@ -532,19 +532,78 @@ static int create_db_instance() DWORD cwd_len= MAX_PATH; char cmdline[3*MAX_PATH]; FILE *in; + bool cleanup_datadir= true; + DWORD last_error; verbose("Running bootstrap"); GetCurrentDirectory(cwd_len, cwd); - CreateDirectory(opt_datadir, NULL); /*ignore error, it might already exist */ + + /* Create datadir and datadir/mysql, if they do not already exist. */ + + if (!CreateDirectory(opt_datadir, NULL) && (GetLastError() != ERROR_ALREADY_EXISTS)) + { + last_error = GetLastError(); + switch(last_error) + { + case ERROR_ACCESS_DENIED: + die("Can't create data directory '%s' (access denied)\n", + opt_datadir); + break; + case ERROR_PATH_NOT_FOUND: + die("Can't create data directory '%s' " + "(one or more intermediate directories do not exist)\n", + opt_datadir); + break; + default: + die("Can't create data directory '%s', last error %u\n", + opt_datadir, last_error); + break; + } + } if (!SetCurrentDirectory(opt_datadir)) { - die("Cannot set current directory to '%s'\n",opt_datadir); - return -1; + last_error = GetLastError(); + switch (last_error) + { + case ERROR_DIRECTORY: + die("Can't set current directory to '%s', the path is not a valid directory \n", + opt_datadir); + break; + default: + die("Can' set current directory to '%s', last error %u\n", + opt_datadir, last_error); + break; + } } - CreateDirectory("mysql",NULL); + if (PathIsDirectoryEmpty(opt_datadir)) + { + cleanup_datadir= false; + } + + if (!CreateDirectory("mysql",NULL)) + { + last_error = GetLastError(); + DWORD attributes; + switch(last_error) + { + case ERROR_ACCESS_DENIED: + die("Can't create subdirectory 'mysql' in '%s' (access denied)\n",opt_datadir); + break; + case ERROR_ALREADY_EXISTS: + attributes = GetFileAttributes("mysql"); + + if (attributes == INVALID_FILE_ATTRIBUTES) + die("GetFileAttributes() failed for existing file '%s\\mysql', last error %u", + opt_datadir, GetLastError()); + else if (!(attributes & FILE_ATTRIBUTE_DIRECTORY)) + die("File '%s\\mysql' exists, but it is not a directory", opt_datadir); + + break; + } + } /* Set data directory permissions for both current user and @@ -565,11 +624,11 @@ static int create_db_instance() if (setvbuf(in, NULL, _IONBF, 0)) { - verbose("WARNING: Cannot disable buffering on mysqld's stdin"); + verbose("WARNING: Can't disable buffering on mysqld's stdin"); } if (fwrite("use mysql;\n",11,1, in) != 1) { - verbose("ERROR: Cannot write to mysqld's stdin"); + verbose("ERROR: Can't write to mysqld's stdin"); ret= 1; goto end; } @@ -580,7 +639,7 @@ static int create_db_instance() /* Write the bootstrap script to stdin. */ if (fwrite(mysql_bootstrap_sql[i], strlen(mysql_bootstrap_sql[i]), 1, in) != 1) { - verbose("ERROR: Cannot write to mysqld's stdin"); + verbose("ERROR: Can't write to mysqld's stdin"); ret= 1; goto end; } @@ -649,7 +708,7 @@ static int create_db_instance() } end: - if (ret) + if (ret && cleanup_datadir) { SetCurrentDirectory(cwd); clean_directory(opt_datadir); From 5b63a660dda7de2cb74106edc2267568ac70b70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 20 Oct 2018 09:58:34 +0300 Subject: [PATCH 051/127] MDEV-11369/MDEV-12288: Reset DB_TRX_ID on the metadata record On the hidden metadata record, if instant ALTER TABLE was executed multiple times on the same table, purge could fail to reset the DB_TRX_ID,DB_ROLL_PTR on the updated metadata record. This is only a cosmetic failure that was caught (and separately fixed) in 10.4 during the MDEV-15562 development. The problem was that occasionally, innodb.instant_alter_crash would fail with a result difference due to the DB_TRX_ID,DB_ROLL_PTR not having been reset on the metadata record. This bug should have no noticeable impact, because the metadata record is invisible to the SQL layer, and never subjected to MVCC or locking. I was unable to repeat the problem on 10.3. row_purge_parse_undo_rec(): Set node->ref for the metadata record. --- storage/innobase/row/row0purge.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 527cf0336d5..5699c8b2f56 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1147,10 +1147,13 @@ err_exit: /* Read to the partial row the fields that occur in indexes */ if (!(node->cmpl_info & UPD_NODE_NO_ORD_CHANGE)) { + ut_ad(!(node->update->info_bits & REC_INFO_MIN_REC_FLAG)); ptr = trx_undo_rec_get_partial_row( ptr, clust_index, node->update, &node->row, type == TRX_UNDO_UPD_DEL_REC, node->heap); + } else if (node->update->info_bits & REC_INFO_MIN_REC_FLAG) { + node->ref = &trx_undo_metadata; } return(true); From 55fc3fb08830cfe8e55b45f5f0e1acf55d00cb4b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 20 Oct 2018 13:29:27 +0200 Subject: [PATCH 052/127] Revert "MDEV-14429 sql_safe_updates in my.cnf not work" This reverts commit dc3a20b191362227a6f85fadfc3a8b6c76f69ec6. It requires GET_BIT in include/my_getopt.h, which is available only in 10.3+ --- mysql-test/r/mysqld--help.result | 2 -- sql/sys_vars.cc | 2 +- sql/sys_vars.h | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 0d8728a6768..825c7e786a3 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -790,7 +790,6 @@ The following specify which files/extra groups are read (specified before remain this size --sql-mode=name Syntax: sql-mode=mode[,mode[,mode...]]. See the manual for the complete list of valid sql modes - --sql-safe-updates sql_safe_updates --stack-trace Print a symbolic stack trace on failure (Defaults to on; use --skip-stack-trace to disable.) --stored-program-cache=# @@ -1082,7 +1081,6 @@ slow-launch-time 2 slow-query-log FALSE sort-buffer-size 2097152 sql-mode -sql-safe-updates FALSE stack-trace TRUE stored-program-cache 256 symbolic-links FALSE diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 080e74ce367..ffe861ddfbc 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2719,7 +2719,7 @@ static Sys_var_bit Sys_auto_is_null( static Sys_var_bit Sys_safe_updates( "sql_safe_updates", "sql_safe_updates", - SESSION_VAR(option_bits), CMD_LINE(REQUIRED_ARG), OPTION_SAFE_UPDATES, + SESSION_VAR(option_bits), NO_CMD_LINE, OPTION_SAFE_UPDATES, DEFAULT(FALSE)); static Sys_var_bit Sys_buffer_results( diff --git a/sql/sys_vars.h b/sql/sys_vars.h index c21bf8244f0..dbe27ab107e 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -1443,6 +1443,7 @@ public: bitmask= reverse_semantics ? ~bitmask_arg : bitmask_arg; set(global_var_ptr(), def_val); SYSVAR_ASSERT(def_val < 2); + SYSVAR_ASSERT(getopt.id == -1); // force NO_CMD_LINE SYSVAR_ASSERT(size == sizeof(ulonglong)); } bool session_update(THD *thd, set_var *var) From 554ce5a0ccc64881e04da69d7a2da3ace60589a6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 20 Oct 2018 12:56:28 +0100 Subject: [PATCH 053/127] Attempt to fix build by a workaround for a bug in Windows8.1 SDK Problem description and workaround are here https://developercommunity.visualstudio.com/content/problem/185399/error-c2760-in-combaseapih-with-windows-sdk-81-and.html --- sql/mysql_install_db.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index 737ed97d790..fc8bce08276 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -26,6 +26,7 @@ #include #include #include +struct IUnknown; #include #define USAGETEXT \ From 5b735e8f09c81a2533c30cdd62d60dffa8226cd4 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 17 Sep 2018 21:40:23 +0300 Subject: [PATCH 054/127] MDEV-17133 dump thread reads from a past position According to logs analysis the Dump thread attempted to read again data which was already sent. The reason of regressed read turns out in an _my_b_cache_read() early exit branch which missed to distinguish between total zero size read (e.g ineffective read when Count argument is zero) from a case when the requested amount of data is fully read out by sole accessing the cache's file. In the latter case such then *effective* reading was not reflected in the cache's state to screw the cache's state. Fixed with a check introduced of whether the file reading was effective prior to early exit. When this is the case conduct standard cache state change to account the actual read size. Notice the bug can show up also as an error to read binlog event e.g through BINLOG_GTID_POS() (of MDEV-16886). --- mysys/mf_iocache.c | 7 ++- unittest/sql/mf_iocache-t.cc | 87 +++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index 56b1ae3fc6e..79893d6f7c1 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -563,7 +563,7 @@ int _my_b_write(IO_CACHE *info, const uchar *Buffer, size_t Count) int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count) { - size_t length, diff_length, left_length= 0, max_length; + size_t length= 0, diff_length, left_length= 0, max_length; my_off_t pos_in_file; DBUG_ENTER("_my_b_cache_read"); @@ -668,7 +668,10 @@ int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count) else { info->error= 0; - DBUG_RETURN(0); /* EOF */ + if (length == 0) /* nothing was read */ + DBUG_RETURN(0); /* EOF */ + + length= 0; /* non-zero size read was done */ } } else if ((length= mysql_file_read(info->file,info->buffer, max_length, diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc index 8f97745f0fc..95451d52f86 100644 --- a/unittest/sql/mf_iocache-t.cc +++ b/unittest/sql/mf_iocache-t.cc @@ -285,11 +285,95 @@ void mdev14014() close_cached_file(&info); } +void mdev17133() +{ + int res, k; + const int eof_iter=4, read_iter= 4; + uchar buf_i[1024*256]; // read + uchar buf_o[sizeof(buf_i)]; // write + const size_t eof_block_size= sizeof(buf_o) / eof_iter; + const size_t read_size= eof_block_size / read_iter; + size_t total; + + srand((uint) time(NULL)); + memset(buf_i, 0, sizeof( buf_i)); + memset(buf_o, FILL, sizeof(buf_o)); + + diag("MDEV-17133 Dump thread reads from the past"); + + init_io_cache_encryption(); + + res= open_cached_file(&info, 0, 0, CACHE_SIZE, 0); + ok(res == 0, "open_cached_file" INFO_TAIL); + + res= my_b_write(&info, buf_o, sizeof(buf_o)); + ok(res == 0, "buffer is written" INFO_TAIL); + res= my_b_tell(&info); + ok(res == sizeof(buf_o), "cache size as expected"); + + res= my_b_flush_io_cache(&info, 1); + ok(res == 0, "flush" INFO_TAIL); + res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0); + ok(res == 0, "reinit READ_CACHE" INFO_TAIL); + + // read the written data by chunks of variable size eof_iter times + for (k= eof_iter, info.end_of_file=0, total= 0; k; k--) + { + int i; + size_t curr_read_size; + info.end_of_file= + k == 1 ? sizeof(buf_o) : + MY_MIN(sizeof(buf_o), + info.end_of_file + eof_block_size + + // plus 25% of block for randomization to the average + (eof_block_size/4 - rand() % (eof_block_size/2))); + + // read a chunk by blocks of variable size read_iter times + // the last block completes the current chunk + for (i= 0; i < read_iter; i++, total += curr_read_size) + { + char buf_check[eof_block_size]; + uint a,b; + + a= info.end_of_file - total; + b= read_size + read_size/4 - rand() % (read_size/2); + curr_read_size= (i == read_iter - 1) ? info.end_of_file - total : + MY_MIN(a, b); + + DBUG_ASSERT(curr_read_size <= info.end_of_file - total); + + res= my_b_read(&info, buf_i + total, MY_MIN(19, curr_read_size)); + ok(res == 0, "read of 19"); + // mark read bytes in the used part of the cache buffer + memset(info.buffer, 0, info.read_pos - info.buffer); + + // random size 2nd read + res= my_b_read(&info, buf_i + total + MY_MIN(19, curr_read_size), + 19 >= curr_read_size ? 0 : curr_read_size - 19); + ok(res == 0, "rest of read %lu", curr_read_size - 19); + // mark read bytes in the used part of the cache buffer + memset(info.buffer, 0, info.read_pos - info.buffer); + + // check that no marked bytes are read + memset(buf_check, FILL, curr_read_size); + ok(memcmp(buf_i + total, buf_check, curr_read_size) == 0, + "read correct data"); + } + ok(info.pos_in_file + (info.read_end - info.buffer) == info.end_of_file, + "cache is read up to eof"); + ok(total == info.end_of_file, "total matches eof"); + } + ok(total == sizeof(buf_i), "read total size match"); + ok(buf_i[sizeof(buf_i) - 1] == FILL, "data read correctly"); + + close_cached_file(&info); +} + int main(int argc __attribute__((unused)),char *argv[]) { MY_INIT(argv[0]); - plan(51); + plan(114); /* temp files with and without encryption */ encrypt_tmp_files= 1; @@ -306,6 +390,7 @@ int main(int argc __attribute__((unused)),char *argv[]) encrypt_tmp_files= 0; mdev14014(); + mdev17133(); my_end(0); return exit_status(); From 98f15dac60749ea6957147da9fb423d7dcc6fb05 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 23 Oct 2018 15:59:51 +0200 Subject: [PATCH 055/127] Bug#27799513: POTENTIAL DOUBLE FREE OR CORRUPTION OF HEAP INFO (HP_INFO) test case --- mysql-test/suite/heap/heap_btree.result | 4 ++++ mysql-test/suite/heap/heap_btree.test | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/mysql-test/suite/heap/heap_btree.result b/mysql-test/suite/heap/heap_btree.result index 12a011778c6..83d1bcb6c92 100644 --- a/mysql-test/suite/heap/heap_btree.result +++ b/mysql-test/suite/heap/heap_btree.result @@ -379,3 +379,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range uniq_id uniq_id 8 NULL 4 Using where drop table t1; End of 5.3 tests +create table t1 (id int, a varchar(300) not null, key using btree(a)) engine=heap; +insert t1 values (1, repeat('a', 300)); +drop table t1; +End of 5.5 tests diff --git a/mysql-test/suite/heap/heap_btree.test b/mysql-test/suite/heap/heap_btree.test index 02c09f52263..aca41c430b3 100644 --- a/mysql-test/suite/heap/heap_btree.test +++ b/mysql-test/suite/heap/heap_btree.test @@ -279,3 +279,12 @@ explain select 0+a from t1 where a in (869751,736494,226312,802616); drop table t1; --echo End of 5.3 tests + +# +# Bug#27799513: POTENTIAL DOUBLE FREE OR CORRUPTION OF HEAP INFO (HP_INFO) +# +create table t1 (id int, a varchar(300) not null, key using btree(a)) engine=heap; +insert t1 values (1, repeat('a', 300)); +drop table t1; + +--echo End of 5.5 tests From 73e1ffdc6899aea69182c15930f712a178813660 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 23 Oct 2018 16:00:45 +0200 Subject: [PATCH 056/127] Bug#27919254 MYSQL USER ESCALATES ITS PRIVILEGE BY PLACING ARBITRARY PIDS INTO ITS PID FILES --- support-files/mysql.server.sh | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 1776919e950..c1d85ba2664 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -128,8 +128,9 @@ esac parse_server_arguments() { for arg do + val=`echo "$arg" | sed -e 's/^[^=]*=//'` case "$arg" in - --basedir=*) basedir=`echo "$arg" | sed -e 's/^[^=]*=//'` + --basedir=*) basedir="$val" bindir="$basedir/bin" if test -z "$datadir_set"; then datadir="$basedir/data" @@ -143,14 +144,15 @@ parse_server_arguments() { fi libexecdir="$basedir/libexec" ;; - --datadir=*) datadir=`echo "$arg" | sed -e 's/^[^=]*=//'` + --datadir=*) datadir="$val" datadir_set=1 ;; --log-basename=*|--hostname=*|--loose-log-basename=*) - mysqld_pid_file_path=`echo "$arg.pid" | sed -e 's/^[^=]*=//'` + mysqld_pid_file_path="$val.pid" ;; - --pid-file=*) mysqld_pid_file_path=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; - --service-startup-timeout=*) service_startup_timeout=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; + --pid-file=*) mysqld_pid_file_path="$val" ;; + --service-startup-timeout=*) service_startup_timeout="$val" ;; + --user=*) user="$val"; ;; esac done } @@ -182,6 +184,12 @@ else test -z "$print_defaults" && print_defaults="my_print_defaults" fi +user='@MYSQLD_USER@' + +su_kill() { + su - $user -s /bin/sh -c "kill $*" >/dev/null 2>&1 +} + # # Read defaults file from 'basedir'. If there is no defaults file there # check if it's in the old (depricated) place (datadir) and read it from there @@ -210,7 +218,7 @@ wait_for_gone () { while test $i -ne $service_startup_timeout ; do - if kill -0 "$pid" 2>/dev/null; then + if su_kill -0 "$pid" ; then : # the server still runs else if test ! -s "$pid_file_path"; then @@ -250,7 +258,7 @@ wait_for_ready () { if $bindir/mysqladmin ping >/dev/null 2>&1; then log_success_msg return 0 - elif kill -0 $! 2>/dev/null ; then + elif kill -0 $! ; then : # mysqld_safe is still running else # mysqld_safe is no longer running, abort the wait loop @@ -319,10 +327,9 @@ case "$mode" in then mysqld_pid=`cat "$mysqld_pid_file_path"` - if (kill -0 $mysqld_pid 2>/dev/null) - then + if su_kill -0 $mysqld_pid ; then echo $echo_n "Shutting down MariaDB" - kill $mysqld_pid + su_kill $mysqld_pid # mysqld should remove the pid file when it exits, so wait for it. wait_for_gone $mysqld_pid "$mysqld_pid_file_path"; return_value=$? else @@ -355,7 +362,7 @@ case "$mode" in 'reload'|'force-reload') if test -s "$mysqld_pid_file_path" ; then read mysqld_pid < "$mysqld_pid_file_path" - kill -HUP $mysqld_pid && log_success_msg "Reloading service MariaDB" + su_kill -HUP $mysqld_pid && log_success_msg "Reloading service MariaDB" touch "$mysqld_pid_file_path" else log_failure_msg "MariaDB PID file could not be found!" @@ -366,7 +373,7 @@ case "$mode" in # First, check to see if pid file exists if test -s "$mysqld_pid_file_path" ; then read mysqld_pid < "$mysqld_pid_file_path" - if kill -0 $mysqld_pid 2>/dev/null ; then + if su_kill -0 $mysqld_pid ; then log_success_msg "MariaDB running ($mysqld_pid)" exit 0 else From 7223369d89877cea20e959227a51d6511bd3ebd9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 23 Oct 2018 16:42:10 +0200 Subject: [PATCH 057/127] Revert "Update rules" This reverts commit ebaacf0747bad9f41686a23f0b8ba4ff9f364f47. Pushed by mistake. Broke tokudb on trusty-amd64. --- debian/dist/Ubuntu/rules | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/debian/dist/Ubuntu/rules b/debian/dist/Ubuntu/rules index e570a99c80a..5f3ea672770 100755 --- a/debian/dist/Ubuntu/rules +++ b/debian/dist/Ubuntu/rules @@ -1,10 +1,7 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -#export DEB_BUILD_HARDENING=1 -export DEB_BUILD_MAINT_OPTIONS = hardening=+all,-pie -DPKG_EXPORT_BUILDFLAGS = 1 -include /usr/share/dpkg/buildflags.mk +export DEB_BUILD_HARDENING=1 PACKAGE=mariadb-5.5 From 5bc30247c4b79021f99303cd65dfa75d4504a773 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 24 Oct 2018 10:30:31 +0200 Subject: [PATCH 058/127] 5.5.61-38.13 --- storage/xtradb/.clang-format | 111 ++++++++++++++++++++++++++++ storage/xtradb/CMakeLists.txt | 5 ++ storage/xtradb/fsp/fsp0fsp.c | 37 ---------- storage/xtradb/handler/ha_innodb.cc | 12 ++- storage/xtradb/handler/ha_innodb.h | 2 +- storage/xtradb/include/data0type.ic | 2 + storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0online.c | 4 +- storage/xtradb/os/os0sync.c | 18 ----- storage/xtradb/row/row0sel.c | 1 + storage/xtradb/srv/srv0srv.c | 2 + storage/xtradb/sync/sync0sync.c | 4 +- 12 files changed, 135 insertions(+), 65 deletions(-) create mode 100644 storage/xtradb/.clang-format diff --git a/storage/xtradb/.clang-format b/storage/xtradb/.clang-format new file mode 100644 index 00000000000..d757d0a5a05 --- /dev/null +++ b/storage/xtradb/.clang-format @@ -0,0 +1,111 @@ +# generated with: +# clang-format-5.0 -style=Google --dump-config + +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +SortUsingDeclarations: true +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +TabWidth: 8 + + +# changes for MySQL 5.x (InnoDB) +AlignConsecutiveDeclarations: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: All +ColumnLimit: 78 +DerivePointerAlignment: false +IndentWidth: 8 +MaxEmptyLinesToKeep: 2 +PointerAlignment: Right +ReflowComments: false +SortIncludes: false +SpaceAfterCStyleCast: true +UseTab: Always diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index b97473849ef..91033e60dc6 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -308,3 +308,8 @@ MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE DEFAULT MODULE_OUTPUT_NAME ha_innodb LINK_LIBRARIES ${ZLIB_LIBRARY}) + +IF(WITH_INNOBASE_STORAGE_ENGINE) + # Remove -DMYSQL_SERVER, it breaks embedded build + SET_TARGET_PROPERTIES(innobase PROPERTIES COMPILE_DEFINITIONS "") +ENDIF() diff --git a/storage/xtradb/fsp/fsp0fsp.c b/storage/xtradb/fsp/fsp0fsp.c index 8927e7c5841..9abbd8172bc 100644 --- a/storage/xtradb/fsp/fsp0fsp.c +++ b/storage/xtradb/fsp/fsp0fsp.c @@ -431,43 +431,6 @@ xdes_find_bit( return(ULINT_UNDEFINED); } -/**********************************************************************//** -Looks for a descriptor bit having the desired value. Scans the extent in -a direction opposite to xdes_find_bit. -@return bit index of the bit, ULINT_UNDEFINED if not found */ -UNIV_INLINE -ulint -xdes_find_bit_downward( -/*===================*/ - xdes_t* descr, /*!< in: descriptor */ - ulint bit, /*!< in: XDES_FREE_BIT or XDES_CLEAN_BIT */ - ibool val, /*!< in: desired bit value */ - ulint hint, /*!< in: hint of which bit position would be desirable */ - mtr_t* mtr) /*!< in: mtr */ -{ - ulint i; - - ut_ad(descr && mtr); - ut_ad(val <= TRUE); - ut_ad(hint < FSP_EXTENT_SIZE); - ut_ad(mtr_memo_contains_page(mtr, descr, MTR_MEMO_PAGE_X_FIX)); - for (i = hint + 1; i > 0; i--) { - if (val == xdes_get_bit(descr, bit, i - 1, mtr)) { - - return(i - 1); - } - } - - for (i = FSP_EXTENT_SIZE - 1; i > hint; i--) { - if (val == xdes_get_bit(descr, bit, i, mtr)) { - - return(i); - } - } - - return(ULINT_UNDEFINED); -} - /**********************************************************************//** Returns the number of used pages in a descriptor. @return number of pages used */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 3850b4a12e8..cc4b782cb18 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. @@ -1938,6 +1938,7 @@ trx_is_registered_for_2pc( /*********************************************************************//** Note that a transaction owns the prepare_commit_mutex. */ +#ifndef EXTENDED_FOR_COMMIT_ORDERED static inline void trx_owns_prepare_commit_mutex_set( @@ -1947,6 +1948,7 @@ trx_owns_prepare_commit_mutex_set( ut_a(trx_is_registered_for_2pc(trx)); trx->owns_prepare_mutex = 1; } +#endif // EXTENDED_FOR_COMMIT_ORDERED /*********************************************************************//** Note that a transaction has been registered with MySQL 2PC coordinator. */ @@ -5830,7 +5832,7 @@ build_template( if (UNIV_LIKELY(templ_type == ROW_MYSQL_REC_FIELDS)) { /* Decide which columns we should fetch and which we can skip. */ - register const ibool index_contains_field = + const ibool index_contains_field = dict_index_contains_col_or_prefix(index, i); if (!index_contains_field && prebuilt->read_just_key) { @@ -6199,6 +6201,8 @@ no_commit: innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_insert_for_mysql((byte*) record, prebuilt); + DEBUG_SYNC(user_thd, "ib_after_row_insert"); + #ifdef EXTENDED_FOR_USERSTAT if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { @@ -7648,8 +7652,6 @@ create_table_def( } } - ut_a(field->type() < 256); /* we assume in dtype_form_prtype() - that this fits in one byte */ col_len = field->pack_length(); /* The MySQL pack length contains 1 or 2 bytes length field @@ -10404,8 +10406,10 @@ ha_innobase::start_stmt( case SQLCOM_INSERT: case SQLCOM_UPDATE: case SQLCOM_DELETE: + case SQLCOM_REPLACE: init_table_handle_for_HANDLER(); prebuilt->select_lock_type = LOCK_X; + prebuilt->stored_select_lock_type = LOCK_X; error = row_lock_table_for_mysql(prebuilt, NULL, 1); if (error != DB_SUCCESS) { diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index 4f21dcff7e8..aaf897afdf8 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -183,7 +183,7 @@ class ha_innobase: public handler ha_rows estimate_rows_upper_bound(); void update_create_info(HA_CREATE_INFO* create_info); - int create(const char *name, register TABLE *form, + int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info); int truncate(); int delete_table(const char *name); diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index a04fa632886..48ab18ef6b6 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -446,6 +446,7 @@ dtype_get_fixed_size_low( return(0); } #endif /* UNIV_DEBUG */ + // fallthrough case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: @@ -524,6 +525,7 @@ dtype_get_min_size_low( return(0); } #endif /* UNIV_DEBUG */ + // fallthrough case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 64a2bfa70fe..b6861d62394 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -64,7 +64,7 @@ component, i.e. we show M.N.P as M.N */ (INNODB_VERSION_MAJOR << 8 | INNODB_VERSION_MINOR) #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 38.11 +#define PERCONA_INNODB_VERSION 38.13 #endif #define INNODB_VERSION_STR MYSQL_SERVER_VERSION diff --git a/storage/xtradb/log/log0online.c b/storage/xtradb/log/log0online.c index 0354ef29ea8..c6e8c7c5198 100644 --- a/storage/xtradb/log/log0online.c +++ b/storage/xtradb/log/log0online.c @@ -1561,10 +1561,10 @@ log_online_open_bitmap_file_read_only( if (srv_data_home_len && srv_data_home[srv_data_home_len-1] != SRV_PATH_SEPARATOR) { - ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%c%s", + ut_snprintf(bitmap_file->name, sizeof(bitmap_file->name), "%s%c%s", srv_data_home, SRV_PATH_SEPARATOR, name); } else { - ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%s", + ut_snprintf(bitmap_file->name, sizeof(bitmap_file->name), "%s%s", srv_data_home, name); } bitmap_file->file diff --git a/storage/xtradb/os/os0sync.c b/storage/xtradb/os/os0sync.c index 3a182692da3..642451dcc2c 100644 --- a/storage/xtradb/os/os0sync.c +++ b/storage/xtradb/os/os0sync.c @@ -226,24 +226,6 @@ os_cond_broadcast( #endif } -/*********************************************************//** -Wakes one thread waiting for condition variable */ -UNIV_INLINE -void -os_cond_signal( -/*==========*/ - os_cond_t* cond) /*!< in: condition variable. */ -{ - ut_a(cond); - -#ifdef __WIN__ - ut_a(wake_condition_variable != NULL); - wake_condition_variable(cond); -#else - ut_a(pthread_cond_signal(cond) == 0); -#endif -} - /*********************************************************//** Destroys condition variable */ UNIV_INLINE diff --git a/storage/xtradb/row/row0sel.c b/storage/xtradb/row/row0sel.c index 140cfc61383..d021ff1445f 100644 --- a/storage/xtradb/row/row0sel.c +++ b/storage/xtradb/row/row0sel.c @@ -4298,6 +4298,7 @@ no_gap_lock: prebuilt->new_rec_locks = 1; } err = DB_SUCCESS; + break; case DB_SUCCESS: break; case DB_LOCK_WAIT: diff --git a/storage/xtradb/srv/srv0srv.c b/storage/xtradb/srv/srv0srv.c index 3770a8804dd..b55e9fc44cd 100644 --- a/storage/xtradb/srv/srv0srv.c +++ b/storage/xtradb/srv/srv0srv.c @@ -1402,6 +1402,8 @@ retry: os_fast_mutex_unlock(&srv_conc_mutex); + DEBUG_SYNC_C("user_thread_waiting"); + trx->op_info = "sleeping before joining InnoDB queue"; /* Peter Zaitsev suggested that we take the sleep away diff --git a/storage/xtradb/sync/sync0sync.c b/storage/xtradb/sync/sync0sync.c index 7654fade6e8..27138629dbc 100644 --- a/storage/xtradb/sync/sync0sync.c +++ b/storage/xtradb/sync/sync0sync.c @@ -315,9 +315,9 @@ mutex_create_func( /* NOTE! The very first mutexes are not put to the mutex list */ - if ((mutex == &mutex_list_mutex) + if (mutex == &mutex_list_mutex #ifdef UNIV_SYNC_DEBUG - || (mutex == &sync_thread_mutex) + || mutex == &sync_thread_mutex #endif /* UNIV_SYNC_DEBUG */ ) { From 0140bfac5e216bd7ba8ad324bd914d596bf59a1f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 2 Oct 2018 22:50:28 +0200 Subject: [PATCH 059/127] MDEV-16127 mroonga/storage.* tests fail with GCC 8 Tests were failing because in TIME_from_longlong_datetime_packed() GCC8 at -O2 assumed that tmp is always positive and used mul and shr while it used imul and sar at -O1 (where tests passed). GCC8 used multiplication (by 0x4ec4ec4ec4ec4ec5) and shift to implement division by 13. It could assume that tmp is always positive, because the function starts with `if (tmp < 0) tmp= -tmp;` But this assumption breaks if tmp=0x8000000000000000; This is invalid value and TIME_from_longlong_datetime_packed() should never see it, garbage in - garbage out. It was getting this invalid value because mroonga tried to convert a NULL key part to MYSQL_TIME. If the key part value is NULL, datetime2 value of it happens to be bzero-ed, which is invalid binary datetime2 value. The correct behavior is not to try to interpret the key part value, if it is marked as NULL. But this minimal fix only covers the datetime2 type. --- sql/compat56.cc | 3 +++ storage/mroonga/ha_mroonga.cpp | 9 ++++++--- storage/mroonga/ha_mroonga.hpp | 2 +- storage/mroonga/lib/mrn_multiple_column_key_codec.cpp | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sql/compat56.cc b/sql/compat56.cc index 3bd6b21a154..357b4bcf78b 100644 --- a/sql/compat56.cc +++ b/sql/compat56.cc @@ -252,6 +252,9 @@ void TIME_from_longlong_datetime_packed(MYSQL_TIME *ltime, longlong tmp) { longlong ymd, hms; longlong ymdhms, ym; + + DBUG_ASSERT(tmp != LONGLONG_MIN); + if ((ltime->neg= (tmp < 0))) tmp= -tmp; diff --git a/storage/mroonga/ha_mroonga.cpp b/storage/mroonga/ha_mroonga.cpp index d317fef44fb..02213f0f179 100644 --- a/storage/mroonga/ha_mroonga.cpp +++ b/storage/mroonga/ha_mroonga.cpp @@ -11800,7 +11800,8 @@ int ha_mroonga::storage_encode_key_timestamp2(Field *field, const uchar *key, #endif #ifdef MRN_HAVE_MYSQL_TYPE_DATETIME2 -int ha_mroonga::storage_encode_key_datetime2(Field *field, const uchar *key, +int ha_mroonga::storage_encode_key_datetime2(Field *field, bool is_null, + const uchar *key, uchar *buf, uint *size) { MRN_DBUG_ENTER_METHOD(); @@ -11808,7 +11809,7 @@ int ha_mroonga::storage_encode_key_datetime2(Field *field, const uchar *key, bool truncated = false; Field_datetimef *datetime2_field = (Field_datetimef *)field; - longlong packed_time = + longlong packed_time = is_null ? 0 : my_datetime_packed_from_binary(key, datetime2_field->decimals()); MYSQL_TIME mysql_time; TIME_from_longlong_datetime_packed(&mysql_time, packed_time); @@ -11935,6 +11936,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, MRN_DBUG_ENTER_METHOD(); int error; bool truncated = false; + bool is_null = false; const uchar *ptr = key; error = mrn_change_encoding(ctx, field->charset()); @@ -11942,6 +11944,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, DBUG_RETURN(error); if (field->null_bit) { + is_null = *ptr; ptr += 1; } @@ -12039,7 +12042,7 @@ int ha_mroonga::storage_encode_key(Field *field, const uchar *key, #endif #ifdef MRN_HAVE_MYSQL_TYPE_DATETIME2 case MYSQL_TYPE_DATETIME2: - error = storage_encode_key_datetime2(field, ptr, buf, size); + error = storage_encode_key_datetime2(field, is_null, ptr, buf, size); break; #endif #ifdef MRN_HAVE_MYSQL_TYPE_TIME2 diff --git a/storage/mroonga/ha_mroonga.hpp b/storage/mroonga/ha_mroonga.hpp index b769583d434..71ec40ee63a 100644 --- a/storage/mroonga/ha_mroonga.hpp +++ b/storage/mroonga/ha_mroonga.hpp @@ -796,7 +796,7 @@ private: uchar *buf, uint *size); #endif #ifdef MRN_HAVE_MYSQL_TYPE_DATETIME2 - int storage_encode_key_datetime2(Field *field, const uchar *key, + int storage_encode_key_datetime2(Field *field, bool is_null, const uchar *key, uchar *buf, uint *size); #endif #ifdef MRN_HAVE_MYSQL_TYPE_TIME2 diff --git a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp index 0038a7fe34f..fa3c49a4236 100644 --- a/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp +++ b/storage/mroonga/lib/mrn_multiple_column_key_codec.cpp @@ -89,11 +89,13 @@ namespace mrn { for (int i = 0; i < n_key_parts && current_mysql_key < mysql_key_end; i++) { KEY_PART_INFO *key_part = &(key_info_->key_part[i]); Field *field = key_part->field; + bool is_null = false; DBUG_PRINT("info", ("mroonga: key_part->length=%u", key_part->length)); if (field->null_bit) { DBUG_PRINT("info", ("mroonga: field has null bit")); *current_grn_key = 0; + is_null = *current_mysql_key; current_mysql_key += 1; current_grn_key += 1; (*grn_key_length)++; @@ -164,7 +166,7 @@ namespace mrn { { Field_datetimef *datetimef_field = static_cast(field); - long long int mysql_datetime_packed = + long long int mysql_datetime_packed = is_null ? 0 : my_datetime_packed_from_binary(current_mysql_key, datetimef_field->decimals()); MYSQL_TIME mysql_time; From 329058be298a471006e55a674db1c4d769acaf67 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 27 Sep 2018 17:28:01 +0200 Subject: [PATCH 060/127] wsrep: create a macro for the error: label that is used by WSREP_TO_ISOLATION_BEGIN and other galera macros, to avoid the need for wrapping this label in #ifdef WITH_WSREP/#endif --- include/wsrep.h | 11 ++++++----- sql/events.cc | 16 +++++++--------- sql/sql_acl.cc | 4 ++-- sql/sql_admin.cc | 3 +++ sql/sql_alter.cc | 11 ++++------- sql/sql_base.cc | 1 + sql/sql_parse.cc | 1 + sql/sql_plugin.cc | 10 ++++------ sql/sql_trigger.cc | 5 ++--- sql/sql_view.cc | 6 +++--- sql/wsrep_mysqld.cc | 2 +- 11 files changed, 34 insertions(+), 36 deletions(-) diff --git a/include/wsrep.h b/include/wsrep.h index 0d32547d69c..14d624aec26 100644 --- a/include/wsrep.h +++ b/include/wsrep.h @@ -25,12 +25,12 @@ #define WSREP_MYSQL_DB (char *)"mysql" #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) \ if (WSREP_ON && WSREP(thd) && wsrep_to_isolation_begin(thd, db_, table_, table_list_)) \ - goto error; + goto wsrep_error_label; #define WSREP_TO_ISOLATION_BEGIN_ALTER(db_, table_, table_list_, alter_info_) \ if (WSREP_ON && WSREP(thd) && wsrep_to_isolation_begin(thd, db_, table_, \ table_list_, alter_info_)) \ - goto error; + goto wsrep_error_label; #define WSREP_TO_ISOLATION_END \ if (WSREP_ON && (WSREP(thd) || (thd && thd->wsrep_exec_mode==TOTAL_ORDER))) \ @@ -42,7 +42,7 @@ */ #define WSREP_TO_ISOLATION_BEGIN_WRTCHK(db_, table_, table_list_) \ if (WSREP(thd) && !thd->lex->no_write_to_binlog \ - && wsrep_to_isolation_begin(thd, db_, table_, table_list_)) goto error; + && wsrep_to_isolation_begin(thd, db_, table_, table_list_)) goto wsrep_error_label; #define WSREP_DEBUG(...) \ if (wsrep_debug) WSREP_LOG(sql_print_information, ##__VA_ARGS__) @@ -52,8 +52,8 @@ #define WSREP_SYNC_WAIT(thd_, before_) \ { if (WSREP_CLIENT(thd_) && \ - wsrep_sync_wait(thd_, before_)) goto error; } - + wsrep_sync_wait(thd_, before_)) goto wsrep_error_label; } +#define WSREP_ERROR_LABEL wsrep_error_label #else #define IF_WSREP(A,B) B #define DBUG_ASSERT_IF_WSREP(A) @@ -65,6 +65,7 @@ #define WSREP_TO_ISOLATION_END #define WSREP_TO_ISOLATION_BEGIN_WRTCHK(db_, table_, table_list_) #define WSREP_SYNC_WAIT(thd_, before_) +#define WSREP_ERROR_LABEL goto wsrep_error_label; wsrep_error_label #endif /* WITH_WSREP */ diff --git a/sql/events.cc b/sql/events.cc index 3fdb432cba6..187a3208d9f 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -418,10 +418,10 @@ Events::create_event(THD *thd, Event_parse_data *parse_data) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); -#ifdef WITH_WSREP - error: + +WSREP_ERROR_LABEL: DBUG_RETURN(TRUE); -#endif /* WITH_WSREP */ + } @@ -549,10 +549,9 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); -#ifdef WITH_WSREP -error: + +WSREP_ERROR_LABEL: DBUG_RETURN(TRUE); -#endif /* WITH_WSREP */ } @@ -616,10 +615,9 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); -#ifdef WITH_WSREP -error: + +WSREP_ERROR_LABEL: DBUG_RETURN(TRUE); -#endif /* WITH_WSREP */ } diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 72d65e14a94..886ff90f35b 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -2861,7 +2861,7 @@ end: close_mysql_tables(thd); #ifdef WITH_WSREP -error: // this label is used in WSREP_TO_ISOLATION_BEGIN +WSREP_ERROR_LABEL: if (WSREP(thd) && !thd->wsrep_applier) { WSREP_TO_ISOLATION_END; @@ -3014,7 +3014,7 @@ end: close_mysql_tables(thd); #ifdef WITH_WSREP -error: // this label is used in WSREP_TO_ISOLATION_END +WSREP_ERROR_LABEL: if (WSREP(thd) && !thd->wsrep_applier) { WSREP_TO_ISOLATION_END; diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index be23cd3341b..b8881f8f108 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -1262,6 +1262,7 @@ bool Sql_cmd_analyze_table::execute(THD *thd) m_lex->query_tables= first_table; error: +WSREP_ERROR_LABEL: DBUG_RETURN(res); } @@ -1320,6 +1321,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) m_lex->query_tables= first_table; error: +WSREP_ERROR_LABEL: DBUG_RETURN(res); } @@ -1354,5 +1356,6 @@ bool Sql_cmd_repair_table::execute(THD *thd) m_lex->query_tables= first_table; error: +WSREP_ERROR_LABEL: DBUG_RETURN(res); } diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 0accc04c10d..758489ddcfb 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -330,13 +330,10 @@ bool Sql_cmd_alter_table::execute(THD *thd) lex->ignore); DBUG_RETURN(result); -#ifdef WITH_WSREP -error: - { - WSREP_WARN("ALTER TABLE isolation failure"); - DBUG_RETURN(TRUE); - } -#endif /* WITH_WSREP */ + +WSREP_ERROR_LABEL: + WSREP_WARN("ALTER TABLE isolation failure"); + DBUG_RETURN(TRUE); } bool Sql_cmd_discard_import_tablespace::execute(THD *thd) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index c4b243e5aba..e6e1b9beb96 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -4813,6 +4813,7 @@ restart: } error: +WSREP_ERROR_LABEL: THD_STAGE_INFO(thd, stage_after_opening_tables); thd_proc_info(thd, 0); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index add059340c9..a76d9f0bbd3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -5712,6 +5712,7 @@ end_with_restore_list: goto finish; error: +WSREP_ERROR_LABEL: res= TRUE; finish: diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 8d055fd612f..1000fc3711a 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2185,10 +2185,9 @@ err: if (argv) free_defaults(argv); DBUG_RETURN(error); -#ifdef WITH_WSREP -error: + +WSREP_ERROR_LABEL: DBUG_RETURN(TRUE); -#endif /* WITH_WSREP */ } @@ -2330,10 +2329,9 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(error); -#ifdef WITH_WSREP -error: + +WSREP_ERROR_LABEL: DBUG_RETURN(TRUE); -#endif /* WITH_WSREP */ } diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 4a484723f02..14e1e84739b 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -612,10 +612,9 @@ end: my_ok(thd); DBUG_RETURN(result); -#ifdef WITH_WSREP - error: + +WSREP_ERROR_LABEL: DBUG_RETURN(true); -#endif /* WITH_WSREP */ } diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 6bd6b6a7b63..b7ff4513a32 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -691,10 +691,10 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); DBUG_RETURN(0); -#ifdef WITH_WSREP - error: + +WSREP_ERROR_LABEL: res= TRUE; -#endif /* WITH_WSREP */ + err: THD_STAGE_INFO(thd, stage_end); lex->link_first_table_back(view, link_to_local); diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 1b2d7fe04e6..c3f5dd1ce97 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -2715,7 +2715,7 @@ bool wsrep_create_like_table(THD* thd, TABLE_LIST* table, return(false); -error: +WSREP_ERROR_LABEL: thd->wsrep_TOI_pre_query= NULL; return (true); } From a21e01a53d309c4c949d41f85a43211008bac1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 09:08:44 +0300 Subject: [PATCH 061/127] MDEV-17541 KILL QUERY during lock wait in FOREIGN KEY check causes hang row_ins_check_foreign_constraint(): Do not overwrite hard errors with the soft error DB_LOCK_WAIT. This prevents an infinite wait loop when DB_INTERRUPTED was returned. For DB_LOCK_WAIT, row_insert_for_mysql() would keep invoking row_ins_step() and the transaction would remain active until the server shutdown is initiated. --- mysql-test/suite/innodb/r/foreign_key.result | 21 ++++++++++- mysql-test/suite/innodb/t/foreign_key.test | 39 ++++++++++++++++++-- storage/innobase/row/row0ins.cc | 7 +++- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/innodb/r/foreign_key.result b/mysql-test/suite/innodb/r/foreign_key.result index a151651b594..831417aff5c 100644 --- a/mysql-test/suite/innodb/r/foreign_key.result +++ b/mysql-test/suite/innodb/r/foreign_key.result @@ -252,7 +252,6 @@ DELETE FROM t1 WHERE id = 1; ERROR HY000: Lock wait timeout exceeded; try restarting transaction connection con1; COMMIT; -disconnect con1; connection default; SELECT * FROM t2; id ref_id f @@ -332,7 +331,25 @@ PRIMARY KEY (store_id), UNIQUE KEY idx_unique_manager (manager_staff_id), CONSTRAINT fk_store_staff FOREIGN KEY (manager_staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE ) ENGINE=InnoDB; -SET FOREIGN_KEY_CHECKS=DEFAULT; LOCK TABLE staff WRITE; UNLOCK TABLES; DROP TABLES staff, store; +SET FOREIGN_KEY_CHECKS=1; +# +# MDEV-17541 KILL QUERY during lock wait in FOREIGN KEY check hangs +# +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 (a INT PRIMARY KEY, FOREIGN KEY (a) REFERENCES t1(a)) +ENGINE=InnoDB; +connection con1; +INSERT INTO t1 SET a=1; +BEGIN; +DELETE FROM t1; +connection default; +INSERT INTO t2 SET a=1; +connection con1; +kill query @id; +connection default; +ERROR 70100: Query execution was interrupted +disconnect con1; +DROP TABLE t2,t1; diff --git a/mysql-test/suite/innodb/t/foreign_key.test b/mysql-test/suite/innodb/t/foreign_key.test index 7a8a9295ee7..44b9e28d8fc 100644 --- a/mysql-test/suite/innodb/t/foreign_key.test +++ b/mysql-test/suite/innodb/t/foreign_key.test @@ -228,7 +228,6 @@ DELETE FROM t1 WHERE id = 1; --connection con1 COMMIT; ---disconnect con1 --connection default SELECT * FROM t2; @@ -288,8 +287,6 @@ insert into t1 values(1, 1); insert into t2(f1) values(1); drop table t2, t1; ---source include/wait_until_count_sessions.inc - # # MDEV-12669 Circular foreign keys cause a loop and OOM upon LOCK TABLE # @@ -308,8 +305,42 @@ CREATE TABLE store ( UNIQUE KEY idx_unique_manager (manager_staff_id), CONSTRAINT fk_store_staff FOREIGN KEY (manager_staff_id) REFERENCES staff (staff_id) ON DELETE RESTRICT ON UPDATE CASCADE ) ENGINE=InnoDB; -SET FOREIGN_KEY_CHECKS=DEFAULT; LOCK TABLE staff WRITE; UNLOCK TABLES; DROP TABLES staff, store; +SET FOREIGN_KEY_CHECKS=1; + +--echo # +--echo # MDEV-17541 KILL QUERY during lock wait in FOREIGN KEY check hangs +--echo # + +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE t2 (a INT PRIMARY KEY, FOREIGN KEY (a) REFERENCES t1(a)) +ENGINE=InnoDB; + +connection con1; +INSERT INTO t1 SET a=1; +BEGIN; +DELETE FROM t1; + +connection default; +let $ID= `SELECT @id := CONNECTION_ID()`; +send INSERT INTO t2 SET a=1; + +connection con1; +let $wait_condition= + select count(*) = 1 from information_schema.processlist + where state = 'update' and info = 'INSERT INTO t2 SET a=1'; +--source include/wait_condition.inc +let $ignore= `SELECT @id := $ID`; +kill query @id; + +connection default; +--error ER_QUERY_INTERRUPTED +reap; +disconnect con1; + +DROP TABLE t2,t1; + +--source include/wait_until_count_sessions.inc diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 4e08377916d..f72ade422b3 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1871,9 +1871,12 @@ do_possible_lock_wait: thr->lock_state = QUE_THR_LOCK_NOLOCK; - if (check_table->to_be_dropped - || trx->error_state == DB_LOCK_WAIT_TIMEOUT) { + err = trx->error_state; + if (err != DB_SUCCESS) { + } else if (check_table->to_be_dropped) { err = DB_LOCK_WAIT_TIMEOUT; + } else { + err = DB_LOCK_WAIT; } my_atomic_addlint(&check_table->n_foreign_key_checks_running, From 5dd3b52f950d688bbe9ea2e8cd10b5206198a096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 24 Oct 2018 11:04:38 +0300 Subject: [PATCH 062/127] MDEV-17531 Crash in RENAME TABLE with FOREIGN KEY and FULLTEXT INDEX In RENAME TABLE, when an error occurs while renaming FOREIGN KEY constraint, that error would be overwritten when renaming the InnoDB internal tables related to FULLTEXT INDEX. row_rename_table_for_mysql(): Do not attempt to rename the internal tables if an error already occurred. This problem was originally reported as Oracle Bug#27545888. --- mysql-test/suite/innodb/r/foreign_key.result | 32 +++++++++++++ mysql-test/suite/innodb/t/foreign_key.test | 50 ++++++++++++++++++++ storage/innobase/row/row0mysql.cc | 5 +- storage/xtradb/row/row0mysql.cc | 5 +- 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb/r/foreign_key.result b/mysql-test/suite/innodb/r/foreign_key.result index 0d04f27f51f..b6462000b46 100644 --- a/mysql-test/suite/innodb/r/foreign_key.result +++ b/mysql-test/suite/innodb/r/foreign_key.result @@ -17,3 +17,35 @@ SET FOREIGN_KEY_CHECKS=DEFAULT; LOCK TABLE staff WRITE; UNLOCK TABLES; DROP TABLES staff, store; +SET FOREIGN_KEY_CHECKS=1; +# +# MDEV-17531 Crash in RENAME TABLE with FOREIGN KEY and FULLTEXT INDEX +# +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE DATABASE best; +CREATE TABLE t3 (a INT PRIMARY KEY, +CONSTRAINT t2_ibfk_1 FOREIGN KEY (a) REFERENCES t1(a)) ENGINE=InnoDB; +CREATE TABLE best.t2 (a INT PRIMARY KEY, b TEXT, FULLTEXT INDEX(b), +FOREIGN KEY (a) REFERENCES test.t1(a)) ENGINE=InnoDB; +RENAME TABLE best.t2 TO test.t2; +ERROR 42S01: Table 't2' already exists +SHOW CREATE TABLE best.t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) NOT NULL, + `b` text, + PRIMARY KEY (`a`), + FULLTEXT KEY `b` (`b`), + CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`a`) REFERENCES `test`.`t1` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +DROP DATABASE best; +# +# MDEV-17541 KILL QUERY during lock wait in FOREIGN KEY check hangs +# +INSERT INTO t1 SET a=1; +BEGIN; +DELETE FROM t1; +INSERT INTO t3 SET a=1; +kill query @id; +ERROR 70100: Query execution was interrupted +DROP TABLE t3,t1; diff --git a/mysql-test/suite/innodb/t/foreign_key.test b/mysql-test/suite/innodb/t/foreign_key.test index 223a1596883..c1a92697dab 100644 --- a/mysql-test/suite/innodb/t/foreign_key.test +++ b/mysql-test/suite/innodb/t/foreign_key.test @@ -23,3 +23,53 @@ SET FOREIGN_KEY_CHECKS=DEFAULT; LOCK TABLE staff WRITE; UNLOCK TABLES; DROP TABLES staff, store; + +SET FOREIGN_KEY_CHECKS=1; + +--echo # +--echo # MDEV-17531 Crash in RENAME TABLE with FOREIGN KEY and FULLTEXT INDEX +--echo # + +--disable_query_log +call mtr.add_suppression("InnoDB: Error; possible reasons:"); +--enable_query_log + +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +CREATE DATABASE best; +CREATE TABLE t3 (a INT PRIMARY KEY, +CONSTRAINT t2_ibfk_1 FOREIGN KEY (a) REFERENCES t1(a)) ENGINE=InnoDB; +CREATE TABLE best.t2 (a INT PRIMARY KEY, b TEXT, FULLTEXT INDEX(b), +FOREIGN KEY (a) REFERENCES test.t1(a)) ENGINE=InnoDB; +--replace_regex /Table '.*t2'/Table 't2'/ +--error ER_TABLE_EXISTS_ERROR +RENAME TABLE best.t2 TO test.t2; +SHOW CREATE TABLE best.t2; +DROP DATABASE best; + +--echo # +--echo # MDEV-17541 KILL QUERY during lock wait in FOREIGN KEY check hangs +--echo # +connect (fk, localhost, root,,); +INSERT INTO t1 SET a=1; +BEGIN; +DELETE FROM t1; + +connection default; +let $ID= `SELECT @id := CONNECTION_ID()`; +send INSERT INTO t3 SET a=1; + +connection fk; +# Check that the above SELECT is blocked +let $wait_condition= + select count(*) = 1 from information_schema.processlist + where state = 'update' and info = 'INSERT INTO t3 SET a=1'; +--source include/wait_condition.inc +let $ignore= `SELECT @id := $ID`; +kill query @id; + +connection default; +--error ER_QUERY_INTERRUPTED +reap; +disconnect fk; + +DROP TABLE t3,t1; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 87d40ee2ba4..c997c9f98ab 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -5154,8 +5154,9 @@ row_rename_table_for_mysql( } } - if ((dict_table_has_fts_index(table) - || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) + if (err == DB_SUCCESS + && (dict_table_has_fts_index(table) + || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) && !dict_tables_have_same_db(old_name, new_name)) { err = fts_rename_aux_tables(table, new_name, trx); if (err != DB_TABLE_NOT_FOUND) { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 44e358aab89..8559b352ec9 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -5180,8 +5180,9 @@ row_rename_table_for_mysql( } } - if ((dict_table_has_fts_index(table) - || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) + if (err == DB_SUCCESS + && (dict_table_has_fts_index(table) + || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) && !dict_tables_have_same_db(old_name, new_name)) { err = fts_rename_aux_tables(table, new_name, trx); if (err != DB_TABLE_NOT_FOUND) { From 2549f982891f4598768f10f2aa549861ea7152d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 24 Oct 2018 16:01:18 +0300 Subject: [PATCH 063/127] =?UTF-8?q?MDEV-17532=20Performance=5Fschema=20rep?= =?UTF-8?q?orts=20wrong=20directory=20for=20the=20temporary=20files=20of?= =?UTF-8?q?=20ALTER=20TABLE=E2=80=A6ALGORITHM=3DINPLACE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit row_merge_file_create_low(): Pass the directory of the temporary file to the PSI_FILE_CALL. --- .../innodb/r/alter_inplace_perfschema.result | 15 +++++++ .../innodb/t/alter_inplace_perfschema.opt | 2 + .../innodb/t/alter_inplace_perfschema.test | 40 +++++++++++++++++++ storage/innobase/row/row0merge.cc | 11 ++++- storage/xtradb/row/row0merge.cc | 11 ++++- 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/innodb/r/alter_inplace_perfschema.result create mode 100644 mysql-test/suite/innodb/t/alter_inplace_perfschema.opt create mode 100644 mysql-test/suite/innodb/t/alter_inplace_perfschema.test diff --git a/mysql-test/suite/innodb/r/alter_inplace_perfschema.result b/mysql-test/suite/innodb/r/alter_inplace_perfschema.result new file mode 100644 index 00000000000..38c8ca2f553 --- /dev/null +++ b/mysql-test/suite/innodb/r/alter_inplace_perfschema.result @@ -0,0 +1,15 @@ +update performance_schema.setup_instruments set enabled='yes'; +update performance_schema.setup_consumers set enabled='yes'; +CREATE TABLE t1 (a serial, b varchar(255)) ENGINE=InnoDB; +BEGIN; +COMMIT; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL go WAIT_FOR gone'; +ALTER TABLE t1 ADD INDEX(b), ALGORITHM=INPLACE; +SET DEBUG_SYNC = 'now WAIT_FOR go'; +SELECT DISTINCT object_name FROM performance_schema.events_waits_history_long +WHERE event_name LIKE '%wait%io%file%innodb%innodb_temp_file%'; +object_name +tmp/Innodb Merge Temp File +SET DEBUG_SYNC = 'now SIGNAL gone'; +SET DEBUG_SYNC = 'RESET'; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/alter_inplace_perfschema.opt b/mysql-test/suite/innodb/t/alter_inplace_perfschema.opt new file mode 100644 index 00000000000..f56125521fc --- /dev/null +++ b/mysql-test/suite/innodb/t/alter_inplace_perfschema.opt @@ -0,0 +1,2 @@ +--innodb-sort-buffer-size=64k +--tmpdir=$MYSQLTEST_VARDIR/tmp diff --git a/mysql-test/suite/innodb/t/alter_inplace_perfschema.test b/mysql-test/suite/innodb/t/alter_inplace_perfschema.test new file mode 100644 index 00000000000..5f1e1f3491d --- /dev/null +++ b/mysql-test/suite/innodb/t/alter_inplace_perfschema.test @@ -0,0 +1,40 @@ +--source include/have_innodb.inc +--source include/have_perfschema.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/not_embedded.inc + +connect (ddl, localhost, root,,); +update performance_schema.setup_instruments set enabled='yes'; +update performance_schema.setup_consumers set enabled='yes'; +CREATE TABLE t1 (a serial, b varchar(255)) ENGINE=InnoDB; + +BEGIN; +let $n=10; +--disable_query_log +while ($n) { +dec $n; +INSERT INTO t1 SELECT NULL, REPEAT('b',255); +} +--enable_query_log +COMMIT; + +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL go WAIT_FOR gone'; +send ALTER TABLE t1 ADD INDEX(b), ALGORITHM=INPLACE; + +connection default; +SET DEBUG_SYNC = 'now WAIT_FOR go'; +--replace_regex /.*[\\\/]tmp/tmp/ +SELECT DISTINCT object_name FROM performance_schema.events_waits_history_long +WHERE event_name LIKE '%wait%io%file%innodb%innodb_temp_file%'; + +#--exec lsof -p `pidof mysqld` +SET DEBUG_SYNC = 'now SIGNAL gone'; + +connection ddl; +reap; +disconnect ddl; + +connection default; +SET DEBUG_SYNC = 'RESET'; +DROP TABLE t1; diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 4090e388610..ec9b8f79e49 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3139,9 +3139,17 @@ row_merge_file_create_low( performance schema */ struct PSI_file_locker* locker = NULL; PSI_file_locker_state state; + if (!path) { + path = mysql_tmpdir; + } + static const char label[] = "/Innodb Merge Temp File"; + char* name = static_cast( + ut_malloc(strlen(path) + sizeof label)); + strcpy(name, path); + strcat(name, label); locker = PSI_FILE_CALL(get_thread_file_name_locker)( &state, innodb_file_temp_key, PSI_FILE_OPEN, - "Innodb Merge Temp File", &locker); + path ? name : label, &locker); if (locker != NULL) { PSI_FILE_CALL(start_file_open_wait)(locker, __FILE__, @@ -3154,6 +3162,7 @@ row_merge_file_create_low( PSI_FILE_CALL(end_file_open_wait_and_bind_to_descriptor)( locker, fd); } + ut_free(name); #endif if (fd < 0) { diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index e15b87b8ed5..603deb4b27e 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -3143,9 +3143,17 @@ row_merge_file_create_low( performance schema */ struct PSI_file_locker* locker = NULL; PSI_file_locker_state state; + if (!path) { + path = mysql_tmpdir; + } + static const char label[] = "/Innodb Merge Temp File"; + char* name = static_cast( + ut_malloc(strlen(path) + sizeof label)); + strcpy(name, path); + strcat(name, label); locker = PSI_FILE_CALL(get_thread_file_name_locker)( &state, innodb_file_temp_key, PSI_FILE_OPEN, - "Innodb Merge Temp File", &locker); + path ? name : label, &locker); if (locker != NULL) { PSI_FILE_CALL(start_file_open_wait)(locker, __FILE__, @@ -3158,6 +3166,7 @@ row_merge_file_create_low( PSI_FILE_CALL(end_file_open_wait_and_bind_to_descriptor)( locker, fd); } + ut_free(name); #endif if (fd < 0) { From 30c3d6db328d73f1b30be537aceb55d37936fdb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 24 Oct 2018 16:28:46 +0300 Subject: [PATCH 064/127] MDEV-17533 Merge new release of InnoDB 5.6.42 to 10.0 Also, add a test for a bug that does not seem to affect MariaDB. --- .../suite/innodb/r/innodb-alter-debug.result | 14 +++++++++++ .../suite/innodb/t/innodb-alter-debug.test | 25 ++++++++++++++++++- storage/innobase/include/univ.i | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-alter-debug.result b/mysql-test/suite/innodb/r/innodb-alter-debug.result index 78976030ac8..d888fb5b034 100644 --- a/mysql-test/suite/innodb/r/innodb-alter-debug.result +++ b/mysql-test/suite/innodb/r/innodb-alter-debug.result @@ -54,3 +54,17 @@ SET DEBUG_SYNC = 'now SIGNAL s2'; ERROR 23000: Duplicate entry '1' for key 'uk' SET DEBUG_SYNC = 'RESET'; drop table t1; +# +# Bug #27753193 ASSERTION `PREBUILT->TRX->ERROR_KEY_NUM < +# HA_ALTER_INFO->KEY_COUNT' +CREATE TABLE t1 (a INT, UNIQUE KEY(a)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +SET DEBUG_SYNC = 'row_log_table_apply1_before signal S1 WAIT_FOR S2'; +ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; +SET DEBUG_SYNC = 'now WAIT_FOR S1'; +INSERT INTO t1 VALUES (1); +ERROR 23000: Duplicate entry '1' for key 'a' +SET DEBUG_SYNC = 'now SIGNAL S2'; +ERROR 23000: Duplicate entry '1' for key 'a' +SET DEBUG_SYNC='RESET'; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-alter-debug.test b/mysql-test/suite/innodb/t/innodb-alter-debug.test index f4996916e9f..bc4b2ad8e56 100644 --- a/mysql-test/suite/innodb/t/innodb-alter-debug.test +++ b/mysql-test/suite/innodb/t/innodb-alter-debug.test @@ -64,7 +64,6 @@ set DEBUG_SYNC = 'now WAIT_FOR s1'; --error ER_DUP_ENTRY update t1 set a=1 where id=2; SET DEBUG_SYNC = 'now SIGNAL s2'; -disconnect con1; --echo /* connection default */ connection default; @@ -75,5 +74,29 @@ SET DEBUG_SYNC = 'RESET'; drop table t1; +--echo # +--echo # Bug #27753193 ASSERTION `PREBUILT->TRX->ERROR_KEY_NUM < +--echo # HA_ALTER_INFO->KEY_COUNT' + +CREATE TABLE t1 (a INT, UNIQUE KEY(a)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +SET DEBUG_SYNC = 'row_log_table_apply1_before signal S1 WAIT_FOR S2'; +send ALTER TABLE t1 FORCE, ALGORITHM=INPLACE; + +connection con1; +SET DEBUG_SYNC = 'now WAIT_FOR S1'; +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (1); +SET DEBUG_SYNC = 'now SIGNAL S2'; +disconnect con1; + +CONNECTION default; +--error ER_DUP_ENTRY +reap; +SET DEBUG_SYNC='RESET'; + +DROP TABLE t1; + # Wait till all disconnects are completed --source include/wait_until_count_sessions.inc diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 8cd3f85f835..a9d886607c0 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -45,7 +45,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 41 +#define INNODB_VERSION_BUGFIX 42 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; From 3e47b41a16b5e2212290e719dc101ea46de8e400 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 25 Oct 2018 12:07:23 +0400 Subject: [PATCH 065/127] MDEV-17542 10.3: gcc-8.0 produces lots of -Wclass-memaccess warnings in Table_scope_and_contents_source_st --- sql/handler.cc | 2 +- sql/handler.h | 47 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 58253b90594..897d468f2ba 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6954,7 +6954,7 @@ bool Vers_parse_info::fix_implicit(THD *thd, Alter_info *alter_info) return false; } -bool Table_scope_and_contents_source_st::vers_native(THD *thd) const +bool Table_scope_and_contents_source_pod_st::vers_native(THD *thd) const { if (ha_check_storage_engine_flag(db_type, HTON_NATIVE_SYS_VERSIONING)) return true; diff --git a/sql/handler.h b/sql/handler.h index 8f31f741eab..d2d49b4601c 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1906,6 +1906,8 @@ enum vers_sys_type_t VERS_TRX_ID }; +extern const LEX_CSTRING null_clex_str; + struct Vers_parse_info { Vers_parse_info() : @@ -1914,6 +1916,15 @@ struct Vers_parse_info unversioned_fields(false) {} + void init() // Deep initialization + { + system_time= start_end_t(null_clex_str, null_clex_str); + as_row= start_end_t(null_clex_str, null_clex_str); + check_unit= VERS_UNDEFINED; + versioned_fields= false; + unversioned_fields= false; + } + struct start_end_t { start_end_t() @@ -1993,7 +2004,7 @@ public: - [AS] SELECT ... // Copy structure from a subquery */ -struct Table_scope_and_contents_source_st +struct Table_scope_and_contents_source_pod_st // For trivial members { CHARSET_INFO *table_charset; LEX_CUSTRING tabledef_version; @@ -2019,7 +2030,6 @@ struct Table_scope_and_contents_source_st uint options; /* OR of HA_CREATE_ options */ uint merge_insert_method; uint extra_size; /* length of extra data segment */ - SQL_I_List merge_list; handlerton *db_type; /** Row type of the table definition. @@ -2053,15 +2063,6 @@ struct Table_scope_and_contents_source_st bool table_was_deleted; sequence_definition *seq_create_info; - Vers_parse_info vers_info; - - bool vers_fix_system_fields(THD *thd, Alter_info *alter_info, - const TABLE_LIST &create_table, - bool create_select= false); - - bool vers_check_system_fields(THD *thd, Alter_info *alter_info, - const TABLE_LIST &create_table); - bool vers_native(THD *thd) const; void init() @@ -2082,6 +2083,30 @@ struct Table_scope_and_contents_source_st }; +struct Table_scope_and_contents_source_st: + public Table_scope_and_contents_source_pod_st +{ + SQL_I_List merge_list; + + Vers_parse_info vers_info; + + void init() + { + Table_scope_and_contents_source_pod_st::init(); + merge_list.empty(); + vers_info.init(); + } + + bool vers_fix_system_fields(THD *thd, Alter_info *alter_info, + const TABLE_LIST &create_table, + bool create_select= false); + + bool vers_check_system_fields(THD *thd, Alter_info *alter_info, + const TABLE_LIST &create_table); + +}; + + /** This struct is passed to handler table routines, e.g. ha_create(). It does not include the "OR REPLACE" and "IF NOT EXISTS" parts, as these From 7e9728a913989936b31b6182cf10ffe107612bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 15:04:37 +0300 Subject: [PATCH 066/127] MDEV-17545 Predicate lock for SPATIAL INDEX should lock non-matching record rtr_pcur_getnext_from_path(): Remove a bogus condition. The predicate lock should be acquired also on no match, to ensure that the locking read will be repeatable. This is based on the following fix in MySQL 5.7.24: commit 365111c590082984dbae42e1d1da28ac3f7fb5bd Author: Jimmy Yang Date: Wed Jun 6 16:23:00 2018 -0700 Fix Bug 27577612 - CONCURRENT SERIALIZABLE TRANSACTIONS CAN INSERT INTO AN AREA SELECTED FOR UPDATE Backport fix to mysql-5.7 Reviewed-by: Allen Lai No test case is added, because the MySQL 5.7 test case would pass even when the fix is not present. We would need a test case that only causes a locking conflict on the spatial index. --- storage/innobase/gis/gis0sea.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 9167cf9511b..5f8657238b0 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -389,8 +389,7 @@ rtr_pcur_getnext_from_path( if (mode != PAGE_CUR_RTREE_INSERT && mode != PAGE_CUR_RTREE_LOCATE && mode >= PAGE_CUR_CONTAIN - && btr_cur->rtr_info->need_prdt_lock - && found) { + && btr_cur->rtr_info->need_prdt_lock) { lock_prdt_t prdt; trx_t* trx = thr_get_trx( From 8c9c583aff8603cb6b89c36c6d6efc9910960201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 15:31:13 +0300 Subject: [PATCH 067/127] Remove a deprecation warning --- mysql-test/suite/innodb_gis/r/rtree_concurrent_srch.result | 4 +--- mysql-test/suite/innodb_gis/t/rtree_concurrent_srch.test | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb_gis/r/rtree_concurrent_srch.result b/mysql-test/suite/innodb_gis/r/rtree_concurrent_srch.result index 1b52059fff9..9c5db3e06aa 100644 --- a/mysql-test/suite/innodb_gis/r/rtree_concurrent_srch.result +++ b/mysql-test/suite/innodb_gis/r/rtree_concurrent_srch.result @@ -260,9 +260,7 @@ INSERT INTO t1 VALUES (1, ST_GeomFromText('LineString(6 6, 190 190)')); INSERT INTO t1 VALUES (1, ST_GeomFromText('LineString(7 7, 200 200)')); INSERT INTO t1 VALUES (1, ST_GeomFromText('LineString(8 8, 210 210)')); connect a,localhost,root,,; -SET SESSION debug="+d,rtr_pcur_move_to_next_return"; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET SESSION debug_dbug='+d,rtr_pcur_move_to_next_return'; set transaction isolation level serializable; start transaction; set @g1 = ST_GeomFromText('Polygon((100 100, 100 110, 110 110, 110 100, 100 100))'); diff --git a/mysql-test/suite/innodb_gis/t/rtree_concurrent_srch.test b/mysql-test/suite/innodb_gis/t/rtree_concurrent_srch.test index c10ce368b11..43119415d47 100644 --- a/mysql-test/suite/innodb_gis/t/rtree_concurrent_srch.test +++ b/mysql-test/suite/innodb_gis/t/rtree_concurrent_srch.test @@ -307,7 +307,7 @@ INSERT INTO t1 VALUES (1, ST_GeomFromText('LineString(7 7, 200 200)')); INSERT INTO t1 VALUES (1, ST_GeomFromText('LineString(8 8, 210 210)')); connect (a,localhost,root,,); -SET SESSION debug="+d,rtr_pcur_move_to_next_return"; +SET SESSION debug_dbug='+d,rtr_pcur_move_to_next_return'; set transaction isolation level serializable; start transaction; From e548d92b23a6b9582dec37c26461ddceec4589ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 16:11:52 +0300 Subject: [PATCH 068/127] MDEV-17546 SPATIAL INDEX should not be allowed for FOREIGN KEY dict_foreign_qualify_index(): Reject both FULLTEXT and SPATIAL indexes. Remove these checks from the callers. It is unclear whether this bug affected MariaDB Server. FOREIGN KEY constraints on geometry column types could have been rejected due to column type restrictions. --- storage/innobase/dict/dict0dict.cc | 6 ++++-- storage/innobase/handler/handler0alter.cc | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 57cc98593ad..b241192e286 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -3357,8 +3357,6 @@ dict_foreign_find_index( while (index != NULL) { if (types_idx != index - && !(index->type & DICT_FTS) - && !dict_index_is_spatial(index) && !index->to_be_dropped && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( @@ -6881,6 +6879,10 @@ dict_foreign_qualify_index( return(false); } + if (index->type & (DICT_SPATIAL | DICT_FTS)) { + return false; + } + for (ulint i = 0; i < n_cols; i++) { dict_field_t* field; const char* col_name; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index c55850d68cb..82256b1738b 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1343,8 +1343,7 @@ innobase_find_fk_index( index = dict_table_get_first_index(table); while (index != NULL) { - if (!(index->type & DICT_FTS) - && dict_foreign_qualify_index( + if (dict_foreign_qualify_index( table, col_names, columns, n_cols, index, NULL, true, 0, NULL, NULL, NULL)) { From 31366c6c93ef6b3fb182c0984e03bd9b2d2a8d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 17:06:18 +0300 Subject: [PATCH 069/127] MDEV-17548 Incorrect access to off-page column for indexed virtual column row_build_index_entry_low(): ext does not contain virtual columns. row_upd_store_v_row(): Copy virtual column values This is based on the following fix in MySQL 5.7.24: commit 4ec2158bec73f1582501c4b3e3de250fed9edc9a Author: Sachin Agarwal Date: Fri Aug 24 14:44:13 2018 +0530 Bug #27968952 INNODB CRASH/CORRUPTION WITH TEXT PREFIX INDEXES Problem: There are two problems: 1. If there is one secondary index on extenally stored column and another seconday index on virtual column (whose base column is not externally stored). then while updating seconday index on vitrual column, virtual column data is replaced by externally stoared column. 2. In row update operation, node->row contains shallow copy of virtual data fields. While building an update vector containing all the fields to be modified, compute virtual column. which may causes change in virtual data fields in node->row. In both the above cases, while updating seconday index on virtual column, couldn't find the row and hit an explicite assert inside ROW_NOT_FOUND. Fix: 1. Added check if column is virtual then its ext flag should be ZERO and virtual column data will not be replaced by offset column data. 2. Deep copy of virtual data fields for node->row. RB: #20382 Reviewed by : Jimmy.Yang@oracle.com --- .../suite/gcol/r/innodb_virtual_index.result | 32 +++++++++++++++++++ .../suite/gcol/t/innodb_virtual_index.test | 31 ++++++++++++++++++ storage/innobase/row/row0row.cc | 4 +-- storage/innobase/row/row0upd.cc | 2 ++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/gcol/r/innodb_virtual_index.result b/mysql-test/suite/gcol/r/innodb_virtual_index.result index a1faac8e1e8..1e3b4127f8f 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_index.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_index.result @@ -211,3 +211,35 @@ t1 CREATE TABLE `t1` ( KEY `n` (`col2`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DROP TABLE t1; +# +# Bug #27968952 INNODB CRASH/CORRUPTION WITH TEXT PREFIX INDEXES +# +CREATE TABLE t1( +a INT NOT NULL UNIQUE, +b INT NOT NULL, +c TEXT GENERATED ALWAYS AS (a <> b) VIRTUAL, +d TEXT NOT NULL, +UNIQUE KEY (c(1)), KEY(d(1)) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +SET @t = REPEAT('t',@@innodb_page_size); +INSERT INTO t1 (a,b,d) VALUES (1,0,@t), (0,0,@t); +UPDATE t1 SET b = a; +ERROR 23000: Duplicate entry '0' for key 'c' +REPLACE INTO t1 SET a = 0, b = 1, d = 'd'; +SELECT * FROM t1; +a b c d +0 1 1 d +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +CREATE TABLE t1( +a VARCHAR(1000) GENERATED ALWAYS AS ('1') VIRTUAL, +b VARCHAR(1000) NOT NULL, +c VARCHAR(1000) GENERATED ALWAYS AS (b) STORED, +KEY (b(1)), +KEY (a(1)) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +INSERT INTO t1(b) VALUES(REPEAT('b',1000)); +DELETE FROM t1; +DROP TABLE t1; diff --git a/mysql-test/suite/gcol/t/innodb_virtual_index.test b/mysql-test/suite/gcol/t/innodb_virtual_index.test index 6604a6d94f4..8f4e09fdf31 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_index.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_index.test @@ -232,3 +232,34 @@ CREATE TABLE t1 (col1 int(10)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ALTER TABLE t1 ADD col2 char(21) AS (col1 * col1), ADD INDEX n (col2); SHOW CREATE TABLE t1; DROP TABLE t1; + +--echo # +--echo # Bug #27968952 INNODB CRASH/CORRUPTION WITH TEXT PREFIX INDEXES +--echo # + +CREATE TABLE t1( + a INT NOT NULL UNIQUE, + b INT NOT NULL, + c TEXT GENERATED ALWAYS AS (a <> b) VIRTUAL, + d TEXT NOT NULL, + UNIQUE KEY (c(1)), KEY(d(1)) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +SET @t = REPEAT('t',@@innodb_page_size); +INSERT INTO t1 (a,b,d) VALUES (1,0,@t), (0,0,@t); +--error ER_DUP_ENTRY +UPDATE t1 SET b = a; +REPLACE INTO t1 SET a = 0, b = 1, d = 'd'; +SELECT * FROM t1; +CHECK TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1( + a VARCHAR(1000) GENERATED ALWAYS AS ('1') VIRTUAL, + b VARCHAR(1000) NOT NULL, + c VARCHAR(1000) GENERATED ALWAYS AS (b) STORED, + KEY (b(1)), + KEY (a(1)) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +INSERT INTO t1(b) VALUES(REPEAT('b',1000)); +DELETE FROM t1; +DROP TABLE t1; diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index ad4991fb4e9..4c88130334e 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -295,7 +295,7 @@ row_build_index_entry_low( stored off-page. */ ut_ad(col->ord_part); - if (ext) { + if (ext && !col->is_virtual()) { /* See if the column is stored externally. */ const byte* buf = row_ext_lookup(ext, col_no, &len); diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 171e06894ca..28658428e98 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2157,6 +2157,7 @@ row_upd_store_v_row( } dfield_copy_data(dfield, upd_field->old_v_val); + dfield_dup(dfield, node->heap); break; } @@ -2177,6 +2178,7 @@ row_upd_store_v_row( update->old_vrow, col_no); dfield_copy_data(dfield, vfield); + dfield_dup(dfield, node->heap); } } else { /* Need to compute, this happens when From 2abf2648a6124c2fba0ff4fc46b1604c2fdf32ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 25 Oct 2018 17:09:54 +0300 Subject: [PATCH 070/127] MDEV-17536 Merge new release of InnoDB 5.7.24 to 10.2 Update the InnoDB version number to 5.7.24. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/include/univ.i | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 995f21dccf4..451e5224329 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -3074,7 +3074,7 @@ READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL -GLOBAL_VALUE 5.7.23 +GLOBAL_VALUE 5.7.24 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 453f15b8784..47d65052f16 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -41,7 +41,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 7 -#define INNODB_VERSION_BUGFIX 23 +#define INNODB_VERSION_BUGFIX 24 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; From 14be81438098923e95bda6d635d638991e60116b Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Thu, 18 Oct 2018 18:23:12 +0300 Subject: [PATCH 071/127] MDEV-17491 micro optimize page_id_t page_id_t: remove m_fold member various places: pass page_id_t by value instead of by reference --- storage/innobase/btr/btr0btr.cc | 6 +- storage/innobase/btr/btr0cur.cc | 8 +- storage/innobase/btr/btr0sea.cc | 2 +- storage/innobase/buf/buf0buf.cc | 57 +++++-------- storage/innobase/buf/buf0flu.cc | 4 +- storage/innobase/buf/buf0lru.cc | 3 +- storage/innobase/buf/buf0rea.cc | 10 +-- storage/innobase/fil/fil0fil.cc | 30 ++++++- storage/innobase/fsp/fsp0fsp.cc | 16 ++-- storage/innobase/ibuf/ibuf0ibuf.cc | 28 +++---- storage/innobase/include/btr0btr.h | 8 +- storage/innobase/include/btr0btr.ic | 4 +- storage/innobase/include/btr0cur.h | 2 +- storage/innobase/include/btr0sea.h | 2 +- storage/innobase/include/buf0buf.h | 116 +++++++------------------- storage/innobase/include/buf0buf.ic | 26 +++--- storage/innobase/include/buf0rea.h | 8 +- storage/innobase/include/fil0fil.h | 32 ++----- storage/innobase/include/fsp0fsp.h | 2 +- storage/innobase/include/fsp0fsp.ic | 2 +- storage/innobase/include/ibuf0ibuf.h | 12 ++- storage/innobase/include/ibuf0ibuf.ic | 2 +- storage/innobase/include/page0page.h | 2 +- storage/innobase/include/trx0sys.h | 5 +- storage/innobase/include/trx0sys.ic | 5 +- storage/innobase/include/trx0undo.h | 4 +- storage/innobase/include/trx0undo.ic | 4 +- storage/innobase/log/log0recv.cc | 5 +- storage/innobase/page/page0page.cc | 2 +- storage/innobase/row/row0import.cc | 5 +- storage/innobase/srv/srv0start.cc | 8 +- 31 files changed, 169 insertions(+), 251 deletions(-) diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index fcb3e24b42f..696d493e078 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -1149,7 +1149,7 @@ btr_free_root_invalidate( static MY_ATTRIBUTE((warn_unused_result)) buf_block_t* btr_free_root_check( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, index_id_t index_id, mtr_t* mtr) @@ -1421,7 +1421,7 @@ top_loop: @param[in,out] mtr mini-transaction */ void btr_free_if_exists( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, index_id_t index_id, mtr_t* mtr) @@ -1445,7 +1445,7 @@ btr_free_if_exists( @param[in] page_size page size */ void btr_free( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { mtr_t mtr; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 5af0a8b899a..373328939c2 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -231,7 +231,7 @@ btr_rec_free_externally_stored_fields( btr_latch_leaves_t btr_cur_latch_leaves( buf_block_t* block, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint latch_mode, btr_cur_t* cursor, @@ -1684,7 +1684,7 @@ need_opposite_intention: lock_intention = BTR_INTENTION_BOTH; - page_id.reset(space, dict_index_get_page(index)); + page_id = page_id_t(space, dict_index_get_page(index)); up_match = 0; low_match = 0; height = ULINT_UNDEFINED; @@ -1900,7 +1900,7 @@ need_opposite_intention: ulint idx = n_blocks - (leftmost_from_level - 1); - page_id.reset( + page_id = page_id_t( space, tree_blocks[idx]->page.id.page_no()); @@ -1935,7 +1935,7 @@ need_opposite_intention: } /* Go to the child node */ - page_id.reset( + page_id = page_id_t( space, btr_node_ptr_get_child_page_no(node_ptr, offsets)); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index fae929fc8e5..b494565b288 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1304,7 +1304,7 @@ cleanup: /** Drop possible adaptive hash index entries when a page is evicted from the buffer pool or freed in a file, or the index is being dropped. @param[in] page_id page id */ -void btr_search_drop_page_hash_when_freed(const page_id_t& page_id) +void btr_search_drop_page_hash_when_freed(const page_id_t page_id) { buf_block_t* block; mtr_t mtr; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 724fea07ba2..be70a723232 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2280,7 +2280,8 @@ buf_page_realloc( memset(block->frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 0xff, 4); UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); buf_block_set_state(block, BUF_BLOCK_REMOVE_HASH); - block->page.id.reset(ULINT32_UNDEFINED, ULINT32_UNDEFINED); + block->page.id + = page_id_t(ULINT32_UNDEFINED, ULINT32_UNDEFINED); /* Relocate buf_pool->flush_list. */ if (block->page.oldest_modification) { @@ -3505,7 +3506,7 @@ hash_lock and reacquire it. static buf_page_t* buf_pool_watch_set( - const page_id_t& page_id, + const page_id_t page_id, rw_lock_t** hash_lock) { buf_page_t* bpage; @@ -3583,7 +3584,7 @@ page_found: buf_block_t::mutex or buf_pool->zip_mutex or both. */ bpage->state = BUF_BLOCK_ZIP_PAGE; - bpage->id.copy_from(page_id); + bpage->id = page_id; bpage->buf_fix_count = 1; ut_d(bpage->in_page_hash = TRUE); @@ -3647,9 +3648,7 @@ buf_pool_watch_remove( /** Stop watching if the page has been read in. buf_pool_watch_set(same_page_id) must have returned NULL before. @param[in] page_id page id */ -void -buf_pool_watch_unset( - const page_id_t& page_id) +void buf_pool_watch_unset(const page_id_t page_id) { buf_page_t* bpage; buf_pool_t* buf_pool = buf_pool_get(page_id); @@ -3681,12 +3680,10 @@ buf_pool_watch_unset( This may only be called after buf_pool_watch_set(same_page_id) has returned NULL and before invoking buf_pool_watch_unset(same_page_id). @param[in] page_id page id -@return FALSE if the given page was not read in, TRUE if it was */ -ibool -buf_pool_watch_occurred( - const page_id_t& page_id) +@return false if the given page was not read in, true if it was */ +bool buf_pool_watch_occurred(const page_id_t page_id) { - ibool ret; + bool ret; buf_page_t* bpage; buf_pool_t* buf_pool = buf_pool_get(page_id); rw_lock_t* hash_lock = buf_page_hash_lock_get(buf_pool, page_id); @@ -3756,9 +3753,7 @@ debug version to check that it is not accessed any more unless reallocated. @param[in] page_id page id @return control block if found in page hash table, otherwise NULL */ -buf_page_t* -buf_page_set_file_page_was_freed( - const page_id_t& page_id) +buf_page_t* buf_page_set_file_page_was_freed(const page_id_t page_id) { buf_page_t* bpage; buf_pool_t* buf_pool = buf_pool_get(page_id); @@ -3786,9 +3781,7 @@ debug version to check that it is not accessed any more unless reallocated. @param[in] page_id page id @return control block if found in page hash table, otherwise NULL */ -buf_page_t* -buf_page_reset_file_page_was_freed( - const page_id_t& page_id) +buf_page_t* buf_page_reset_file_page_was_freed(const page_id_t page_id) { buf_page_t* bpage; buf_pool_t* buf_pool = buf_pool_get(page_id); @@ -3810,12 +3803,8 @@ buf_page_reset_file_page_was_freed( /** Attempts to discard the uncompressed frame of a compressed page. The caller should not be holding any mutexes when this function is called. -@param[in] page_id page id -@return TRUE if successful, FALSE otherwise. */ -static -void -buf_block_try_discard_uncompressed( - const page_id_t& page_id) +@param[in] page_id page id */ +static void buf_block_try_discard_uncompressed(const page_id_t page_id) { buf_page_t* bpage; buf_pool_t* buf_pool = buf_pool_get(page_id); @@ -3849,7 +3838,7 @@ the same set of mutexes or latches. @return pointer to the block */ buf_page_t* buf_page_get_zip( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { buf_page_t* bpage; @@ -4300,7 +4289,7 @@ BUF_PEEK_IF_IN_POOL, BUF_GET_NO_LATCH, or BUF_GET_IF_IN_POOL_OR_WATCH @return pointer to the block or NULL */ buf_block_t* buf_page_get_gen( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint rw_latch, buf_block_t* guess, @@ -4375,7 +4364,7 @@ loop: it may have been freed by buf_relocate(). */ if (!buf_block_is_uncompressed(buf_pool, block) - || !page_id.equals_to(block->page.id) + || page_id != block->page.id || buf_block_get_state(block) != BUF_BLOCK_FILE_PAGE) { /* Our guess was bogus or things have changed @@ -5206,7 +5195,7 @@ Suitable for using when holding the lock_sys_t::mutex. @return pointer to a page or NULL */ buf_block_t* buf_page_try_get_func( - const page_id_t& page_id, + const page_id_t page_id, const char* file, unsigned line, mtr_t* mtr) @@ -5235,7 +5224,7 @@ buf_page_try_get_func( #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); - ut_a(page_id.equals_to(block->page.id)); + ut_a(page_id == block->page.id); #endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ buf_block_buf_fix_inc(block, file, line); @@ -5321,7 +5310,7 @@ static void buf_page_init( buf_pool_t* buf_pool, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, buf_block_t* block) { @@ -5389,7 +5378,7 @@ buf_page_init( ut_ad(!block->page.in_page_hash); ut_d(block->page.in_page_hash = TRUE); - block->page.id.copy_from(page_id); + block->page.id = page_id; block->page.size.copy_from(page_size); HASH_INSERT(buf_page_t, hash, buf_pool->page_hash, @@ -5419,7 +5408,7 @@ buf_page_t* buf_page_init_for_read( dberr_t* err, ulint mode, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, bool unzip) { @@ -5588,7 +5577,7 @@ buf_page_init_for_read( buf_page_init_low(bpage); bpage->state = BUF_BLOCK_ZIP_PAGE; - bpage->id.copy_from(page_id); + bpage->id = page_id; bpage->flush_observer = NULL; ut_d(bpage->in_page_hash = FALSE); @@ -5657,7 +5646,7 @@ FILE_PAGE (the other is buf_page_get_gen). @return pointer to the block, page bufferfixed */ buf_block_t* buf_page_create( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, mtr_t* mtr) { @@ -7364,7 +7353,7 @@ buf_pool_check_no_pending_io(void) std::ostream& operator<<( std::ostream& out, - const page_id_t& page_id) + const page_id_t page_id) { out << "[page id: space=" << page_id.m_space << ", page number=" << page_id.m_page_no << "]"; diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 75a0498dde5..a73b841b627 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1296,7 +1296,7 @@ buf_flush_page_try( static bool buf_flush_check_neighbor( - const page_id_t& page_id, + const page_id_t page_id, buf_flush_t flush_type) { buf_page_t* bpage; @@ -1346,7 +1346,7 @@ buf_flush_check_neighbor( static ulint buf_flush_try_neighbors( - const page_id_t& page_id, + const page_id_t page_id, buf_flush_t flush_type, ulint n_flushed, ulint n_to_flush) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 9c652687f72..331f6d4a3f2 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -2161,7 +2161,8 @@ buf_LRU_block_free_hashed_page( buf_page_mutex_enter(block); if (buf_pool->flush_rbt == NULL) { - block->page.id.reset(ULINT32_UNDEFINED, ULINT32_UNDEFINED); + block->page.id + = page_id_t(ULINT32_UNDEFINED, ULINT32_UNDEFINED); } buf_block_set_state(block, BUF_BLOCK_MEMORY); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 4c4f6292a53..3d5a319a820 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -117,7 +117,7 @@ buf_read_page_low( bool sync, ulint type, ulint mode, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, bool unzip, bool ignore_missing_space = false) @@ -239,7 +239,7 @@ pages, it may happen that the page at the given page number does not get read even if we return a positive value! */ ulint buf_read_ahead_random( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ibool inside_ibuf) { @@ -419,7 +419,7 @@ after decryption normal page checksum does not match. @retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ dberr_t buf_read_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { ulint count; @@ -457,7 +457,7 @@ released by the i/o-handler thread. @param[in] sync true if synchronous aio is desired */ void buf_read_page_background( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, bool sync) { @@ -529,7 +529,7 @@ which could result in a deadlock if the OS does not support asynchronous io. @return number of page read requests issued */ ulint buf_read_ahead_linear( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ibool inside_ibuf) { diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 368144fe2b2..d08e362af3b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -265,7 +265,7 @@ i/o on a tablespace which does not exist */ UNIV_INLINE dberr_t fil_read( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint byte_offset, ulint len, @@ -291,7 +291,7 @@ i/o on a tablespace which does not exist */ UNIV_INLINE dberr_t fil_write( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint byte_offset, ulint len, @@ -4988,7 +4988,7 @@ dberr_t fil_io( const IORequest& type, bool sync, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint byte_offset, ulint len, @@ -5574,7 +5574,7 @@ Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. @param[in,out] mtr mini-transaction */ void fil_page_reset_type( - const page_id_t& page_id, + const page_id_t page_id, byte* page, ulint type, mtr_t* mtr) @@ -6311,3 +6311,25 @@ fil_space_set_punch_hole( { node->space->punch_hole = val; } + +/** Check (and if needed, reset) the page type. +Data files created before MySQL 5.1 may contain +garbage in the FIL_PAGE_TYPE field. +In MySQL 3.23.53, only undo log pages and index pages were tagged. +Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. +@param[in] page_id page number +@param[in,out] page page with possibly invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ +void +fil_block_check_type( + const buf_block_t& block, + ulint type, + mtr_t* mtr) +{ + ulint page_type = fil_page_get_type(block.frame); + + if (page_type != type) { + fil_page_reset_type(block.page.id, block.frame, type, mtr); + } +} diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index d4d5abeb32f..2b44d7e60cc 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1197,7 +1197,7 @@ fsp_fill_free_list( header, space, i, mtr, init_space, &desc_block); if (desc_block != NULL) { fil_block_check_type( - desc_block, FIL_PAGE_TYPE_XDES, mtr); + *desc_block, FIL_PAGE_TYPE_XDES, mtr); } xdes_init(descr, mtr); @@ -1256,7 +1256,7 @@ fsp_alloc_free_extent( header, space, hint, mtr, false, &desc_block); if (desc_block != NULL) { - fil_block_check_type(desc_block, FIL_PAGE_TYPE_XDES, mtr); + fil_block_check_type(*desc_block, FIL_PAGE_TYPE_XDES, mtr); } if (descr && (xdes_get_state(descr, mtr) == XDES_FREE)) { @@ -1798,7 +1798,7 @@ fsp_alloc_seg_inode( block = buf_page_get(page_id, page_size, RW_SX_LATCH, mtr); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - fil_block_check_type(block, FIL_PAGE_INODE, mtr); + fil_block_check_type(*block, FIL_PAGE_INODE, mtr); page = buf_block_get_frame(block); @@ -2105,7 +2105,7 @@ fseg_create_general( ? FIL_PAGE_TYPE_TRX_SYS : FIL_PAGE_TYPE_SYS; - fil_block_check_type(block, type, mtr); + fil_block_check_type(*block, type, mtr); } if (!has_done_reservation @@ -2690,7 +2690,7 @@ fseg_alloc_free_page_general( const page_size_t page_size(space->flags); inode = fseg_inode_get(seg_header, space_id, page_size, mtr, &iblock); - fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); + fil_block_check_type(*iblock, FIL_PAGE_INODE, mtr); if (!has_done_reservation && !fsp_reserve_free_extents(&n_reserved, space_id, 2, @@ -3173,7 +3173,7 @@ fseg_free_page_func( seg_inode = fseg_inode_get(seg_header, space_id, page_size, mtr, &iblock); - fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); + fil_block_check_type(*iblock, FIL_PAGE_INODE, mtr); fseg_free_page_low(seg_inode, space, page, page_size, ahi, mtr); @@ -3353,7 +3353,7 @@ fseg_free_step_func( DBUG_RETURN(TRUE); } - fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); + fil_block_check_type(*iblock, FIL_PAGE_INODE, mtr); descr = fseg_get_first_extent(inode, space, page_size, mtr); if (descr != NULL) { @@ -3421,7 +3421,7 @@ fseg_free_step_not_header_func( buf_block_t* iblock; inode = fseg_inode_get(header, space_id, page_size, mtr, &iblock); - fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); + fil_block_check_type(*iblock, FIL_PAGE_INODE, mtr); descr = fseg_get_first_extent(inode, space, page_size, mtr); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index ca1b10932db..b45333eba5a 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -212,7 +212,7 @@ static ulint ibuf_counts[IBUF_COUNT_N_SPACES][IBUF_COUNT_N_PAGES]; UNIV_INLINE void ibuf_count_check( - const page_id_t& page_id) + const page_id_t page_id) { if (page_id.space() < IBUF_COUNT_N_SPACES && page_id.page_no() < IBUF_COUNT_N_PAGES) { @@ -423,9 +423,7 @@ ibuf_tree_root_get( @param[in] page_id page id @return number of entries in the insert buffer currently buffered for this page */ -ulint -ibuf_count_get( - const page_id_t& page_id) +ulint ibuf_count_get(const page_id_t page_id) { ibuf_count_check(page_id); @@ -438,7 +436,7 @@ ibuf_count_get( static void ibuf_count_set( - const page_id_t& page_id, + const page_id_t page_id, ulint val) { ibuf_count_check(page_id); @@ -674,7 +672,7 @@ UNIV_INLINE ulint ibuf_bitmap_page_get_bits_low( const page_t* page, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, #ifdef UNIV_DEBUG ulint latch_type, @@ -725,7 +723,7 @@ static void ibuf_bitmap_page_set_bits( page_t* page, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint bit, ulint val, @@ -778,7 +776,7 @@ ibuf_bitmap_page_set_bits( UNIV_INLINE const page_id_t ibuf_bitmap_page_no_calc( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { ulint bitmap_page_no; @@ -802,7 +800,7 @@ is x-latched */ static page_t* ibuf_bitmap_get_map_page_func( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, const char* file, unsigned line, @@ -1105,7 +1103,7 @@ ibuf_update_free_bits_for_two_pages_low( UNIV_INLINE ibool ibuf_fixed_addr_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { return((page_id.space() == IBUF_SPACE_ID @@ -1127,7 +1125,7 @@ in which case a new transaction is created. @return TRUE if level 2 or level 3 page */ ibool ibuf_page_low( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, #ifdef UNIV_DEBUG ibool x_latch, @@ -3360,7 +3358,7 @@ ibuf_insert_low( const dtuple_t* entry, ulint entry_size, dict_index_t* index, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, que_thr_t* thr) { @@ -3700,7 +3698,7 @@ ibuf_insert( ibuf_op_t op, const dtuple_t* entry, dict_index_t* index, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, que_thr_t* thr) { @@ -4436,7 +4434,7 @@ want to update a non-existent bitmap page */ void ibuf_merge_or_delete_for_page( buf_block_t* block, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t* page_size, ibool update_ibuf_bitmap) { @@ -4454,7 +4452,7 @@ ibuf_merge_or_delete_for_page( ulint mops[IBUF_OP_COUNT]; ulint dops[IBUF_OP_COUNT]; - ut_ad(block == NULL || page_id.equals_to(block->page.id)); + ut_ad(block == NULL || page_id == block->page.id); ut_ad(block == NULL || buf_block_get_io_fix(block) == BUF_IO_READ); if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 2fccdfc431c..dc933bef6c4 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -231,7 +231,7 @@ tree UNIV_INLINE buf_block_t* btr_block_get_func( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint mode, const char* file, @@ -272,7 +272,7 @@ UNIV_INLINE page_t* btr_page_get( /*=========*/ - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint mode, dict_index_t* index, @@ -371,7 +371,7 @@ btr_create( @param[in,out] mtr mini-transaction */ void btr_free_if_exists( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, index_id_t index_id, mtr_t* mtr); @@ -381,7 +381,7 @@ btr_free_if_exists( @param[in] page_size page size */ void btr_free( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size); /** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC. diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index bd4f2a40267..d68f74170c2 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -47,7 +47,7 @@ tree UNIV_INLINE buf_block_t* btr_block_get_func( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint mode, const char* file, @@ -113,7 +113,7 @@ UNIV_INLINE page_t* btr_page_get( /*=========*/ - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint mode, dict_index_t* index, diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 1df382bb995..6d3e029998c 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -795,7 +795,7 @@ btr_rec_set_deleted_flag( btr_latch_leaves_t btr_cur_latch_leaves( buf_block_t* block, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint latch_mode, btr_cur_t* cursor, diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index e6983cacffb..bacaca583c0 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -133,7 +133,7 @@ btr_search_drop_page_hash_index(buf_block_t* block); /** Drop possible adaptive hash index entries when a page is evicted from the buffer pool or freed in a file, or the index is being dropped. @param[in] page_id page id */ -void btr_search_drop_page_hash_when_freed(const page_id_t& page_id); +void btr_search_drop_page_hash_when_freed(const page_id_t page_id); /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index a79b39235f3..b8bde3bd776 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -223,25 +223,29 @@ public: @param[in] space tablespace id @param[in] page_no page number */ page_id_t(ulint space, ulint page_no) - : - m_space(static_cast(space)), - m_page_no(static_cast(page_no)), - m_fold(ULINT_UNDEFINED) + : m_space(static_cast(space)), + m_page_no(static_cast(page_no)) { ut_ad(space <= 0xFFFFFFFFU); ut_ad(page_no <= 0xFFFFFFFFU); } + bool operator==(const page_id_t& rhs) const + { + return m_space == rhs.m_space && m_page_no == rhs.m_page_no; + } + bool operator!=(const page_id_t& rhs) const { return !(*this == rhs); } + /** Retrieve the tablespace id. @return tablespace id */ - inline ib_uint32_t space() const + inline uint32_t space() const { return(m_space); } /** Retrieve the page number. @return page number */ - inline ib_uint32_t page_no() const + inline uint32_t page_no() const { return(m_page_no); } @@ -250,69 +254,25 @@ public: @return fold value */ inline ulint fold() const { - /* Initialize m_fold if it has not been initialized yet. */ - if (m_fold == ULINT_UNDEFINED) { - m_fold = (m_space << 20) + m_space + m_page_no; - ut_ad(m_fold != ULINT_UNDEFINED); - } - - return(m_fold); - } - - /** Copy the values from a given page_id_t object. - @param[in] src page id object whose values to fetch */ - inline void copy_from(const page_id_t& src) - { - m_space = src.space(); - m_page_no = src.page_no(); - m_fold = src.fold(); - } - - /** Reset the values from a (space, page_no). - @param[in] space tablespace id - @param[in] page_no page number */ - inline void reset(ulint space, ulint page_no) - { - m_space = static_cast(space); - m_page_no = static_cast(page_no); - m_fold = ULINT_UNDEFINED; - - ut_ad(space <= 0xFFFFFFFFU); - ut_ad(page_no <= 0xFFFFFFFFU); + return (m_space << 20) + m_space + m_page_no; } /** Reset the page number only. @param[in] page_no page number */ inline void set_page_no(ulint page_no) { - m_page_no = static_cast(page_no); - m_fold = ULINT_UNDEFINED; + m_page_no = static_cast(page_no); ut_ad(page_no <= 0xFFFFFFFFU); } - /** Check if a given page_id_t object is equal to the current one. - @param[in] a page_id_t object to compare - @return true if equal */ - inline bool equals_to(const page_id_t& a) const - { - return(a.space() == m_space && a.page_no() == m_page_no); - } - private: /** Tablespace id. */ - ib_uint32_t m_space; + uint32_t m_space; /** Page number. */ - ib_uint32_t m_page_no; - - /** A fold value derived from m_space and m_page_no, - used in hashing. */ - mutable ulint m_fold; - - /* Disable implicit copying. */ - void operator=(const page_id_t&); + uint32_t m_page_no; /** Declare the overloaded global operator<< as a friend of this class. Refer to the global declaration for further details. Print @@ -324,7 +284,7 @@ private: std::ostream& operator<<( std::ostream& out, - const page_id_t& page_id); + const page_id_t page_id); }; /** Print the given page_id_t object. @@ -334,7 +294,7 @@ private: std::ostream& operator<<( std::ostream& out, - const page_id_t& page_id); + const page_id_t page_id); #ifndef UNIV_INNOCHECKSUM /********************************************************************//** @@ -517,7 +477,7 @@ Suitable for using when holding the lock_sys_t::mutex. @return pointer to a page or NULL */ buf_block_t* buf_page_try_get_func( - const page_id_t& page_id, + const page_id_t page_id, const char* file, unsigned line, mtr_t* mtr); @@ -543,7 +503,7 @@ the same set of mutexes or latches. @return pointer to the block */ buf_page_t* buf_page_get_zip( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size); /** This is the general function used to get access to a database page. @@ -559,7 +519,7 @@ BUF_PEEK_IF_IN_POOL, BUF_GET_NO_LATCH, or BUF_GET_IF_IN_POOL_OR_WATCH @return pointer to the block or NULL */ buf_block_t* buf_page_get_gen( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint rw_latch, buf_block_t* guess, @@ -579,7 +539,7 @@ FILE_PAGE (the other is buf_page_get_gen). @return pointer to the block, page bufferfixed */ buf_block_t* buf_page_create( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, mtr_t* mtr); @@ -613,10 +573,7 @@ NOTE that it is possible that the page is not yet read from disk, though. @param[in] page_id page id @return TRUE if found in the page hash table */ -UNIV_INLINE -ibool -buf_page_peek( - const page_id_t& page_id); +inline bool buf_page_peek(const page_id_t page_id); #ifdef UNIV_DEBUG @@ -626,9 +583,7 @@ debug version to check that it is not accessed any more unless reallocated. @param[in] page_id page id @return control block if found in page hash table, otherwise NULL */ -buf_page_t* -buf_page_set_file_page_was_freed( - const page_id_t& page_id); +buf_page_t* buf_page_set_file_page_was_freed(const page_id_t page_id); /** Sets file_page_was_freed FALSE if the page is found in the buffer pool. This function should be called when we free a file page and want the @@ -636,9 +591,7 @@ debug version to check that it is not accessed any more unless reallocated. @param[in] page_id page id @return control block if found in page hash table, otherwise NULL */ -buf_page_t* -buf_page_reset_file_page_was_freed( - const page_id_t& page_id); +buf_page_t* buf_page_reset_file_page_was_freed(const page_id_t page_id); #endif /* UNIV_DEBUG */ /********************************************************************//** @@ -1086,7 +1039,7 @@ UNIV_INLINE void buf_block_set_file_page( buf_block_t* block, - const page_id_t& page_id); + const page_id_t page_id); /*********************************************************************//** Gets the io_fix state of a block. @@ -1267,7 +1220,7 @@ buf_page_t* buf_page_init_for_read( dberr_t* err, ulint mode, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, bool unzip); @@ -1315,10 +1268,7 @@ buf_pool_from_block( /** Returns the buffer pool instance given a page id. @param[in] page_id page id @return buffer pool */ -UNIV_INLINE -buf_pool_t* -buf_pool_get( - const page_id_t& page_id); +inline buf_pool_t* buf_pool_get(const page_id_t page_id); /******************************************************************//** Returns the buffer pool instance given its array index @@ -1338,7 +1288,7 @@ UNIV_INLINE buf_page_t* buf_page_hash_get_low( buf_pool_t* buf_pool, - const page_id_t& page_id); + const page_id_t page_id); /** Returns the control block of a file page, NULL if not found. If the block is found and lock is not NULL then the appropriate @@ -1360,7 +1310,7 @@ UNIV_INLINE buf_page_t* buf_page_hash_get_locked( buf_pool_t* buf_pool, - const page_id_t& page_id, + const page_id_t page_id, rw_lock_t** lock, ulint lock_mode, bool watch = false); @@ -1383,7 +1333,7 @@ UNIV_INLINE buf_block_t* buf_block_hash_get_locked( buf_pool_t* buf_pool, - const page_id_t& page_id, + const page_id_t page_id, rw_lock_t** lock, ulint lock_mode); @@ -1423,18 +1373,14 @@ buf_pool_watch_is_sentinel( /** Stop watching if the page has been read in. buf_pool_watch_set(space,offset) must have returned NULL before. @param[in] page_id page id */ -void -buf_pool_watch_unset( - const page_id_t& page_id); +void buf_pool_watch_unset(const page_id_t page_id); /** Check if the page has been read in. This may only be called after buf_pool_watch_set(space,offset) has returned NULL and before invoking buf_pool_watch_unset(space,offset). @param[in] page_id page id @return FALSE if the given page was not read in, TRUE if it was */ -ibool -buf_pool_watch_occurred( - const page_id_t& page_id) +bool buf_pool_watch_occurred(const page_id_t page_id) MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 38c52d5e608..da7bff31821 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -463,10 +463,10 @@ UNIV_INLINE void buf_block_set_file_page( buf_block_t* block, - const page_id_t& page_id) + const page_id_t page_id) { buf_block_set_state(block, BUF_BLOCK_FILE_PAGE); - block->page.id.copy_from(page_id); + block->page.id = page_id; } /*********************************************************************//** @@ -1042,10 +1042,7 @@ buf_block_buf_fix_dec( /** Returns the buffer pool instance given a page id. @param[in] page_id page id @return buffer pool */ -UNIV_INLINE -buf_pool_t* -buf_pool_get( - const page_id_t& page_id) +inline buf_pool_t* buf_pool_get(const page_id_t page_id) { /* 2log of BUF_READ_AHEAD_AREA (64) */ ulint ignored_page_no = page_id.page_no() >> 6; @@ -1080,7 +1077,7 @@ UNIV_INLINE buf_page_t* buf_page_hash_get_low( buf_pool_t* buf_pool, - const page_id_t& page_id) + const page_id_t page_id) { buf_page_t* bpage; @@ -1098,7 +1095,7 @@ buf_page_hash_get_low( bpage, ut_ad(bpage->in_page_hash && !bpage->in_zip_hash && buf_page_in_file(bpage)), - page_id.equals_to(bpage->id)); + page_id == bpage->id); if (bpage) { ut_a(buf_page_in_file(bpage)); ut_ad(bpage->in_page_hash); @@ -1129,7 +1126,7 @@ UNIV_INLINE buf_page_t* buf_page_hash_get_locked( buf_pool_t* buf_pool, - const page_id_t& page_id, + const page_id_t page_id, rw_lock_t** lock, ulint lock_mode, bool watch) @@ -1173,7 +1170,7 @@ buf_page_hash_get_locked( } ut_ad(buf_page_in_file(bpage)); - ut_ad(page_id.equals_to(bpage->id)); + ut_ad(page_id == bpage->id); if (lock == NULL) { /* The caller wants us to release the page_hash lock */ @@ -1212,7 +1209,7 @@ UNIV_INLINE buf_block_t* buf_block_hash_get_locked( buf_pool_t* buf_pool, - const page_id_t& page_id, + const page_id_t page_id, rw_lock_t** lock, ulint lock_mode) { @@ -1252,11 +1249,8 @@ buf_block_hash_get_locked( NOTE that it is possible that the page is not yet read from disk, though. @param[in] page_id page id -@return TRUE if found in the page hash table */ -UNIV_INLINE -ibool -buf_page_peek( - const page_id_t& page_id) +@return true if found in the page hash table */ +inline bool buf_page_peek(const page_id_t page_id) { buf_pool_t* buf_pool = buf_pool_get(page_id); diff --git a/storage/innobase/include/buf0rea.h b/storage/innobase/include/buf0rea.h index 5c766b9412a..a7c43e01467 100644 --- a/storage/innobase/include/buf0rea.h +++ b/storage/innobase/include/buf0rea.h @@ -44,7 +44,7 @@ after decryption normal page checksum does not match. @retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ dberr_t buf_read_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size); /********************************************************************//** @@ -57,7 +57,7 @@ released by the i/o-handler thread. @param[in] sync true if synchronous aio is desired */ void buf_read_page_background( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, bool sync); @@ -79,7 +79,7 @@ pages, it may happen that the page at the given page number does not get read even if we return a positive value! */ ulint buf_read_ahead_random( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ibool inside_ibuf); @@ -111,7 +111,7 @@ which could result in a deadlock if the OS does not support asynchronous io. @return number of page read requests issued */ ulint buf_read_ahead_linear( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ibool inside_ibuf); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index e738931485e..d3ccac2057e 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -40,8 +40,8 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration extern ibool srv_use_doublewrite_buf; extern struct buf_dblwr_t* buf_dblwr; -struct trx_t; class page_id_t; +struct trx_t; class truncate_t; typedef std::list > space_name_list_t; @@ -1244,7 +1244,7 @@ dberr_t fil_io( const IORequest& type, bool sync, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, ulint byte_offset, ulint len, @@ -1325,7 +1325,7 @@ Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. @param[in,out] mtr mini-transaction */ void fil_page_reset_type( - const page_id_t& page_id, + const page_id_t page_id, byte* page, ulint type, mtr_t* mtr); @@ -1349,31 +1349,11 @@ Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. @param[in,out] page page with possibly invalid FIL_PAGE_TYPE @param[in] type expected page type @param[in,out] mtr mini-transaction */ -inline void -fil_page_check_type( - const page_id_t& page_id, - byte* page, +fil_block_check_type( + const buf_block_t& block, ulint type, - mtr_t* mtr) -{ - ulint page_type = fil_page_get_type(page); - - if (page_type != type) { - fil_page_reset_type(page_id, page, type, mtr); - } -} - -/** Check (and if needed, reset) the page type. -Data files created before MySQL 5.1 may contain -garbage in the FIL_PAGE_TYPE field. -In MySQL 3.23.53, only undo log pages and index pages were tagged. -Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in,out] block block with possibly invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ -#define fil_block_check_type(block, type, mtr) \ - fil_page_check_type(block->page.id, block->frame, type, mtr) + mtr_t* mtr); #ifdef UNIV_DEBUG /** Increase redo skipped of a tablespace. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 368f0daa201..7b50f84eb92 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -647,7 +647,7 @@ fseg_free_step_not_header_func( UNIV_INLINE ibool fsp_descr_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size); /***********************************************************//** diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 2da3320eef7..2160287bd1a 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -33,7 +33,7 @@ Created 12/18/1995 Heikki Tuuri UNIV_INLINE ibool fsp_descr_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { return((page_id.page_no() & (page_size.physical() - 1)) diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index 6cff26635bd..8b30f812323 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -251,7 +251,7 @@ ibuf_inside( UNIV_INLINE ibool ibuf_bitmap_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size); /** Checks if a page is a level 2 or 3 page in the ibuf hierarchy of pages. @@ -268,7 +268,7 @@ in which case a new transaction is created. @return TRUE if level 2 or level 3 page */ ibool ibuf_page_low( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, #ifdef UNIV_DEBUG ibool x_latch, @@ -324,7 +324,7 @@ ibuf_insert( ibuf_op_t op, const dtuple_t* entry, dict_index_t* index, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size, que_thr_t* thr); @@ -343,7 +343,7 @@ want to update a non-existent bitmap page */ void ibuf_merge_or_delete_for_page( buf_block_t* block, - const page_id_t& page_id, + const page_id_t page_id, const page_size_t* page_size, ibool update_ibuf_bitmap); @@ -391,9 +391,7 @@ ibuf_parse_bitmap_init( @param[in] page_id page id @return number of entries in the insert buffer currently buffered for this page */ -ulint -ibuf_count_get( - const page_id_t& page_id); +ulint ibuf_count_get(const page_id_t page_id); #endif /******************************************************************//** Looks if the insert buffer is empty. diff --git a/storage/innobase/include/ibuf0ibuf.ic b/storage/innobase/include/ibuf0ibuf.ic index 09070c14059..829e7d5a74f 100644 --- a/storage/innobase/include/ibuf0ibuf.ic +++ b/storage/innobase/include/ibuf0ibuf.ic @@ -156,7 +156,7 @@ ibuf_inside( UNIV_INLINE ibool ibuf_bitmap_page( - const page_id_t& page_id, + const page_id_t page_id, const page_size_t& page_size) { return((page_id.page_no() & (page_size.physical() - 1)) diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index 53a58de229d..9c86f64ccbd 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -1240,7 +1240,7 @@ void page_warn_strict_checksum( srv_checksum_algorithm_t curr_algo, srv_checksum_algorithm_t page_checksum, - const page_id_t& page_id); + const page_id_t page_id); #ifdef UNIV_MATERIALIZE #undef UNIV_INLINE diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index ebe70a1c70e..e1bc4850a61 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -57,10 +57,7 @@ extern trx_sys_t* trx_sys; /** Checks if a page address is the trx sys header page. @param[in] page_id page id @return true if trx sys header page */ -UNIV_INLINE -bool -trx_sys_hdr_page( - const page_id_t& page_id); +inline bool trx_sys_hdr_page(const page_id_t page_id); /** Initialize the transaction system main-memory data structures. */ void trx_sys_init_at_db_start(); diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 861800ef40e..525ae5a6b98 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -54,10 +54,7 @@ trx_sys_flush_max_trx_id(void); /** Checks if a page address is the trx sys header page. @param[in] page_id page id @return true if trx sys header page */ -UNIV_INLINE -bool -trx_sys_hdr_page( - const page_id_t& page_id) +inline bool trx_sys_hdr_page(const page_id_t page_id) { return(page_id.space() == TRX_SYS_SPACE && page_id.page_no() == TRX_SYS_PAGE_NO); diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 0de4ef309a5..a9f828a49d2 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -107,7 +107,7 @@ trx_read_roll_ptr( @return pointer to page x-latched */ UNIV_INLINE page_t* -trx_undo_page_get(const page_id_t& page_id, mtr_t* mtr); +trx_undo_page_get(const page_id_t page_id, mtr_t* mtr); /** Gets an undo log page and s-latches it. @param[in] page_id page id @@ -115,7 +115,7 @@ trx_undo_page_get(const page_id_t& page_id, mtr_t* mtr); @return pointer to page s-latched */ UNIV_INLINE page_t* -trx_undo_page_get_s_latched(const page_id_t& page_id, mtr_t* mtr); +trx_undo_page_get_s_latched(const page_id_t page_id, mtr_t* mtr); /******************************************************************//** Returns the previous undo record on the page in the specified log, or diff --git a/storage/innobase/include/trx0undo.ic b/storage/innobase/include/trx0undo.ic index 0285c212bdd..dc19840777d 100644 --- a/storage/innobase/include/trx0undo.ic +++ b/storage/innobase/include/trx0undo.ic @@ -158,7 +158,7 @@ trx_read_roll_ptr( @return pointer to page x-latched */ UNIV_INLINE page_t* -trx_undo_page_get(const page_id_t& page_id, mtr_t* mtr) +trx_undo_page_get(const page_id_t page_id, mtr_t* mtr) { buf_block_t* block = buf_page_get(page_id, univ_page_size, RW_X_LATCH, mtr); @@ -174,7 +174,7 @@ trx_undo_page_get(const page_id_t& page_id, mtr_t* mtr) @return pointer to page s-latched */ UNIV_INLINE page_t* -trx_undo_page_get_s_latched(const page_id_t& page_id, mtr_t* mtr) +trx_undo_page_get_s_latched(const page_id_t page_id, mtr_t* mtr) { buf_block_t* block = buf_page_get(page_id, univ_page_size, RW_S_LATCH, mtr); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 880935e304e..42623aa9655 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2128,10 +2128,7 @@ recv_recover_page(bool just_read_in, buf_block_t* block) page number. @param[in] page_id page id @return number of pages found */ -static -ulint -recv_read_in_area( - const page_id_t& page_id) +static ulint recv_read_in_area(const page_id_t page_id) { recv_addr_t* recv_addr; ulint page_nos[RECV_READ_AHEAD_AREA]; diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 1b010448135..4b5b2cd999b 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -2833,7 +2833,7 @@ void page_warn_strict_checksum( srv_checksum_algorithm_t curr_algo, srv_checksum_algorithm_t page_checksum, - const page_id_t& page_id) + const page_id_t page_id) { srv_checksum_algorithm_t curr_algo_nonstrict; switch (curr_algo) { diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 371b9aa556f..71520dd1b84 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3668,7 +3668,7 @@ fil_tablespace_iterate( buf_block_t* block = reinterpret_cast (ut_zalloc_nokey(sizeof *block)); block->frame = page; - block->page.id.copy_from(page_id_t(0, 0)); + block->page.id = page_id_t(0, 0); block->page.io_fix = BUF_IO_NONE; block->page.buf_fix_count = 1; block->page.state = BUF_BLOCK_FILE_PAGE; @@ -3686,8 +3686,7 @@ fil_tablespace_iterate( } if (err == DB_SUCCESS) { - block->page.id.copy_from( - page_id_t(callback.get_space_id(), 0)); + block->page.id = page_id_t(callback.get_space_id(), 0); block->page.size.copy_from(callback.get_page_size()); if (block->page.size.is_compressed()) { page_zip_set_size(&block->page.zip, diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index c6551595a08..5cb3f6572c0 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2453,24 +2453,24 @@ files_checked: page_id_t(IBUF_SPACE_ID, FSP_IBUF_HEADER_PAGE_NO), univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + fil_block_check_type(*block, FIL_PAGE_TYPE_SYS, &mtr); /* Already MySQL 3.23.53 initialized FSP_IBUF_TREE_ROOT_PAGE_NO to FIL_PAGE_INDEX. No need to reset that one. */ block = buf_page_get( page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO), univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS, + fil_block_check_type(*block, FIL_PAGE_TYPE_TRX_SYS, &mtr); block = buf_page_get( page_id_t(TRX_SYS_SPACE, FSP_FIRST_RSEG_PAGE_NO), univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + fil_block_check_type(*block, FIL_PAGE_TYPE_SYS, &mtr); block = buf_page_get( page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO), univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + fil_block_check_type(*block, FIL_PAGE_TYPE_SYS, &mtr); mtr.commit(); } From 76318d55aa0f5cb2ef5282538d8997b308c48204 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 24 Oct 2018 09:03:47 +0200 Subject: [PATCH 072/127] MDEV-17525: Window functions not working in ONLY_FULL_GROUP_BY mode The ONLY_FULL_GROUP_BY mode states that for SELECT ... GROUP BY queries, disallow SELECTing columns which are not referred to in the GROUP BY clause, unless they are passed to an aggregate function like COUNT() or MAX(). This holds only for the GROUP BY clause of the query. The code also checks this for the partition clause of the window function which is incorrect. --- mysql-test/r/win.result | 37 +++++++++++++++++++++++++++++++++++++ mysql-test/t/win.test | 25 +++++++++++++++++++++++++ sql/sql_select.cc | 3 ++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index aef9b613433..b73fe346303 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -3326,3 +3326,40 @@ SELECT DISTINCT BIT_AND(0) OVER (), MAX(1) FROM t1; BIT_AND(0) OVER () MAX(1) 0 1 drop table t1; +# +# MDEV-17525: Window functions not working in ONLY_FULL_GROUP_BY mode +# +CREATE TABLE t1 (name CHAR(10), test CHAR(10), score TINYINT); +INSERT INTO t1 VALUES +('Chun', 'SQL', 75), ('Chun', 'Tuning', 73), +('Esben', 'SQL', 43), ('Esben', 'Tuning', 31), +('Kaolin', 'SQL', 56), ('Kaolin', 'Tuning', 88), +('Tatiana', 'SQL', 87), ('Tatiana', 'Tuning', 83); +SET @save_sql_mode= @@sql_mode; +SET sql_mode = 'ONLY_FULL_GROUP_BY'; +SELECT name, test, score, +AVG(score) OVER (PARTITION BY test) AS average_by_test +FROM t1; +name test score average_by_test +Chun SQL 75 65.2500 +Chun Tuning 73 68.7500 +Esben SQL 43 65.2500 +Esben Tuning 31 68.7500 +Kaolin SQL 56 65.2500 +Kaolin Tuning 88 68.7500 +Tatiana SQL 87 65.2500 +Tatiana Tuning 83 68.7500 +set @@sql_mode= @save_sql_mode; +SELECT name, test, score, +AVG(score) OVER (PARTITION BY test) AS average_by_test +FROM t1; +name test score average_by_test +Chun SQL 75 65.2500 +Chun Tuning 73 68.7500 +Esben SQL 43 65.2500 +Esben Tuning 31 68.7500 +Kaolin SQL 56 65.2500 +Kaolin Tuning 88 68.7500 +Tatiana SQL 87 65.2500 +Tatiana Tuning 83 68.7500 +drop table t1; diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index 1617e85caf4..e24c6533dcd 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -2092,3 +2092,28 @@ insert into t1 values (1),(2); SELECT DISTINCT row_number() OVER (), MAX(1) FROM t1; SELECT DISTINCT BIT_AND(0) OVER (), MAX(1) FROM t1; drop table t1; + +--echo # +--echo # MDEV-17525: Window functions not working in ONLY_FULL_GROUP_BY mode +--echo # + +CREATE TABLE t1 (name CHAR(10), test CHAR(10), score TINYINT); + +INSERT INTO t1 VALUES +('Chun', 'SQL', 75), ('Chun', 'Tuning', 73), +('Esben', 'SQL', 43), ('Esben', 'Tuning', 31), +('Kaolin', 'SQL', 56), ('Kaolin', 'Tuning', 88), +('Tatiana', 'SQL', 87), ('Tatiana', 'Tuning', 83); + +SET @save_sql_mode= @@sql_mode; +SET sql_mode = 'ONLY_FULL_GROUP_BY'; + +SELECT name, test, score, + AVG(score) OVER (PARTITION BY test) AS average_by_test +FROM t1; + +set @@sql_mode= @save_sql_mode; +SELECT name, test, score, + AVG(score) OVER (PARTITION BY test) AS average_by_test +FROM t1; +drop table t1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index a9adbd168a6..1fa80da85a6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -22611,7 +22611,8 @@ setup_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, return 1; } } - if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY) + if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY && + context_analysis_place == IN_GROUP_BY) { /* Don't allow one to use fields that is not used in GROUP BY From 65cfc5873e8a9d528451d741b512d12a77eff81f Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Fri, 26 Oct 2018 04:00:00 -0400 Subject: [PATCH 073/127] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4acb69e9e07..34a2418fde1 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=62 +MYSQL_VERSION_PATCH=63 MYSQL_VERSION_EXTRA= From bbcb1734365b0e81d7d82e1d3d9e37074048970e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 27 Oct 2018 20:53:19 +0200 Subject: [PATCH 074/127] 5.6.41-84.1 --- storage/xtradb/.clang-format | 111 ++++++++++++++++++++++++ storage/xtradb/CMakeLists.txt | 17 ++-- storage/xtradb/buf/buf0buf.cc | 14 ++- storage/xtradb/buf/buf0dump.cc | 5 +- storage/xtradb/dict/dict0dict.cc | 7 +- storage/xtradb/fts/fts0blex.cc | 30 +++---- storage/xtradb/fts/fts0fts.cc | 67 ++++++-------- storage/xtradb/fts/fts0pars.cc | 10 +-- storage/xtradb/fts/fts0pars.y | 10 +-- storage/xtradb/fts/fts0tlex.cc | 30 +++---- storage/xtradb/handler/ha_innodb.cc | 50 +++++++++-- storage/xtradb/handler/ha_innodb.h | 2 +- storage/xtradb/handler/handler0alter.cc | 51 +++++++---- storage/xtradb/handler/i_s.cc | 2 +- storage/xtradb/include/buf0buf.h | 1 + storage/xtradb/include/data0type.ic | 2 + storage/xtradb/include/dict0mem.h | 3 +- storage/xtradb/include/os0proc.h | 3 +- storage/xtradb/include/univ.i | 2 +- storage/xtradb/log/log0online.cc | 4 +- storage/xtradb/os/os0proc.cc | 63 +++++++++++++- storage/xtradb/page/page0page.cc | 8 +- storage/xtradb/pars/lexyy.cc | 30 +++---- storage/xtradb/row/row0import.cc | 35 +++++--- storage/xtradb/row/row0log.cc | 14 ++- storage/xtradb/row/row0merge.cc | 2 +- storage/xtradb/row/row0mysql.cc | 20 ++++- storage/xtradb/row/row0sel.cc | 3 +- storage/xtradb/srv/srv0conc.cc | 1 + storage/xtradb/srv/srv0srv.cc | 1 - storage/xtradb/srv/srv0start.cc | 3 +- storage/xtradb/trx/trx0trx.cc | 3 +- storage/xtradb/trx/trx0undo.cc | 2 +- 33 files changed, 433 insertions(+), 173 deletions(-) create mode 100644 storage/xtradb/.clang-format diff --git a/storage/xtradb/.clang-format b/storage/xtradb/.clang-format new file mode 100644 index 00000000000..d757d0a5a05 --- /dev/null +++ b/storage/xtradb/.clang-format @@ -0,0 +1,111 @@ +# generated with: +# clang-format-5.0 -style=Google --dump-config + +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +SortUsingDeclarations: true +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +TabWidth: 8 + + +# changes for MySQL 5.x (InnoDB) +AlignConsecutiveDeclarations: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: All +ColumnLimit: 78 +DerivePointerAlignment: false +IndentWidth: 8 +MaxEmptyLinesToKeep: 2 +PointerAlignment: Right +ReflowComments: false +SortIncludes: false +SpaceAfterCStyleCast: true +UseTab: Always diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index 4284c3a0753..5fa6f143d9d 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -29,9 +29,6 @@ IF(UNIX) ADD_DEFINITIONS(-DLINUX_NATIVE_AIO=1) LINK_LIBRARIES(aio) ENDIF() - IF(HAVE_LIBNUMA) - LINK_LIBRARIES(numa) - ENDIF() ELSEIF(CMAKE_SYSTEM_NAME MATCHES "HP*") ADD_DEFINITIONS("-DUNIV_HPUX") ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "AIX") @@ -448,9 +445,19 @@ IF(WITH_INNODB) SET(WITH_INNOBASE_STORAGE_ENGINE TRUE) ENDIF() +UNSET(NUMA_LIBRARY) +IF(HAVE_LIBNUMA) + SET(NUMA_LIBRARY "numa") +ENDIF() + MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE DEFAULT MODULE_OUTPUT_NAME ha_innodb - LINK_LIBRARIES ${ZLIB_LIBRARY}) + LINK_LIBRARIES ${ZLIB_LIBRARY} ${NUMA_LIBRARY}) -ADD_DEPENDENCIES(innobase GenError) +IF(WITH_INNOBASE_STORAGE_ENGINE) + # Remove -DMYSQL_SERVER, it breaks embedded build + SET_TARGET_PROPERTIES(innobase PROPERTIES COMPILE_DEFINITIONS "") + + ADD_DEPENDENCIES(innobase GenError) +ENDIF() diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index b946b1dd4e4..9d3e5b3db42 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -53,6 +53,9 @@ Created 11/5/1995 Heikki Tuuri #include "page0zip.h" #include "srv0mon.h" #include "buf0checksum.h" + +UNIV_INTERN my_bool srv_numa_interleave = FALSE; + #ifdef HAVE_LIBNUMA #include #include @@ -1129,7 +1132,8 @@ buf_chunk_init( /*===========*/ buf_pool_t* buf_pool, /*!< in: buffer pool instance */ buf_chunk_t* chunk, /*!< out: chunk of buffers */ - ulint mem_size) /*!< in: requested size in bytes */ + ulint mem_size, /*!< in: requested size in bytes */ + bool populate) /*!< in: virtual page preallocation */ { buf_block_t* block; byte* frame; @@ -1145,7 +1149,7 @@ buf_chunk_init( + (UNIV_PAGE_SIZE - 1), UNIV_PAGE_SIZE); chunk->mem_size = mem_size; - chunk->mem = os_mem_alloc_large(&chunk->mem_size); + chunk->mem = os_mem_alloc_large(&chunk->mem_size, populate); if (UNIV_UNLIKELY(chunk->mem == NULL)) { @@ -1364,6 +1368,7 @@ buf_pool_init_instance( /*===================*/ buf_pool_t* buf_pool, /*!< in: buffer pool instance */ ulint buf_pool_size, /*!< in: size in bytes */ + bool populate, /*!< in: virtual page preallocation */ ulint instance_no) /*!< in: id of the instance */ { ulint i; @@ -1392,7 +1397,7 @@ buf_pool_init_instance( UT_LIST_INIT(buf_pool->free); - if (!buf_chunk_init(buf_pool, chunk, buf_pool_size)) { + if (!buf_chunk_init(buf_pool, chunk, buf_pool_size, populate)) { mem_free(chunk); mem_free(buf_pool); @@ -1518,6 +1523,7 @@ dberr_t buf_pool_init( /*==========*/ ulint total_size, /*!< in: size of the total pool in bytes */ + bool populate, /*!< in: virtual page preallocation */ ulint n_instances) /*!< in: number of instances */ { ulint i; @@ -1548,7 +1554,7 @@ buf_pool_init( for (i = 0; i < n_instances; i++) { buf_pool_t* ptr = &buf_pool_ptr[i]; - if (buf_pool_init_instance(ptr, size, i) != DB_SUCCESS) { + if (buf_pool_init_instance(ptr, size, populate, i) != DB_SUCCESS) { /* Free all the instances created so far. */ buf_pool_free(i); diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index 8ecdedc613b..c7c3a4b49ba 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -201,8 +201,9 @@ buf_dump( { #define SHOULD_QUIT() (SHUTTING_DOWN() && obey_shutdown) + static const char format_name[]= "%s.incomplete"; char full_filename[OS_FILE_MAX_PATH]; - char tmp_filename[OS_FILE_MAX_PATH]; + char tmp_filename[OS_FILE_MAX_PATH + sizeof(format_name)]; char now[32]; FILE* f; ulint i; @@ -213,7 +214,7 @@ buf_dump( srv_buf_dump_filename); ut_snprintf(tmp_filename, sizeof(tmp_filename), - "%s.incomplete", full_filename); + format_name, full_filename); buf_dump_status(STATUS_NOTICE, "Dumping buffer pool(s) to %s", full_filename); diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 816c6f913a4..c39066b6d4f 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. This program is free software; you can redistribute it and/or modify it under @@ -3391,7 +3391,10 @@ dict_foreign_find_index( && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, - check_charsets, check_null)) { + check_charsets, check_null) + && (!(index->online_status == + ONLINE_INDEX_ABORTED_DROPPED + ||index->online_status == ONLINE_INDEX_ABORTED))) { return(index); } diff --git a/storage/xtradb/fts/fts0blex.cc b/storage/xtradb/fts/fts0blex.cc index 2d71934fa0e..a8f231268ac 100644 --- a/storage/xtradb/fts/fts0blex.cc +++ b/storage/xtradb/fts/fts0blex.cc @@ -701,9 +701,9 @@ extern int fts0blex (yyscan_t yyscanner); */ YY_DECL { - register yy_state_type yy_current_state; - register char *yy_cp, *yy_bp; - register int yy_act; + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; #line 43 "fts0blex.l" @@ -753,7 +753,7 @@ YY_DECL yy_match: do { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -984,9 +984,9 @@ case YY_STATE_EOF(INITIAL): static int yy_get_next_buffer (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - register char *source = yyg->yytext_ptr; - register int number_to_move, i; + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = yyg->yytext_ptr; + int number_to_move, i; int ret_val; if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) @@ -1118,15 +1118,15 @@ static int yy_get_next_buffer (yyscan_t yyscanner) static yy_state_type yy_get_previous_state (yyscan_t yyscanner) { - register yy_state_type yy_current_state; - register char *yy_cp; + yy_state_type yy_current_state; + char *yy_cp; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yy_current_state = yyg->yy_start; for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp ) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1151,11 +1151,11 @@ static yy_state_type yy_get_previous_state (yyscan_t yyscanner) */ static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner) { - register int yy_is_jam; + int yy_is_jam; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */ - register char *yy_cp = yyg->yy_c_buf_p; + char *yy_cp = yyg->yy_c_buf_p; - register YY_CHAR yy_c = 1; + YY_CHAR yy_c = 1; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1912,7 +1912,7 @@ int fts0blex_destroy (yyscan_t yyscanner) #ifndef yytext_ptr static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused))) { - register int i; + int i; for ( i = 0; i < n; ++i ) s1[i] = s2[i]; } @@ -1921,7 +1921,7 @@ static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yys #ifdef YY_NEED_STRLEN static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused))) { - register int n; + int n; for ( n = 0; s[n]; ++n ) ; diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index f3fa6e52d0d..a1f56671fc8 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -868,37 +868,28 @@ fts_drop_index( err = fts_drop_index_tables(trx, index); - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry){ - fts_free(table); - break; - } - DICT_BG_YIELD(trx); - } + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + + fts_free(table); + return(err); } - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry){ - current_doc_id = table->fts->cache->next_doc_id; - first_doc_id = table->fts->cache->first_doc_id; - fts_cache_clear(table->fts->cache); - fts_cache_destroy(table->fts->cache); - table->fts->cache = fts_cache_create(table); - table->fts->cache->next_doc_id = current_doc_id; - table->fts->cache->first_doc_id = first_doc_id; - break; - } - DICT_BG_YIELD(trx); - } + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + + current_doc_id = table->fts->cache->next_doc_id; + first_doc_id = table->fts->cache->first_doc_id; + fts_cache_clear(table->fts->cache); + fts_cache_destroy(table->fts->cache); + table->fts->cache = fts_cache_create(table); + table->fts->cache->next_doc_id = current_doc_id; + table->fts->cache->first_doc_id = first_doc_id; } else { fts_cache_t* cache = table->fts->cache; fts_index_cache_t* index_cache; @@ -908,17 +899,13 @@ fts_drop_index( index_cache = fts_find_index_cache(cache, index); if (index_cache != NULL) { - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry && index_cache->words) { - fts_words_free(index_cache->words); - rbt_free(index_cache->words); - break; - } - DICT_BG_YIELD(trx); + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + if (index_cache->words) { + fts_words_free(index_cache->words); + rbt_free(index_cache->words); } ib_vector_remove(cache->indexes, *(void**) index_cache); diff --git a/storage/xtradb/fts/fts0pars.cc b/storage/xtradb/fts/fts0pars.cc index 7f0ba4e0c1b..f4ac4c34f22 100644 --- a/storage/xtradb/fts/fts0pars.cc +++ b/storage/xtradb/fts/fts0pars.cc @@ -106,8 +106,8 @@ typedef int (*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner); typedef int (*fts_scanner)(); struct fts_lexer_t { - fts_scanner scanner; - void* yyscanner; + fts_scanner_alt scanner; + void* yyscanner; }; @@ -1950,7 +1950,7 @@ fts_lexer_create( reinterpret_cast(query), static_cast(query_len), fts_lexer->yyscanner); - fts_lexer->scanner = reinterpret_cast(fts_blexer); + fts_lexer->scanner = fts_blexer; /* FIXME: Debugging */ /* fts0bset_debug(1 , fts_lexer->yyscanner); */ } else { @@ -1959,7 +1959,7 @@ fts_lexer_create( reinterpret_cast(query), static_cast(query_len), fts_lexer->yyscanner); - fts_lexer->scanner = reinterpret_cast(fts_tlexer); + fts_lexer->scanner = fts_tlexer; } return(fts_lexer); @@ -1973,7 +1973,7 @@ fts_lexer_free( /*===========*/ fts_lexer_t* fts_lexer) { - if (fts_lexer->scanner == (fts_scan) fts_blexer) { + if (fts_lexer->scanner == fts_blexer) { fts0blex_destroy(fts_lexer->yyscanner); } else { fts0tlex_destroy(fts_lexer->yyscanner); diff --git a/storage/xtradb/fts/fts0pars.y b/storage/xtradb/fts/fts0pars.y index e48036e82fe..d2e4612b4f5 100644 --- a/storage/xtradb/fts/fts0pars.y +++ b/storage/xtradb/fts/fts0pars.y @@ -52,8 +52,8 @@ typedef int (*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner); typedef int (*fts_scanner)(); struct fts_lexer_struct { - fts_scanner scanner; - void* yyscanner; + fts_scanner_alt scanner; + void* yyscanner; }; %} @@ -238,13 +238,13 @@ fts_lexer_create( if (boolean_mode) { fts0blex_init(&fts_lexer->yyscanner); fts0b_scan_bytes((char*) query, query_len, fts_lexer->yyscanner); - fts_lexer->scanner = (fts_scan) fts_blexer; + fts_lexer->scanner = fts_blexer; /* FIXME: Debugging */ /* fts0bset_debug(1 , fts_lexer->yyscanner); */ } else { fts0tlex_init(&fts_lexer->yyscanner); fts0t_scan_bytes((char*) query, query_len, fts_lexer->yyscanner); - fts_lexer->scanner = (fts_scan) fts_tlexer; + fts_lexer->scanner = fts_tlexer; } return(fts_lexer); @@ -258,7 +258,7 @@ fts_lexer_free( /*===========*/ fts_lexer_t* fts_lexer) { - if (fts_lexer->scanner == (fts_scan) fts_blexer) { + if (fts_lexer->scanner == fts_blexer) { fts0blex_destroy(fts_lexer->yyscanner); } else { fts0tlex_destroy(fts_lexer->yyscanner); diff --git a/storage/xtradb/fts/fts0tlex.cc b/storage/xtradb/fts/fts0tlex.cc index d4d9b4c48d1..28fac9bf704 100644 --- a/storage/xtradb/fts/fts0tlex.cc +++ b/storage/xtradb/fts/fts0tlex.cc @@ -697,9 +697,9 @@ extern int fts0tlex (yyscan_t yyscanner); */ YY_DECL { - register yy_state_type yy_current_state; - register char *yy_cp, *yy_bp; - register int yy_act; + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; #line 44 "fts0tlex.l" @@ -749,7 +749,7 @@ YY_DECL yy_match: do { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -976,9 +976,9 @@ case YY_STATE_EOF(INITIAL): static int yy_get_next_buffer (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - register char *source = yyg->yytext_ptr; - register int number_to_move, i; + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = yyg->yytext_ptr; + int number_to_move, i; int ret_val; if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) @@ -1110,15 +1110,15 @@ static int yy_get_next_buffer (yyscan_t yyscanner) static yy_state_type yy_get_previous_state (yyscan_t yyscanner) { - register yy_state_type yy_current_state; - register char *yy_cp; + yy_state_type yy_current_state; + char *yy_cp; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yy_current_state = yyg->yy_start; for ( yy_cp = yyg->yytext_ptr + YY_MORE_ADJ; yy_cp < yyg->yy_c_buf_p; ++yy_cp ) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1143,11 +1143,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) */ static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state , yyscan_t yyscanner) { - register int yy_is_jam; + int yy_is_jam; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* This var may be unused depending upon options. */ - register char *yy_cp = yyg->yy_c_buf_p; + char *yy_cp = yyg->yy_c_buf_p; - register YY_CHAR yy_c = 1; + YY_CHAR yy_c = 1; if ( yy_accept[yy_current_state] ) { yyg->yy_last_accepting_state = yy_current_state; @@ -1905,7 +1905,7 @@ int fts0tlex_destroy (yyscan_t yyscanner) #ifndef yytext_ptr static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused))) { - register int i; + int i; for ( i = 0; i < n; ++i ) s1[i] = s2[i]; } @@ -1914,7 +1914,7 @@ static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yys #ifdef YY_NEED_STRLEN static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused)) MY_ATTRIBUTE((unused))) { - register int n; + int n; for ( n = 0; s[n]; ++n ) ; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 7b6d4243ba8..053a181e26c 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. @@ -3162,10 +3162,10 @@ innobase_convert_identifier( const char* s = id; int q; - if (file_id) { + char nz[MAX_TABLE_NAME_LEN + 1]; + char nz2[MAX_TABLE_NAME_LEN + 1]; - char nz[MAX_TABLE_NAME_LEN + 1]; - char nz2[MAX_TABLE_NAME_LEN + 1]; + if (file_id) { /* Decode the table name. The MySQL function expects a NUL-terminated string. The input and output strings @@ -3352,6 +3352,7 @@ ha_innobase::reset_template(void) prebuilt->keep_other_fields_on_keyread = 0; prebuilt->read_just_key = 0; prebuilt->in_fts_query = 0; + prebuilt->end_range = false; /* Reset index condition pushdown state. */ if (prebuilt->idx_cond) { prebuilt->idx_cond = NULL; @@ -11577,6 +11578,7 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t innobase_rename_table( /*==================*/ + THD* thd, /*!< Connection thread handle */ trx_t* trx, /*!< in: transaction */ const char* from, /*!< in: old name of the table */ const char* to) /*!< in: new name of the table */ @@ -11602,6 +11604,37 @@ innobase_rename_table( row_mysql_lock_data_dictionary(trx); + dict_table_t* table = NULL; + table = dict_table_open_on_name(norm_from, TRUE, FALSE, + DICT_ERR_IGNORE_NONE); + + /* Since DICT_BG_YIELD has sleep for 250 milliseconds, + Convert lock_wait_timeout unit from second to 250 milliseconds */ + long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; + if (table != NULL) { + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { + + if (index->type & DICT_FTS) { + /* Found */ + while (index->index_fts_syncing + && !trx_is_interrupted(trx) + && (lock_wait_timeout--) > 0) { + DICT_BG_YIELD(trx); + } + } + } + dict_table_close(table, TRUE, FALSE); + } + + /* FTS sync is in progress. We shall timeout this operation */ + if (lock_wait_timeout < 0) { + error = DB_LOCK_WAIT_TIMEOUT; + row_mysql_unlock_data_dictionary(trx); + DBUG_RETURN(error); + } + /* Transaction must be flagged as a locking transaction or it hasn't been started yet. */ @@ -11716,7 +11749,7 @@ ha_innobase::rename_table( ++trx->will_lock; trx_set_dict_operation(trx, TRX_DICT_OP_INDEX); - error = innobase_rename_table(trx, from, to); + error = innobase_rename_table(thd, trx, from, to); DEBUG_SYNC(thd, "after_innobase_rename_table"); @@ -11762,6 +11795,12 @@ ha_innobase::rename_table( error = DB_ERROR; } + else if (error == DB_LOCK_WAIT_TIMEOUT) { + my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to); + + error = DB_LOCK_WAIT; + } + DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); } @@ -13536,6 +13575,7 @@ ha_innobase::start_stmt( case SQLCOM_INSERT: case SQLCOM_UPDATE: case SQLCOM_DELETE: + case SQLCOM_REPLACE: init_table_handle_for_HANDLER(); prebuilt->select_lock_type = LOCK_X; prebuilt->stored_select_lock_type = LOCK_X; diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index f4ea77bff33..652530eaaaa 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -183,7 +183,7 @@ class ha_innobase: public handler char* norm_name, char* temp_path, char* remote_path); - int create(const char *name, register TABLE *form, + int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info); int truncate(); int delete_table(const char *name); diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 7e155c99b9d..517dd30410b 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2005, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4647,13 +4647,15 @@ processed_field: } /** Get the auto-increment value of the table on commit. -@param ha_alter_info Data used during in-place alter -@param ctx In-place ALTER TABLE context -@param altered_table MySQL table that is being altered -@param old_table MySQL table as it is before the ALTER operation -@return the next auto-increment value (0 if not present) */ +@param[in] ha_alter_info Data used during in-place alter +@param[in,out] ctx In-place ALTER TABLE context + return autoinc value in ctx->max_autoinc +@param altered_table[in] MySQL table that is being altered +@param old_table[in] MySQL table as it is before the ALTER operation +retval true Failure +@retval false Success*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -ulonglong +bool commit_get_autoinc( /*===============*/ Alter_inplace_info* ha_alter_info, @@ -4661,23 +4663,28 @@ commit_get_autoinc( const TABLE* altered_table, const TABLE* old_table) { - ulonglong max_autoinc; DBUG_ENTER("commit_get_autoinc"); if (!altered_table->found_next_number_field) { /* There is no AUTO_INCREMENT column in the table after the ALTER operation. */ - max_autoinc = 0; + ctx->max_autoinc = 0; } else if (ctx->add_autoinc != ULINT_UNDEFINED) { /* An AUTO_INCREMENT column was added. Get the last value from the sequence, which may be based on a supplied AUTO_INCREMENT value. */ - max_autoinc = ctx->sequence.last(); + ctx->max_autoinc = ctx->sequence.last(); } else if ((ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && (ha_alter_info->create_info->used_fields & HA_CREATE_USED_AUTO)) { + + /* Check if the table is discarded */ + if(dict_table_is_discarded(ctx->old_table)) { + DBUG_RETURN(true); + } + /* An AUTO_INCREMENT value was supplied, but the table was not rebuilt. Get the user-supplied value or the last value from the sequence. */ @@ -4690,7 +4697,8 @@ commit_get_autoinc( dict_index_t* index = dict_table_get_index_on_first_col( ctx->old_table, autoinc_field->field_index); - max_autoinc = ha_alter_info->create_info->auto_increment_value; + ctx->max_autoinc = + ha_alter_info->create_info->auto_increment_value; dict_table_autoinc_lock(ctx->old_table); @@ -4699,8 +4707,8 @@ commit_get_autoinc( if (err != DB_SUCCESS) { ut_ad(0); - max_autoinc = 0; - } else if (max_autoinc <= max_value_table) { + ctx->max_autoinc = 0; + } else if (ctx->max_autoinc <= max_value_table) { ulonglong col_max_value; ulonglong offset; @@ -4708,7 +4716,7 @@ commit_get_autoinc( old_table->found_next_number_field); offset = ctx->prebuilt->autoinc_offset; - max_autoinc = innobase_next_autoinc( + ctx->max_autoinc = innobase_next_autoinc( max_value_table, 1, 1, offset, col_max_value); } @@ -4718,11 +4726,11 @@ commit_get_autoinc( Read the old counter value from the table. */ ut_ad(old_table->found_next_number_field); dict_table_autoinc_lock(ctx->old_table); - max_autoinc = ctx->old_table->autoinc; + ctx->max_autoinc = ctx->old_table->autoinc; dict_table_autoinc_unlock(ctx->old_table); } - DBUG_RETURN(max_autoinc); + DBUG_RETURN(false); } /** Add or drop foreign key constraints to the data dictionary tables, @@ -5705,8 +5713,13 @@ ha_innobase::commit_inplace_alter_table( DBUG_ASSERT(new_clustered == ctx->need_rebuild()); - ctx->max_autoinc = commit_get_autoinc( - ha_alter_info, ctx, altered_table, table); + if (commit_get_autoinc(ha_alter_info, ctx, altered_table, + table)) { + fail = true; + my_error(ER_TABLESPACE_DISCARDED, MYF(0), + table->s->table_name.str); + goto rollback_trx; + } if (ctx->need_rebuild()) { ctx->tmp_name = dict_mem_create_temporary_tablename( @@ -5738,6 +5751,8 @@ ha_innobase::commit_inplace_alter_table( #endif } +rollback_trx: + /* Commit or roll back the changes to the data dictionary. */ if (fail) { diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index 659ba5aec91..5b692267b69 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -1482,7 +1482,7 @@ i_s_cmp_fill_low( static_cast(zip_stat->decompressed_usec / 1000000)); if (reset) { - memset(zip_stat, 0, sizeof *zip_stat); + memset(static_cast(zip_stat), 0, sizeof *zip_stat); } if (schema_table_store_record(thd, table)) { diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 929b670ef58..9f52cfb85d9 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -214,6 +214,7 @@ dberr_t buf_pool_init( /*=========*/ ulint size, /*!< in: Size of the total pool in bytes */ + bool populate, /*!< in: Force virtual page preallocation */ ulint n_instances); /*!< in: Number of instances */ /********************************************************************//** Frees the buffer pool at shutdown. This must not be invoked before diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 552d62eef00..ef92ea43517 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -544,6 +544,7 @@ dtype_get_fixed_size_low( return(0); } #endif /* UNIV_DEBUG */ + // fallthrough case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: @@ -622,6 +623,7 @@ dtype_get_min_size_low( return(0); } #endif /* UNIV_DEBUG */ + // fallthrough case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index b6d2ab9fad1..bc7ab94f9d7 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -282,8 +282,7 @@ dict_mem_table_add_col( const char* name, /*!< in: column name, or NULL */ ulint mtype, /*!< in: main datatype */ ulint prtype, /*!< in: precise type */ - ulint len) /*!< in: precision */ - MY_ATTRIBUTE((nonnull(1))); + ulint len); /*!< in: precision */ /**********************************************************************//** Renames a column of a table in the data dictionary cache. */ UNIV_INTERN diff --git a/storage/xtradb/include/os0proc.h b/storage/xtradb/include/os0proc.h index 613e3bd6947..d6aafccf48e 100644 --- a/storage/xtradb/include/os0proc.h +++ b/storage/xtradb/include/os0proc.h @@ -58,7 +58,8 @@ UNIV_INTERN void* os_mem_alloc_large( /*===============*/ - ulint* n); /*!< in/out: number of bytes */ + ulint* n, /*!< in/out: number of bytes */ + bool populate); /*!< in: virtual page preallocation */ /****************************************************************//** Frees large pages memory. */ UNIV_INTERN diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 8eb3e6ef9bc..f625ea46433 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -47,7 +47,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_BUGFIX MYSQL_VERSION_PATCH #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 83.1 +#define PERCONA_INNODB_VERSION 84.1 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 9e9e3a4c284..e3cfbfc2088 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -1538,10 +1538,10 @@ log_online_open_bitmap_file_read_only( if (srv_data_home_len && srv_data_home[srv_data_home_len-1] != SRV_PATH_SEPARATOR) { - ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%c%s", + ut_snprintf(bitmap_file->name, sizeof(bitmap_file->name), "%s%c%s", srv_data_home, SRV_PATH_SEPARATOR, name); } else { - ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%s", + ut_snprintf(bitmap_file->name, sizeof(bitmap_file->name), "%s%s", srv_data_home, name); } bitmap_file->file diff --git a/storage/xtradb/os/os0proc.cc b/storage/xtradb/os/os0proc.cc index ff6d65e4ae6..6c9116e1397 100644 --- a/storage/xtradb/os/os0proc.cc +++ b/storage/xtradb/os/os0proc.cc @@ -32,6 +32,14 @@ Created 9/30/1995 Heikki Tuuri #include "ut0mem.h" #include "ut0byte.h" +/* Linux release version */ +#if defined(UNIV_LINUX) && defined(_GNU_SOURCE) +#include /* strverscmp() */ +#include /* uname() */ +#endif + +#include "ha_prototypes.h" + /* FreeBSD for example has only MAP_ANON, Linux has MAP_ANONYMOUS and MAP_ANON but MAP_ANON is marked as deprecated */ #if defined(MAP_ANONYMOUS) @@ -40,10 +48,36 @@ MAP_ANON but MAP_ANON is marked as deprecated */ #define OS_MAP_ANON MAP_ANON #endif +/* Linux's MAP_POPULATE */ +#if defined(MAP_POPULATE) +#define OS_MAP_POPULATE MAP_POPULATE +#else +#define OS_MAP_POPULATE 0 +#endif + UNIV_INTERN ibool os_use_large_pages; /* Large page size. This may be a boot-time option on some platforms */ UNIV_INTERN ulint os_large_page_size; + +/****************************************************************//** +Retrieve and compare operating system release. +@return TRUE if the OS release is equal to, or later than release. */ +UNIV_INTERN +bool +os_compare_release( +/*===============*/ + const char* release /*!< in: OS release */ + MY_ATTRIBUTE((unused))) +{ +#if defined(UNIV_LINUX) && defined(_GNU_SOURCE) + struct utsname name; + return uname(&name) == 0 && strverscmp(name.release, release) >= 0; +#else + return false; +#endif +} + /****************************************************************//** Converts the current process id to a number. It is not guaranteed that the number is unique. In Linux returns the 'process number' of the current @@ -69,7 +103,8 @@ UNIV_INTERN void* os_mem_alloc_large( /*===============*/ - ulint* n) /*!< in/out: number of bytes */ + ulint* n, /*!< in/out: number of bytes */ + bool populate) /*!< in: virtual page preallocation */ { void* ptr; ulint size; @@ -155,12 +190,13 @@ skip: ut_ad(ut_is_2pow(size)); size = *n = ut_2pow_round(*n + (size - 1), size); ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | OS_MAP_ANON, -1, 0); - if (UNIV_UNLIKELY(ptr == (void*) -1)) { + MAP_PRIVATE | OS_MAP_ANON | + (populate ? OS_MAP_POPULATE : 0), -1, 0); + if (UNIV_UNLIKELY(ptr == MAP_FAILED)) { fprintf(stderr, "InnoDB: mmap(%lu bytes) failed;" " errno %lu\n", (ulong) size, (ulong) errno); - ptr = NULL; + return NULL; } else { os_fast_mutex_lock(&ut_list_mutex); ut_total_allocated_memory += size; @@ -168,6 +204,25 @@ skip: UNIV_MEM_ALLOC(ptr, size); } #endif + +#if OS_MAP_ANON && OS_MAP_POPULATE + /* MAP_POPULATE is only supported for private mappings + since Linux 2.6.23. */ + populate = populate && !os_compare_release("2.6.23"); + + if (populate) { + ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Warning: mmap(MAP_POPULATE) " + "is not supported for private mappings. " + "Forcing preallocation by faulting in pages.\n"); + } +#endif + + /* Initialize the entire buffer to force the allocation + of physical memory page frames. */ + if (populate) { + memset(ptr, '\0', size); + } + return(ptr); } diff --git a/storage/xtradb/page/page0page.cc b/storage/xtradb/page/page0page.cc index a340c04913a..9d9704129da 100644 --- a/storage/xtradb/page/page0page.cc +++ b/storage/xtradb/page/page0page.cc @@ -99,10 +99,10 @@ page_dir_find_owner_slot( const rec_t* rec) /*!< in: the physical record */ { const page_t* page; - register uint16 rec_offs_bytes; - register const page_dir_slot_t* slot; - register const page_dir_slot_t* first_slot; - register const rec_t* r = rec; + uint16 rec_offs_bytes; + const page_dir_slot_t* slot; + const page_dir_slot_t* first_slot; + const rec_t* r = rec; ut_ad(page_rec_check(rec)); diff --git a/storage/xtradb/pars/lexyy.cc b/storage/xtradb/pars/lexyy.cc index bfa8e2ea950..d88c5ad2312 100644 --- a/storage/xtradb/pars/lexyy.cc +++ b/storage/xtradb/pars/lexyy.cc @@ -1187,9 +1187,9 @@ extern int yylex (void); */ YY_DECL { - register yy_state_type yy_current_state; - register char *yy_cp, *yy_bp; - register int yy_act; + yy_state_type yy_current_state; + char *yy_cp, *yy_bp; + int yy_act; #line 112 "pars0lex.l" @@ -1238,7 +1238,7 @@ YY_DECL yy_match: do { - register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; if ( yy_accept[yy_current_state] ) { (yy_last_accepting_state) = yy_current_state; @@ -2380,9 +2380,9 @@ case YY_STATE_EOF(id): */ static int yy_get_next_buffer (void) { - register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; - register char *source = (yytext_ptr); - register int number_to_move, i; + char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + char *source = (yytext_ptr); + int number_to_move, i; int ret_val; if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) @@ -2515,14 +2515,14 @@ static int yy_get_next_buffer (void) yy_state_type yy_get_previous_state (void) { - register yy_state_type yy_current_state; - register char *yy_cp; + yy_state_type yy_current_state; + char *yy_cp; yy_current_state = (yy_start); for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) { - register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); if ( yy_accept[yy_current_state] ) { (yy_last_accepting_state) = yy_current_state; @@ -2547,10 +2547,10 @@ static int yy_get_next_buffer (void) */ static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) { - register int yy_is_jam; - register char *yy_cp = (yy_c_buf_p); + int yy_is_jam; + char *yy_cp = (yy_c_buf_p); - register YY_CHAR yy_c = 1; + YY_CHAR yy_c = 1; if ( yy_accept[yy_current_state] ) { (yy_last_accepting_state) = yy_current_state; @@ -3070,7 +3070,7 @@ MY_ATTRIBUTE((unused)) static int yylex_destroy (void) #ifndef yytext_ptr static void yy_flex_strncpy (char* s1, yyconst char * s2, int n ) { - register int i; + int i; for ( i = 0; i < n; ++i ) s1[i] = s2[i]; } @@ -3079,7 +3079,7 @@ static void yy_flex_strncpy (char* s1, yyconst char * s2, int n ) #ifdef YY_NEED_STRLEN static int yy_flex_strlen (yyconst char * s ) { - register int n; + int n; for ( n = 0; s[n]; ++n ) ; diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index ac26c318cb2..92bcd2697c0 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2012, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1340,11 +1340,22 @@ row_import::match_schema( = m_table->flags & ~DICT_TF_MASK_DATA_DIR; if (relevant_flags != relevant_table_flags) { - ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%x " - "and the meta-data file has 0x%x", - relevant_table_flags, relevant_flags); - + if (dict_tf_to_row_format_string(relevant_flags) != + dict_tf_to_row_format_string(relevant_table_flags)) { + ib_errf(thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Table flags don't match," + "server table has %s " + "and the meta-data file has %s", + dict_tf_to_row_format_string( + relevant_table_flags), + dict_tf_to_row_format_string( + relevant_flags)); + } else { + ib_errf(thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Table flags don't match"); + } return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, @@ -1830,12 +1841,6 @@ PageConverter::update_records( rec_t* rec = m_rec_iter.current(); - /* FIXME: Move out of the loop */ - - if (rec_get_status(rec) == REC_STATUS_NODE_PTR) { - break; - } - ibool deleted = rec_get_deleted_flag(rec, comp); /* For the clustered index we have to adjust the BLOB @@ -1937,6 +1942,10 @@ PageConverter::update_index_page( return(DB_SUCCESS); } + if (!page_is_leaf(block->frame)) { + return (DB_SUCCESS); + } + return(update_records(block)); } @@ -3522,7 +3531,7 @@ row_import_for_mysql( row_import cfg; - memset(&cfg, 0x0, sizeof(cfg)); + memset(static_cast(&cfg), 0x0, sizeof(cfg)); err = row_import_read_cfg(table, trx->mysql_thd, cfg); diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 4fa2ed5cf56..752cf5ba58d 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -225,7 +225,7 @@ row_log_block_allocate( DBUG_ENTER("row_log_block_allocate"); if (log_buf.block == NULL) { log_buf.size = srv_sort_buf_size; - log_buf.block = (byte*) os_mem_alloc_large(&log_buf.size); + log_buf.block = (byte*) os_mem_alloc_large(&log_buf.size, false); DBUG_EXECUTE_IF("simulate_row_log_allocation_failure", if (log_buf.block) os_mem_free_large(log_buf.block, log_buf.size); @@ -2721,7 +2721,15 @@ all_done: while (!trx_is_interrupted(trx)) { mrec = next_mrec; - ut_ad(mrec < mrec_end); + ut_ad(mrec <= mrec_end); + + if (mrec == mrec_end) { + /* We are at the end of the log. + Mark the replay all_done. */ + if (has_index_lock) { + goto all_done; + } + } if (!has_index_lock) { /* We are applying operations from a different diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 346b1616fd0..b000f313d67 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -3672,7 +3672,7 @@ row_merge_build_indexes( block_size = 3 * srv_sort_buf_size; block = static_cast( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size, false)); if (block == NULL) { DBUG_RETURN(DB_OUT_OF_MEMORY); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 9d1fcfc0393..ddde8b8f730 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1972,8 +1972,7 @@ error_exit: doc_ids difference should not exceed FTS_DOC_ID_MAX_STEP value. */ - if (next_doc_id > 1 - && doc_id - next_doc_id >= FTS_DOC_ID_MAX_STEP) { + if (doc_id - next_doc_id >= FTS_DOC_ID_MAX_STEP) { fprintf(stderr, "InnoDB: Doc ID " UINT64PF " is too" " big. Its difference with largest" @@ -5501,6 +5500,18 @@ row_rename_table_for_mysql( goto funct_exit; } + /* Wait for background fts sync to finish */ + for (retry = 1; dict_fts_index_syncing(table); ++retry) { + DICT_BG_YIELD(trx); + if (retry % 100 == 0) { + ib_logf(IB_LOG_LEVEL_INFO, + "Unable to rename table %s to new name" + " %s because FTS sync is running on table." + " Retrying\n", + old_name, new_name); + } + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in updating the dictionary data from system tables. */ @@ -5677,7 +5688,8 @@ row_rename_table_for_mysql( } } - if (dict_table_has_fts_index(table) + if ((dict_table_has_fts_index(table) + || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) && !dict_tables_have_same_db(old_name, new_name)) { err = fts_rename_aux_tables(table, new_name, trx); if (err != DB_TABLE_NOT_FOUND) { diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index 3e79d00b07c..5db61ecdd56 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2759,7 +2759,7 @@ row_sel_field_store_in_mysql_format_func( case DATA_SYS: /* These column types should never be shipped to MySQL. */ ut_ad(0); - + break; case DATA_CHAR: case DATA_FIXBINARY: case DATA_FLOAT: @@ -4775,6 +4775,7 @@ no_gap_lock: prebuilt->new_rec_locks = 1; } err = DB_SUCCESS; + break; case DB_SUCCESS: break; case DB_LOCK_WAIT: diff --git a/storage/xtradb/srv/srv0conc.cc b/storage/xtradb/srv/srv0conc.cc index 63268e3a266..fff21e1ef3e 100644 --- a/storage/xtradb/srv/srv0conc.cc +++ b/storage/xtradb/srv/srv0conc.cc @@ -265,6 +265,7 @@ srv_conc_enter_innodb_with_atomics( notified_mysql = TRUE; } + DEBUG_SYNC_C("user_thread_waiting"); trx->op_info = "sleeping before entering InnoDB"; sleep_in_us = srv_thread_sleep_delay; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index e0e3f995bae..471749e842a 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -157,7 +157,6 @@ use simulated aio we build below with threads. Currently we support native aio on windows and linux */ /* make srv_use_native_aio to be visible for other plugins */ my_bool srv_use_native_aio = TRUE; -UNIV_INTERN my_bool srv_numa_interleave = FALSE; #ifdef __WIN__ /* Windows native condition variables. We use runtime loading / function diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ac634df06dd..a6a3c820813 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2059,7 +2059,8 @@ innobase_start_or_create_for_mysql(void) ib_logf(IB_LOG_LEVEL_INFO, "Initializing buffer pool, size = %.1f%c", size, unit); - err = buf_pool_init(srv_buf_pool_size, srv_buf_pool_instances); + err = buf_pool_init(srv_buf_pool_size, static_cast(srv_numa_interleave), + srv_buf_pool_instances); if (err != DB_SUCCESS) { ib_logf(IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 45f2a1c68aa..db68095d3f5 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -2430,7 +2430,8 @@ trx_get_trx_by_xid_low( /* Invalidate the XID, so that subsequent calls will not find it. */ - memset(&trx->xid, 0, sizeof(trx->xid)); + memset(static_cast(&trx->xid), 0, + sizeof(trx->xid)); trx->xid.formatID = -1; break; } diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index 2ddb35d5ad0..a82e6cdfdcb 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -1313,7 +1313,7 @@ trx_undo_mem_create_at_db_start( /* Read X/Open XA transaction identification if it exists, or set it to NULL. */ - memset(&xid, 0, sizeof(xid)); + memset(static_cast(&xid), 0, sizeof(xid)); xid.formatID = -1; if (xid_exists == TRUE) { From da34c7de5dacac85c4dc1f714bcd7edf3b7fe5f9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 27 Oct 2018 21:05:16 +0200 Subject: [PATCH 075/127] 5.6.42 --- storage/innobase/dict/dict0dict.cc | 7 ++- storage/innobase/fts/fts0fts.cc | 67 +++++++++-------------- storage/innobase/handler/ha_innodb.cc | 45 ++++++++++++++- storage/innobase/handler/handler0alter.cc | 35 +++++++++--- storage/innobase/include/os0file.h | 6 +- storage/innobase/row/row0import.cc | 19 +++++-- storage/innobase/row/row0merge.cc | 17 +++++- storage/innobase/row/row0mysql.cc | 3 + storage/innobase/row/row0sel.cc | 4 +- 9 files changed, 143 insertions(+), 60 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index b2def1edffe..1bbf50791f2 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. This program is free software; you can redistribute it and/or modify it under @@ -3378,7 +3378,10 @@ dict_foreign_find_index( && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, - check_charsets, check_null)) { + check_charsets, check_null) + && (!(index->online_status == + ONLINE_INDEX_ABORTED_DROPPED + ||index->online_status == ONLINE_INDEX_ABORTED))) { return(index); } diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index f3fa6e52d0d..a1f56671fc8 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -868,37 +868,28 @@ fts_drop_index( err = fts_drop_index_tables(trx, index); - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry){ - fts_free(table); - break; - } - DICT_BG_YIELD(trx); - } + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + + fts_free(table); + return(err); } - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry){ - current_doc_id = table->fts->cache->next_doc_id; - first_doc_id = table->fts->cache->first_doc_id; - fts_cache_clear(table->fts->cache); - fts_cache_destroy(table->fts->cache); - table->fts->cache = fts_cache_create(table); - table->fts->cache->next_doc_id = current_doc_id; - table->fts->cache->first_doc_id = first_doc_id; - break; - } - DICT_BG_YIELD(trx); - } + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + + current_doc_id = table->fts->cache->next_doc_id; + first_doc_id = table->fts->cache->first_doc_id; + fts_cache_clear(table->fts->cache); + fts_cache_destroy(table->fts->cache); + table->fts->cache = fts_cache_create(table); + table->fts->cache->next_doc_id = current_doc_id; + table->fts->cache->first_doc_id = first_doc_id; } else { fts_cache_t* cache = table->fts->cache; fts_index_cache_t* index_cache; @@ -908,17 +899,13 @@ fts_drop_index( index_cache = fts_find_index_cache(cache, index); if (index_cache != NULL) { - for(;;) { - bool retry = false; - if (index->index_fts_syncing) { - retry = true; - } - if (!retry && index_cache->words) { - fts_words_free(index_cache->words); - rbt_free(index_cache->words); - break; - } - DICT_BG_YIELD(trx); + while (index->index_fts_syncing + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } + if (index_cache->words) { + fts_words_free(index_cache->words); + rbt_free(index_cache->words); } ib_vector_remove(cache->indexes, *(void**) index_cache); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index eb7fdaaddf4..accebb3f3f4 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1919,6 +1919,11 @@ innobase_get_lower_case_table_names(void) { return(lower_case_table_names); } +/** return one of the tmpdir path +@return tmpdir path*/ +UNIV_INTERN +char* +innobase_mysql_tmpdir(void) { return (mysql_tmpdir); } /** Create a temporary file in the location specified by the parameter path. If the path is null, then it will be created in tmpdir. @@ -10364,6 +10369,7 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t innobase_rename_table( /*==================*/ + THD* thd, /*!< Connection thread handle */ trx_t* trx, /*!< in: transaction */ const char* from, /*!< in: old name of the table */ const char* to) /*!< in: new name of the table */ @@ -10389,6 +10395,37 @@ innobase_rename_table( row_mysql_lock_data_dictionary(trx); + dict_table_t* table = NULL; + table = dict_table_open_on_name(norm_from, TRUE, FALSE, + DICT_ERR_IGNORE_NONE); + + /* Since DICT_BG_YIELD has sleep for 250 milliseconds, + Convert lock_wait_timeout unit from second to 250 milliseconds */ + long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; + if (table != NULL) { + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { + + if (index->type & DICT_FTS) { + /* Found */ + while (index->index_fts_syncing + && !trx_is_interrupted(trx) + && (lock_wait_timeout--) > 0) { + DICT_BG_YIELD(trx); + } + } + } + dict_table_close(table, TRUE, FALSE); + } + + /* FTS sync is in progress. We shall timeout this operation */ + if (lock_wait_timeout < 0) { + error = DB_LOCK_WAIT_TIMEOUT; + row_mysql_unlock_data_dictionary(trx); + DBUG_RETURN(error); + } + /* Transaction must be flagged as a locking transaction or it hasn't been started yet. */ @@ -10498,7 +10535,7 @@ ha_innobase::rename_table( ++trx->will_lock; trx_set_dict_operation(trx, TRX_DICT_OP_INDEX); - error = innobase_rename_table(trx, from, to); + error = innobase_rename_table(thd, trx, from, to); DEBUG_SYNC(thd, "after_innobase_rename_table"); @@ -10544,6 +10581,12 @@ ha_innobase::rename_table( error = DB_ERROR; } + else if (error == DB_LOCK_WAIT_TIMEOUT) { + my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to); + + error = DB_LOCK_WAIT; + } + DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index f945f40fc40..c6f4ea57f71 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -4013,11 +4013,23 @@ oom: table. Either way, we should be seeing and reporting a bogus duplicate key error. */ dup_key = NULL; - } else { - DBUG_ASSERT(prebuilt->trx->error_key_num - < ha_alter_info->key_count); + } else if (prebuilt->trx->error_key_num == 0) { dup_key = &ha_alter_info->key_info_buffer[ prebuilt->trx->error_key_num]; + } else { + /* Check if there is generated cluster index column */ + if (ctx->num_to_add_index > ha_alter_info->key_count) { + DBUG_ASSERT(prebuilt->trx->error_key_num + <= ha_alter_info->key_count); + dup_key = &ha_alter_info->key_info_buffer[ + prebuilt->trx->error_key_num - 1]; + } + else { + DBUG_ASSERT(prebuilt->trx->error_key_num + < ha_alter_info->key_count); + dup_key = &ha_alter_info->key_info_buffer[ + prebuilt->trx->error_key_num]; + } } print_keydup_error(altered_table, dup_key, MYF(0)); break; @@ -4938,11 +4950,20 @@ commit_try_rebuild( FTS_DOC_ID. */ dup_key = NULL; } else { - DBUG_ASSERT(err_key < - ha_alter_info->key_count); - dup_key = &ha_alter_info - ->key_info_buffer[err_key]; + if (ctx->num_to_add_index > ha_alter_info->key_count) { + DBUG_ASSERT(err_key <= + ha_alter_info->key_count); + dup_key = &ha_alter_info + ->key_info_buffer[err_key - 1]; + } + else { + DBUG_ASSERT(err_key < + ha_alter_info->key_count); + dup_key = &ha_alter_info + ->key_info_buffer[err_key]; + } } + print_keydup_error(altered_table, dup_key, MYF(0)); DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 76a389f9051..338fa30204c 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1,6 +1,6 @@ /*********************************************************************** -Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. Portions of this file contain modifications contributed and copyrighted @@ -1319,6 +1319,10 @@ os_file_get_status( file can be opened in RW mode */ #if !defined(UNIV_HOTBACKUP) + +/** return one of the tmpdir path + @return tmpdir path*/ +char *innobase_mysql_tmpdir(void); /** Create a temporary file in the location specified by the parameter path. If the path is null, then it will be created in tmpdir. @param[in] path location for creating temporary file diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index cd3f88c240e..c4f6eed4aad 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1337,11 +1337,20 @@ row_import::match_schema( /* Do some simple checks. */ if (m_flags != m_table->flags) { - ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%lx " - "and the meta-data file has 0x%lx", - (ulong) m_table->n_cols, (ulong) m_flags); - + if (dict_tf_to_row_format_string(m_flags) != + dict_tf_to_row_format_string(m_table->flags)) { + ib_errf(thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Table flags don't match," + "server table has %s " + "and the meta-data file has %s", + dict_tf_to_row_format_string(m_table->flags), + dict_tf_to_row_format_string(m_flags)); + } else { + ib_errf(thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Table flags don't match"); + } return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 4a6fb763da6..002700a592c 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2005, 2018, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3116,15 +3116,27 @@ row_merge_file_create_low( const char* path) { int fd; + char filename[] = "Innodb Merge Temp File\0"; + char* filepath = NULL; + int path_len; + if (path == NULL) { + path = innobase_mysql_tmpdir(); + } #ifdef UNIV_PFS_IO /* This temp file open does not go through normal file APIs, add instrumentation to register with performance schema */ + path_len = strlen(path) + sizeof "/" + strlen(filename)+1; + filepath = static_cast(mem_alloc(path_len)); + memcpy(filepath,path,strlen(path)); + ut_snprintf(filepath + strlen(path),path_len - strlen(path), + "%c%s",'/',filename); struct PSI_file_locker* locker = NULL; + PSI_file_locker_state state; locker = PSI_FILE_CALL(get_thread_file_name_locker)( &state, innodb_file_temp_key, PSI_FILE_OPEN, - "Innodb Merge Temp File", &locker); + filepath, &locker); if (locker != NULL) { PSI_FILE_CALL(start_file_open_wait)(locker, __FILE__, @@ -3137,6 +3149,7 @@ row_merge_file_create_low( PSI_FILE_CALL(end_file_open_wait_and_bind_to_descriptor)( locker, fd); } + mem_free(filepath); #endif if (fd < 0) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index cfd640ed4d5..3782d815e9e 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -5066,6 +5066,9 @@ row_rename_table_for_mysql( " = TO_BINARY(:old_table_name);\n" "END;\n" , FALSE, trx); + if (err != DB_SUCCESS) { + goto end; + } } else if (n_constraints_to_drop > 0) { /* Drop some constraints of tmp tables. */ diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 706b5436981..8c61e959a4d 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -4368,7 +4368,7 @@ rec_loop: passed to InnoDB when there is no ICP and number of loops in row_search_for_mysql for rows found but not reporting due to search views etc. */ - if (prev_rec != NULL + if (prev_rec != NULL && !prebuilt->innodb_api && prebuilt->mysql_handler->end_range != NULL && prebuilt->idx_cond == NULL && end_loop >= 100) { From a9a0d0c3729ad3c6e295f68b6d4d5552b999a2e5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 27 Oct 2018 21:06:41 +0200 Subject: [PATCH 076/127] 5.6.42 --- mysql-test/suite/perfschema/r/relaylog.result | 14 +++++++------- storage/perfschema/unittest/pfs-t.cc | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/perfschema/r/relaylog.result b/mysql-test/suite/perfschema/r/relaylog.result index 2b11019799b..9ff9ae86e69 100644 --- a/mysql-test/suite/perfschema/r/relaylog.result +++ b/mysql-test/suite/perfschema/r/relaylog.result @@ -23,7 +23,7 @@ from performance_schema.file_summary_by_instance where file_name like "%master-%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE master-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -master-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE +master-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE select * from performance_schema.file_summary_by_instance where file_name like "%slave-%" order by file_name; FILE_NAME EVENT_NAME OBJECT_INSTANCE_BEGIN COUNT_STAR SUM_TIMER_WAIT MIN_TIMER_WAIT AVG_TIMER_WAIT MAX_TIMER_WAIT COUNT_READ SUM_TIMER_READ MIN_TIMER_READ AVG_TIMER_READ MAX_TIMER_READ SUM_NUMBER_OF_BYTES_READ COUNT_WRITE SUM_TIMER_WRITE MIN_TIMER_WRITE AVG_TIMER_WRITE MAX_TIMER_WRITE SUM_NUMBER_OF_BYTES_WRITE COUNT_MISC SUM_TIMER_MISC MIN_TIMER_MISC AVG_TIMER_MISC MAX_TIMER_MISC @@ -39,7 +39,7 @@ from performance_schema.file_summary_by_instance where event_name like "%binlog%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE master-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -master-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE +master-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE select EVENT_NAME, if (count_read > 0, "MANY", "NONE") as COUNT_READ, @@ -50,7 +50,7 @@ from performance_schema.file_summary_by_event_name where event_name like "%binlog%" order by event_name; EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE wait/io/file/sql/binlog MANY MANY MANY MANY -wait/io/file/sql/binlog_index MANY NONE NONE NONE +wait/io/file/sql/binlog_index MANY NONE MANY NONE select EVENT_NAME, if (count_star > 0, "MANY", "NONE") as COUNT_STAR @@ -114,10 +114,10 @@ where file_name like "%slave-%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE slave-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -slave-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE +slave-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE slave-relay-bin.000001 wait/io/file/sql/relaylog MANY MANY MANY MANY slave-relay-bin.000002 wait/io/file/sql/relaylog MANY MANY MANY MANY -slave-relay-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE +slave-relay-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE "Expect a slave binlog + binlog_index" select substring(file_name, locate("slave-", file_name)) as FILE_NAME, @@ -130,8 +130,8 @@ from performance_schema.file_summary_by_instance where event_name like "%binlog%" order by file_name; FILE_NAME EVENT_NAME COUNT_READ COUNT_WRITE SUM_NUMBER_OF_BYTES_READ SUM_NUMBER_OF_BYTES_WRITE slave-bin.000001 wait/io/file/sql/binlog MANY MANY MANY MANY -slave-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE -slave-relay-bin.index wait/io/file/sql/binlog_index NONE NONE NONE NONE +slave-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE +slave-relay-bin.index wait/io/file/sql/binlog_index MANY NONE MANY NONE select EVENT_NAME, if (count_read > 0, "MANY", "NONE") as COUNT_READ, diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 7861c74a401..d50ae49435f 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -899,6 +899,7 @@ void test_init_disabled() psi->create_file(file_key_A, "foo-instrumented", (File) 12); file_A1= lookup_file_by_name("foo-instrumented"); ok(file_A1 != NULL, "file_A1 instrumented"); + destroy_file(reinterpret_cast(psi->get_thread()), file_A1); /* broken key + enabled T-1: no instrumentation */ @@ -1150,6 +1151,8 @@ void test_locker_disabled() psi->create_file(file_key_A, "foo", (File) 12); file_A1= (PSI_file*) lookup_file_by_name("foo"); ok(file_A1 != NULL, "instrumented"); + destroy_file(reinterpret_cast(psi->get_thread()), + reinterpret_cast(file_A1)); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); From 411a2540ee043a6805555387df1b26b48b483eef Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 28 Oct 2018 10:09:58 +0100 Subject: [PATCH 077/127] CONNECT: don't mix bundled zlib and system libxml2 System libxml2 uses system zlib, it might conflicts with the bundled. In particular, on centos5 old system zlib conflicts with the newer (after c54271723c6f) bundled zlib which causes CONNECT to crash on xml tests. --- storage/connect/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 5695e981413..d985df9eaa7 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -126,6 +126,7 @@ IF(CONNECT_WITH_LIBXML2) FIND_PACKAGE(LibXml2) IF (LIBXML2_FOUND) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR}) + SET(ZLIB_LIBRARY "") # see ZLIB_INCLUDE_DIR below SET(XML_LIBRARY ${LIBXML2_LIBRARIES}) SET(CONNECT_SOURCES ${CONNECT_SOURCES} libdoc.cpp libdoc.h) add_definitions(-DLIBXML2_SUPPORT) @@ -343,6 +344,14 @@ IF(NOT TARGET connect) RETURN() ENDIF() +# Don't link with bundled zlib and systel libxml2 at the same time. +# System libxml2 uses system zlib, might conflict with the bundled one. +IF (XML_LIBRARY AND BUILD_BUNDLED_ZLIB) + GET_PROPERTY(INCS TARGET connect PROPERTY INCLUDE_DIRECTORIES) + LIST(REMOVE_ITEM INCS ${ZLIB_INCLUDE_DIR}) + SET_PROPERTY(TARGET connect PROPERTY INCLUDE_DIRECTORIES ${INCS}) +ENDIF() + IF(WIN32) IF (libmongoc-1.0_FOUND) SET_TARGET_PROPERTIES(connect PROPERTIES LINK_FLAGS From 70e567f576319fc98ff76552fe2e688aeb46fa2c Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 7 May 2018 22:43:43 +0200 Subject: [PATCH 078/127] Squashed commit of connect/10.0: commit 02d8c21380f Author: Olivier Bertrand Date: Sat Oct 6 16:27:13 2018 +0200 - Fix truncating error messages on first unrecognized latin1 character modified: storage/connect/ha_connect.cc - Fix MDEV-17343 Reject multi-table UPDATE/DELETE commands that crash on some systems modified: storage/connect/ha_connect.cc modified: storage/connect/tabext.cpp - Try to fix failing tests (MariaDB version 10.0 only) modified: storage/connect/mysql-test/connect/disabled.def - Typo modified: storage/connect/global.h commit f83caed8569 Author: Olivier Bertrand Date: Tue Sep 25 15:49:26 2018 +0200 - Try to fix failing tests (MariaDB version 10.0 only) modified: storage/connect/mysql-test/connect/disabled.def modified: storage/connect/mysql-test/connect/r/grant2.result modified: storage/connect/mysql-test/connect/r/infoschema2-9739.result modified: storage/connect/mysql-test/connect/r/mysql_exec.result commit 9fd6f178846 Author: Olivier Bertrand Date: Sun Sep 23 19:45:59 2018 +0200 - Implement the CHECK TABLE statement and accept REPAIR and ANALYZE modified: storage/connect/connect.cc modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/tabjdbc.cpp modified: storage/connect/tabmysql.cpp modified: storage/connect/tabodbc.cpp - MDEV-17212: Test if NumResultCols is implemented by the data source modified: storage/connect/odbconn.cpp - Change error type in Optimize modified: storage/connect/ha_connect.cc - Update version date modified: storage/connect/ha_connect.cc - Record new result from odbc_postgresql.test modified: storage/connect/mysql-test/connect/r/odbc_postgresql.result commit d8cf51319e1 Author: Olivier Bertrand Date: Wed Aug 8 12:18:52 2018 +0200 - Comment out failing Cyrillic test in xml2.test modified: storage/connect/mysql-test/connect/r/xml2.result modified: storage/connect/mysql-test/connect/t/xml2.test commit 9df49e21f9e Author: Olivier Bertrand Date: Tue Aug 7 15:01:06 2018 +0200 - Fix MDEV-16672 Connect: Warnings with 10.0 filamtxt.cpp: DOSFAM::RenameTempFile: Change sprintf to snprintf. filamvct.cpp: VECFAM::RenameTempFile: Change sprintf to snprintf. javaconn.cpp: Add JAVAConn::GetUTFString function. Use it instead of env->GetStringUTFChars. Fix wrong identation. javaconn.h: Add GetUTFString declaration. jdbconn.cpp: Use GetUTFString function instead of env->GetStringUTFChars. jmgoconn.cpp: Use GetUTFString function instead of env->GetStringUTFChars. Fix wrong identation. jsonudf.cpp: change 139 to BMX line 4631. tabjmg.cpp: Add ReleaseStringUTF. Fix wrong identation. tabpivot.cpp: Fix wrong identation. tabutil.cpp: TDBPRX::GetSubTable: Change sprintf to snprintf. modified: storage/connect/filamtxt.cpp modified: storage/connect/filamvct.cpp modified: storage/connect/javaconn.cpp modified: storage/connect/javaconn.h modified: storage/connect/jdbconn.cpp modified: storage/connect/jmgoconn.cpp modified: storage/connect/jsonudf.cpp modified: storage/connect/tabjmg.cpp modified: storage/connect/tabpivot.cpp modified: storage/connect/tabutil.cpp - Fix MDEV-16895 CONNECT engine's get_error_message can cause buffer overflow and server crash with long queries ha_connect_cc: Update version. get_error_message: Remove charset conversion. modified: storage/connect/ha_connect.cc - Fix a server crash on inserting bigint to a JDBC table JDBConn::SetUUID: Suppress check on ctyp that causes a server crash because ctyp can be negative and this triggers an DEBUG_ASSERT on return. modified: storage/connect/jdbconn.cpp - Delete an assert(qrp) from JCATPARM *AllocCatInfo that is called with qrp=NULL from JDBConn::SetUUID. Also delete a clone of this function that was duplicated in javaconn.cpp. modified: storage/connect/javaconn.cpp modified: storage/connect/jdbconn.cpp - Update some disabled tests and results to avoid failure modified: storage/connect/mysql-test/connect/r/jdbc.result modified: storage/connect/mysql-test/connect/r/json_java_2.result modified: storage/connect/mysql-test/connect/r/json_java_3.result modified: storage/connect/mysql-test/connect/r/mongo_java_2.result modified: storage/connect/mysql-test/connect/r/mongo_java_3.result modified: storage/connect/mysql-test/connect/t/json_java_2.test modified: storage/connect/mysql-test/connect/t/json_java_3.test modified: storage/connect/mysql-test/connect/t/mongo_java_2.test modified: storage/connect/mysql-test/connect/t/mongo_java_3.test commit 415273eb193 Author: Olivier Bertrand Date: Thu Jun 28 19:37:49 2018 +0200 - Fix MDEV-16167 Cannot insert unsigned values into a VEC table modified: storage/connect/filamvct.cpp modified: storage/connect/tabvct.cpp commit 9ffcb68a9f2 Author: Olivier Bertrand Date: Mon May 7 22:43:43 2018 +0200 - Fix MDEV-15735 CONNECT [filamtxt.cpp:429]: Suspicious condition modified: storage/connect/filamtxt.cpp - Fix compiler warnings modified: storage/connect/domdoc.cpp modified: storage/connect/ha_connect.cc modified: storage/connect/json.h modified: storage/connect/jsonudf.cpp modified: storage/connect/tabext.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h - Miscelleanous from 10.3 modified: storage/connect/ha_connect.cc modified: storage/connect/mycat.cc modified: storage/connect/user_connect.cc --- storage/connect/connect.cc | 2 +- storage/connect/domdoc.cpp | 4 +- storage/connect/filamtxt.cpp | 6 +- storage/connect/filamvct.cpp | 28 +-- storage/connect/global.h | 4 +- storage/connect/ha_connect.cc | 148 +++++++++----- storage/connect/ha_connect.h | 6 +- storage/connect/javaconn.cpp | 41 ++-- storage/connect/javaconn.h | 1 + storage/connect/jdbconn.cpp | 40 ++-- storage/connect/jmgoconn.cpp | 6 +- storage/connect/json.h | 3 + storage/connect/jsonudf.cpp | 6 +- storage/connect/mycat.cc | 15 +- .../connect/mysql-test/connect/disabled.def | 25 +-- .../mysql-test/connect/r/grant2.result | 2 +- .../connect/r/infoschema2-9739.result | 2 +- .../connect/mysql-test/connect/r/jdbc.result | 3 +- .../mysql-test/connect/r/json_java_2.result | 1 - .../mysql-test/connect/r/json_java_3.result | 1 - .../mysql-test/connect/r/mongo_java_2.result | 1 - .../mysql-test/connect/r/mongo_java_3.result | 1 - .../mysql-test/connect/r/mysql_exec.result | 6 + .../connect/r/odbc_postgresql.result | 18 +- .../connect/mysql-test/connect/r/xml2.result | 31 --- .../mysql-test/connect/t/json_java_2.test | 2 + .../mysql-test/connect/t/json_java_3.test | 2 + .../mysql-test/connect/t/mongo_java_2.test | 2 + .../mysql-test/connect/t/mongo_java_3.test | 2 + .../connect/mysql-test/connect/t/xml2.test | 36 ++-- storage/connect/odbconn.cpp | 10 +- storage/connect/tabext.cpp | 8 +- storage/connect/tabjdbc.cpp | 9 +- storage/connect/tabjmg.cpp | 11 +- storage/connect/tabjson.h | 1 + storage/connect/tabmysql.cpp | 11 +- storage/connect/tabodbc.cpp | 13 +- storage/connect/tabpivot.cpp | 180 +++++++++--------- storage/connect/tabutil.cpp | 2 +- storage/connect/tabvct.cpp | 6 +- storage/connect/user_connect.cc | 1 + 41 files changed, 356 insertions(+), 341 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index 39123b18c59..21bca637eab 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -254,7 +254,7 @@ bool CntOpenTable(PGLOBAL g, PTDB tdbp, MODE mode, char *c1, char *c2, try { if (!c1) { - if (mode == MODE_INSERT) +// if (mode == MODE_INSERT) or CHECK TABLE // Allocate all column blocks for that table tdbp->ColDB(g, NULL, 0); diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index ba8eb829abd..9ae34a3b9ef 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -84,8 +84,8 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) : XMLDOCUMENT(nsl, nsdf, enc) { assert (!fp || fp->Type == TYPE_FB_XML); - Docp = (fp) ? ((PXBLOCK)fp)->Docp : NULL; - Nlist = NULL; + Docp = (fp) ? ((PXBLOCK)fp)->Docp : (MSXML2::IXMLDOMDocumentPtr)NULL; + Nlist = NULL; Hr = 0; } // end of DOMDOC constructor diff --git a/storage/connect/filamtxt.cpp b/storage/connect/filamtxt.cpp index 7c222eb3c80..ca48fc765a1 100644 --- a/storage/connect/filamtxt.cpp +++ b/storage/connect/filamtxt.cpp @@ -427,7 +427,7 @@ int TXTFAM::DeleteSortedRows(PGLOBAL g) for (i = 0; i < Posar->GetNval(); i++) { if ((irc = InitDelete(g, Posar->GetIntValue(ix[i]), - Sosar->GetIntValue(ix[i])) == RC_FX)) + Sosar->GetIntValue(ix[i]))) == RC_FX) goto err; // Now delete the sorted rows @@ -1173,11 +1173,11 @@ int DOSFAM::RenameTempFile(PGLOBAL g) remove(filetemp); // May still be there from previous error if (rename(filename, filetemp)) { // Save file for security - sprintf(g->Message, MSG(RENAME_ERROR), + snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR), filename, filetemp, strerror(errno)); throw 51; } else if (rename(tempname, filename)) { - sprintf(g->Message, MSG(RENAME_ERROR), + snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR), tempname, filename, strerror(errno)); rc = rename(filetemp, filename); // Restore saved file throw 52; diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp index dd827d084fa..6d0779b150a 100755 --- a/storage/connect/filamvct.cpp +++ b/storage/connect/filamvct.cpp @@ -348,7 +348,7 @@ int VCTFAM::Cardinality(PGLOBAL g) } // endif split - return (Block) ? ((Block - 1) * Nrec + Last) : 0; + return (Block) ? ((Block - 1) * Nrec + Last) : 0; } // end of Cardinality /***********************************************************************/ @@ -510,7 +510,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, + cp->IsUnsigned()); return InitInsert(g); // Initialize inserting } else { @@ -544,7 +545,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode @@ -1511,7 +1513,8 @@ bool VCMFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -2062,7 +2065,7 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, To_Bufs[cp->Index - 1], cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); return InitInsert(g); } else { @@ -2111,7 +2114,8 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } // endif mode @@ -2449,11 +2453,11 @@ int VECFAM::RenameTempFile(PGLOBAL g) remove(filetemp); // May still be there from previous error if (rename(filename, filetemp)) { // Save file for security - sprintf(g->Message, MSG(RENAME_ERROR), + snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR), filename, filetemp, strerror(errno)); rc = RC_FX; } else if (rename(tempname, filename)) { - sprintf(g->Message, MSG(RENAME_ERROR), + snprintf(g->Message, MAX_STR, MSG(RENAME_ERROR), tempname, filename, strerror(errno)); rc = rename(filetemp, filename); // Restore saved file rc = RC_FX; @@ -2882,7 +2886,8 @@ bool VMPFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -3664,7 +3669,7 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); InitInsert(g); // Initialize inserting @@ -3712,7 +3717,8 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode diff --git a/storage/connect/global.h b/storage/connect/global.h index 472d09408c3..36e8a311124 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -1,7 +1,7 @@ /***********************************************************************/ /* GLOBAL.H: Declaration file used by all CONNECT implementations. */ /* (C) Copyright MariaDB Corporation Ab */ -/* Author Olivier Bertrand 1993-2017 */ +/* Author Olivier Bertrand 1993-2018 */ /***********************************************************************/ /***********************************************************************/ @@ -192,7 +192,7 @@ typedef struct _global { /* Global structure */ PACTIVITY Activityp; char Message[MAX_STR]; ulong More; /* Used by jsonudf */ - int Createas; /* To pass info to created table */ + int Createas; /* To pass multi to ext tables */ void *Xchk; /* indexes in create/alter */ short Alchecked; /* Checked for ALTER */ short Mrr; /* True when doing mrr */ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 2b3db3753d5..bf890724d5e 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -107,13 +107,9 @@ #define MYSQL_SERVER 1 #define DONT_DEFINE_VOID -#include "sql_class.h" -#include "create_options.h" -#include "mysql_com.h" -#include "field.h" +#include #include "sql_parse.h" #include "sql_base.h" -#include #include "sql_partition.h" #undef OFFSET @@ -174,9 +170,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.06.0007 March 11, 2018"; + char version[]= "Version 1.06.0008 October 06, 2018"; #if defined(__WIN__) - char compver[]= "Version 1.06.0007 " __DATE__ " " __TIME__; + char compver[]= "Version 1.06.0008 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -432,7 +428,7 @@ handlerton *connect_hton= NULL; uint GetTraceValue(void) {return (uint)(connect_hton ? THDVAR(current_thd, xtrace) : 0);} bool ExactInfo(void) {return THDVAR(current_thd, exact_info);} -bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);} +static bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);} USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);} int GetConvSize(void) {return THDVAR(current_thd, conv_size);} TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);} @@ -1781,12 +1777,14 @@ bool ha_connect::CheckVirtualIndex(TABLE_SHARE *s) bool ha_connect::IsPartitioned(void) { - if (tshp) +#ifdef WITH_PARTITION_STORAGE_ENGINE + if (tshp) return tshp->partition_info_str_len > 0; else if (table && table->part_info) return true; else - return false; +#endif + return false; } // end of IsPartitioned @@ -2811,7 +2809,7 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) htrc("Cond type=%d\n", cond->type()); if (cond->type() == COND::COND_ITEM) { - char *pb0, *pb1, *pb2, *ph0, *ph1, *ph2; + char *pb0, *pb1, *pb2, *ph0= 0, *ph1= 0, *ph2= 0; bool bb = false, bh = false; Item_cond *cond_item= (Item_cond *)cond; @@ -3292,6 +3290,58 @@ ha_rows ha_connect::records() } // end of records +int ha_connect::check(THD* thd, HA_CHECK_OPT* check_opt) +{ + int rc = HA_ADMIN_OK; + PGLOBAL g = ((table && table->in_use) ? GetPlug(table->in_use, xp) : + (xp) ? xp->g : NULL); + DBUG_ENTER("ha_connect::check"); + + if (!g || !table || xmod != MODE_READ) + DBUG_RETURN(HA_ADMIN_INTERNAL_ERROR); + + // Do not close the table if it was opened yet (possible?) + if (IsOpened()) { + if (IsPartitioned() && CheckColumnList(g)) // map can have been changed + rc = HA_ADMIN_CORRUPT; + else if (tdbp->OpenDB(g)) // Rewind table + rc = HA_ADMIN_CORRUPT; + + } else if (xp->CheckQuery(valid_query_id)) { + tdbp = NULL; // Not valid anymore + + if (OpenTable(g, false)) + rc = HA_ADMIN_CORRUPT; + + } else // possible? + DBUG_RETURN(HA_ADMIN_INTERNAL_ERROR); + + if (rc == HA_ADMIN_OK) { + TABTYPE type = GetTypeID(GetStringOption("Type", "*")); + + if (IsFileType(type)) { + if (check_opt->flags & T_MEDIUM) { + // TO DO + do { + if ((rc = CntReadNext(g, tdbp)) == RC_FX) + break; + + } while (rc != RC_EF); + + rc = (rc == RC_EF) ? HA_ADMIN_OK : HA_ADMIN_CORRUPT; + } else if (check_opt->flags & T_EXTEND) { + // TO DO + } // endif's flags + + } // endif file type + + } else + PushWarning(g, thd, 1); + + DBUG_RETURN(rc); +} // end of check + + /** Return an error message specific to this handler. @@ -3305,23 +3355,16 @@ bool ha_connect::get_error_message(int error, String* buf) { DBUG_ENTER("ha_connect::get_error_message"); - if (xp && xp->g) { - PGLOBAL g= xp->g; - char msg[3072]; // MAX_STR * 3 - uint dummy_errors; - uint32 len= copy_and_convert(msg, strlen(g->Message) * 3, - system_charset_info, - g->Message, strlen(g->Message), - &my_charset_latin1, - &dummy_errors); + if (xp && xp->g) { + PGLOBAL g = xp->g; - if (trace(1)) - htrc("GEM(%d): len=%u %s\n", error, len, g->Message); + if (trace(1)) + htrc("GEM(%d): %s\n", error, g->Message); - msg[len]= '\0'; - buf->copy(msg, (uint)strlen(msg), system_charset_info); - } else - buf->copy("Cannot retrieve msg", 19, system_charset_info); + buf->append(ErrConvString(g->Message, strlen(g->Message), + &my_charset_latin1).ptr()); + } else + buf->append("Cannot retrieve error message"); DBUG_RETURN(false); } // end of get_error_message @@ -3434,7 +3477,7 @@ int ha_connect::optimize(THD* thd, HA_CHECK_OPT*) push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); rc = 0; } else - rc = HA_ERR_INTERNAL_ERROR; + rc = HA_ERR_CRASHED_ON_USAGE; // Table must be repaired } // endif rc @@ -3450,6 +3493,9 @@ int ha_connect::optimize(THD* thd, HA_CHECK_OPT*) rc = HA_ERR_INTERNAL_ERROR; } // end catch + if (rc) + my_message(ER_WARN_DATA_OUT_OF_RANGE, g->Message, MYF(0)); + return rc; } // end of optimize @@ -4511,14 +4557,16 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, // case SQLCOM_REPLACE_SELECT: // newmode= MODE_UPDATE; // To be checked // break; - case SQLCOM_DELETE: - case SQLCOM_DELETE_MULTI: + case SQLCOM_DELETE_MULTI: + *cras = true; + case SQLCOM_DELETE: case SQLCOM_TRUNCATE: newmode= MODE_DELETE; break; - case SQLCOM_UPDATE: case SQLCOM_UPDATE_MULTI: - newmode= MODE_UPDATE; + *cras = true; + case SQLCOM_UPDATE: + newmode= MODE_UPDATE; break; case SQLCOM_SELECT: case SQLCOM_OPTIMIZE: @@ -4543,8 +4591,10 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, newmode= MODE_ANY; break; // } // endif partitioned - - default: + case SQLCOM_REPAIR: // TODO implement it + newmode = MODE_UPDATE; + break; + default: htrc("Unsupported sql_command=%d\n", thd_sql_command(thd)); strcpy(g->Message, "CONNECT Unsupported command"); my_message(ER_NOT_ALLOWED_COMMAND, g->Message, MYF(0)); @@ -4556,17 +4606,18 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, switch (thd_sql_command(thd)) { case SQLCOM_CREATE_TABLE: *chk= true; - *cras= true; + break; + case SQLCOM_UPDATE_MULTI: + case SQLCOM_DELETE_MULTI: + *cras= true; case SQLCOM_INSERT: case SQLCOM_LOAD: case SQLCOM_INSERT_SELECT: // case SQLCOM_REPLACE: // case SQLCOM_REPLACE_SELECT: case SQLCOM_DELETE: - case SQLCOM_DELETE_MULTI: case SQLCOM_TRUNCATE: case SQLCOM_UPDATE: - case SQLCOM_UPDATE_MULTI: case SQLCOM_SELECT: case SQLCOM_OPTIMIZE: case SQLCOM_SET_OPTION: @@ -4594,8 +4645,9 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, break; // } // endif partitioned - case SQLCOM_CHECK: // TODO implement it - case SQLCOM_END: // Met in procedures: IF(EXISTS(SELECT... + case SQLCOM_CHECK: // TODO implement it + case SQLCOM_ANALYZE: // TODO implement it + case SQLCOM_END: // Met in procedures: IF(EXISTS(SELECT... newmode= MODE_READ; break; default: @@ -4877,7 +4929,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) #endif // 0 if (cras) - g->Createas= 1; // To tell created table to ignore FLAG + g->Createas= 1; // To tell external tables of a multi-table command if (trace(1)) { #if 0 @@ -5568,7 +5620,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } // endif p } else if (ttp != TAB_ODBC || !(fnc & (FNC_TABLE | FNC_COL))) - tab = table_s->table_name.str; // Default value + tab = (char*)table_s->table_name.str; // Default value } // endif tab @@ -6200,7 +6252,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, LEX_STRING cnc = table_arg->s->connect_string; #if defined(WITH_PARTITION_STORAGE_ENGINE) partition_info *part_info= table_arg->part_info; -#endif // WITH_PARTITION_STORAGE_ENGINE +#else // !WITH_PARTITION_STORAGE_ENGINE +#define part_info 0 +#endif // !WITH_PARTITION_STORAGE_ENGINE xp= GetUser(thd, xp); PGLOBAL g= xp->g; @@ -6301,9 +6355,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, // fall through case TAB_MYSQL: -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (!part_info) -#endif // WITH_PARTITION_STORAGE_ENGINE {const char *src= options->srcdef; PCSZ host, db, tab= options->tabname; int port; @@ -6567,7 +6619,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, } else lwt[i]= tolower(options->type[i]); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) { char *p; @@ -6577,7 +6628,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, strcat(strcat(strcpy(buf, p), "."), lwt); *p= 0; } else { -#endif // WITH_PARTITION_STORAGE_ENGINE strcat(strcat(strcpy(buf, GetTableName()), "."), lwt); sprintf(g->Message, "No file name. Table will use %s", buf); @@ -6585,9 +6635,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) } // endif part_info -#endif // WITH_PARTITION_STORAGE_ENGINE PlugSetPath(fn, buf, dbpath); @@ -6652,11 +6700,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, "Unexpected command in create, please contact CONNECT team"); -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info && !inward) strncpy(partname, decode(g, strrchr(name, '#') + 1), sizeof(partname) - 1); // strcpy(partname, part_info->curr_part_elem->partition_name); -#endif // WITH_PARTITION_STORAGE_ENGINE if (g->Alchecked == 0 && (!IsFileType(type) || FileExists(options->filename, false))) { @@ -6692,12 +6738,10 @@ int ha_connect::create(const char *name, TABLE *table_arg, my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); rc = HA_ERR_INTERNAL_ERROR; } else if (cat) { -#if defined(WITH_PARTITION_STORAGE_ENGINE) if (part_info) strncpy(partname, decode(g, strrchr(name, (inward ? slash : '#')) + 1), sizeof(partname) - 1); -#endif // WITH_PARTITION_STORAGE_ENGINE if ((rc= optimize(table->in_use, NULL))) { htrc("Create rc=%d %s\n", rc, g->Message); @@ -7266,7 +7310,7 @@ maria_declare_plugin(connect) 0x0107, /* version number (1.05) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.06.0007", /* string version */ + "1.06.0008", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 4c5bf5856cc..07b1c2d2e7d 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -347,11 +347,7 @@ PFIL CondFilter(PGLOBAL g, Item *cond); //PFIL CheckFilter(PGLOBAL g); /** admin commands - called from mysql_admin_table */ -virtual int check(THD* thd, HA_CHECK_OPT* check_opt) -{ - // TODO: implement it - return HA_ADMIN_OK; // Just to avoid error message with checktables -} // end of check +virtual int check(THD* thd, HA_CHECK_OPT* check_opt); /** Number of rows in table. It will only be called if diff --git a/storage/connect/javaconn.cpp b/storage/connect/javaconn.cpp index ec10b125737..f919344f20b 100644 --- a/storage/connect/javaconn.cpp +++ b/storage/connect/javaconn.cpp @@ -81,29 +81,6 @@ GETDEF JAVAConn::GetDefaultJavaVMInitArgs = NULL; #define DEBUG_ONLY(f) ((void)0) #endif // !_DEBUG -/***********************************************************************/ -/* Allocate the structure used to refer to the result set. */ -/***********************************************************************/ -static JCATPARM *AllocCatInfo(PGLOBAL g, JCATINFO fid, PCSZ db, - PCSZ tab, PQRYRES qrp) -{ - JCATPARM *cap; - -#if defined(_DEBUG) - assert(qrp); -#endif - - if ((cap = (JCATPARM *)PlgDBSubAlloc(g, NULL, sizeof(JCATPARM)))) { - memset(cap, 0, sizeof(JCATPARM)); - cap->Id = fid; - cap->Qrp = qrp; - cap->DB = db; - cap->Tab = tab; - } // endif cap - - return cap; -} // end of AllocCatInfo - /***********************************************************************/ /* JAVAConn construction/destruction. */ /***********************************************************************/ @@ -138,6 +115,16 @@ JAVAConn::JAVAConn(PGLOBAL g, PCSZ wrapper) // EndCom(); // } // end of ~JAVAConn +char *JAVAConn::GetUTFString(jstring s) +{ + char *str; + const char *utf = env->GetStringUTFChars(s, nullptr); + + str = PlugDup(m_G, utf); + env->ReleaseStringUTFChars(s, utf); + env->DeleteLocalRef(s); + return str; +} // end of GetUTFString /***********************************************************************/ /* Screen for errors. */ @@ -152,17 +139,15 @@ bool JAVAConn::Check(jint rc) "toString", "()Ljava/lang/String;"); if (exc != nullptr && tid != nullptr) { - jstring s = (jstring)env->CallObjectMethod(exc, tid); - const char *utf = env->GetStringUTFChars(s, NULL); - env->DeleteLocalRef(s); - Msg = PlugDup(m_G, utf); + s = (jstring)env->CallObjectMethod(exc, tid); + Msg = GetUTFString(s); } else Msg = "Exception occured"; env->ExceptionClear(); } else if (rc < 0) { s = (jstring)env->CallObjectMethod(job, errid); - Msg = (char*)env->GetStringUTFChars(s, NULL); + Msg = GetUTFString(s); } else Msg = NULL; diff --git a/storage/connect/javaconn.h b/storage/connect/javaconn.h index 54b7c4e92b7..73812f6ab3b 100644 --- a/storage/connect/javaconn.h +++ b/storage/connect/javaconn.h @@ -90,6 +90,7 @@ public: // Java operations protected: + char *GetUTFString(jstring s); bool gmID(PGLOBAL g, jmethodID& mid, const char *name, const char *sig); bool Check(jint rc = 0); diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index 7e42ca126d0..e54eeb0fcdd 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -314,10 +314,6 @@ static JCATPARM *AllocCatInfo(PGLOBAL g, JCATINFO fid, PCSZ db, { JCATPARM *cap; -#if defined(_DEBUG) - assert(qrp); -#endif - if ((cap = (JCATPARM *)PlgDBSubAlloc(g, NULL, sizeof(JCATPARM)))) { memset(cap, 0, sizeof(JCATPARM)); cap->Id = fid; @@ -699,20 +695,13 @@ bool JDBConn::SetUUID(PGLOBAL g, PTDBJDBC tjp) goto err; } // endif rc - // Returns 666 is case of error - //jtyp = env->CallIntMethod(job, typid, 5, nullptr); - - //if (Check((jtyp == 666) ? -1 : 1)) { - // sprintf(g->Message, "Getting jtyp: %s", Msg); - // goto err; - //} // endif ctyp - + // Should return 666 is case of error (not done yet) ctyp = (int)env->CallIntMethod(job, intfldid, 5, nullptr); - if (Check(ctyp)) { - sprintf(g->Message, "Getting ctyp: %s", Msg); - goto err; - } // endif ctyp + //if (Check((ctyp == 666) ? -1 : 1)) { + // sprintf(g->Message, "Getting ctyp: %s", Msg); + // goto err; + //} // endif ctyp if (ctyp == 1111) ((PJDBCCOL)colp)->uuid = true; @@ -828,11 +817,11 @@ bool JDBConn::Connect(PJPARM sop) jstring s = (jstring)env->CallObjectMethod(job, qcid); if (s != nullptr) { - char *qch = (char*)env->GetStringUTFChars(s, NULL); + char *qch = GetUTFString(s); m_IDQuoteChar[0] = *qch; } else { s = (jstring)env->CallObjectMethod(job, errid); - Msg = (char*)env->GetStringUTFChars(s, NULL); + Msg = GetUTFString(s); } // endif s } // endif qcid @@ -1010,7 +999,7 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val) cn = nullptr; if (cn) { - field = env->GetStringUTFChars(cn, NULL); + field = GetUTFString(cn); val->SetValue_psz((PSZ)field); } else val->Reset(); @@ -1084,8 +1073,7 @@ void JDBConn::SetColumnValue(int rank, PSZ name, PVAL val) cn = nullptr; if (cn) { - const char *field = env->GetStringUTFChars(cn, NULL); - val->SetValue_psz((PSZ)field); + val->SetValue_psz((PSZ)GetUTFString(cn)); } else val->Reset(); @@ -1364,19 +1352,19 @@ bool JDBConn::SetParam(JDBCCOL *colp) for (i = 0, n = 0; i < size; i++) { crp = qrp->Colresp; js = (jstring)env->GetObjectArrayElement(s, n++); - sval = (PSZ)env->GetStringUTFChars(js, 0); + sval = GetUTFString(js); crp->Kdata->SetValue(sval, i); crp = crp->Next; js = (jstring)env->GetObjectArrayElement(s, n++); - sval = (PSZ)env->GetStringUTFChars(js, 0); + sval = GetUTFString(js); crp->Kdata->SetValue(sval, i); crp = crp->Next; js = (jstring)env->GetObjectArrayElement(s, n++); - sval = (PSZ)env->GetStringUTFChars(js, 0); + sval = GetUTFString(js); crp->Kdata->SetValue(sval, i); crp = crp->Next; js = (jstring)env->GetObjectArrayElement(s, n++); - sval = (PSZ)env->GetStringUTFChars(js, 0); + sval = GetUTFString(js); crp->Kdata->SetValue(sval, i); } // endfor i @@ -1462,7 +1450,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) return NULL; } // endif label - name = env->GetStringUTFChars(label, NULL); + name = GetUTFString(label); crp = qrp->Colresp; // Column_Name crp->Kdata->SetValue((char*)name, i); n = env->GetIntArrayElements(val, 0); diff --git a/storage/connect/jmgoconn.cpp b/storage/connect/jmgoconn.cpp index 33668e69988..bd1ddadd80d 100644 --- a/storage/connect/jmgoconn.cpp +++ b/storage/connect/jmgoconn.cpp @@ -522,7 +522,7 @@ PSZ JMgoConn::GetDocument(void) jdc = (jstring)env->CallObjectMethod(job, getdocid); if (jdc) - doc = (PSZ)env->GetStringUTFChars(jdc, NULL); + doc = (PSZ)GetUTFString(jdc); } // endif getdocid @@ -690,7 +690,7 @@ jobject JMgoConn::MakeDoc(PGLOBAL g, PJNCOL jcp) } // endif Jncolp - return parent; + return parent; } // end of MakeDoc /***********************************************************************/ @@ -807,7 +807,7 @@ PSZ JMgoConn::GetColumnValue(PSZ path) fn = (jstring)env->CallObjectMethod(job, objfldid, jn); if (fn) - fld = (PSZ)env->GetStringUTFChars(fn, NULL); + fld = (PSZ)GetUTFString(fn); } // endif objfldid diff --git a/storage/connect/json.h b/storage/connect/json.h index dcc97287420..cc394401cc3 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -44,6 +44,9 @@ typedef struct { int len; } STRG, *PSG; +char *NextChr(PSZ s, char sep); +char *GetJsonNull(void); + PJSON ParseJson(PGLOBAL g, char *s, int n, int *prty = NULL, bool *b = NULL); PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty); PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index e45846ea23b..26455d572b6 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1620,7 +1620,7 @@ static my_bool CheckMemory(PGLOBAL g, UDF_INIT *initid, UDF_ARGS *args, uint n, if (AllocSarea(g, ml)) { char errmsg[MAX_STR]; - sprintf(errmsg, MSG(WORK_AREA), g->Message); + snprintf(errmsg, sizeof(errmsg) - 1, MSG(WORK_AREA), g->Message); strcpy(g->Message, errmsg); return true; } // endif SareaAlloc @@ -1673,7 +1673,7 @@ static PCSZ MakeKey(PGLOBAL g, UDF_ARGS *args, int i) n = strlen(s); if (IsJson(args, i)) - j = strchr(s, '_') - s + 1; + j = (int)(strchr(s, '_') - s + 1); if (j && n > j) { s += j; @@ -4631,7 +4631,7 @@ char *jbin_array(UDF_INIT *initid, UDF_ARGS *args, char *result, bsp = NULL; if (!bsp && (bsp = JbinAlloc(g, args, initid->max_length, NULL))) - strncpy(bsp->Msg, g->Message, 139); + strncpy(bsp->Msg, g->Message, BMX); // Keep result of constant function g->Xchk = (initid->const_item) ? bsp : NULL; diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index cc8f75b2976..cc5675db5c8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -18,7 +18,7 @@ /* ------------- */ /* Version 1.6 */ /* */ -/* Author: Olivier Bertrand 2012 - 2017 */ +/* Author: Olivier Bertrand 2012 - 2018 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -28,20 +28,13 @@ /***********************************************************************/ /* Include relevant MariaDB header file. */ /***********************************************************************/ -#include +#define DONT_DEFINE_VOID +#include -#if defined(__WIN__) -//#include -//#include -#elif defined(UNIX) -#include +#if defined(UNIX) #include -#include -#include #include #endif -#define DONT_DEFINE_VOID -//#include #include "handler.h" #undef OFFSET diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 0dcf030613d..bf1542f4b1d 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,14 +9,17 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration -jdbc_oracle : Variable settings depend on machine configuration -jdbc_postgresql : Variable settings depend on machine configuration -json_mongo_c : Need MongoDB running and its C Driver installed -json_java_2 : Need MongoDB running and its Java Driver installed -json_java_3 : Need MongoDB running and its Java Driver installed -mongo_c : Need MongoDB running and its C Driver installed -mongo_java_2 : Need MongoDB running and its Java Driver installed -mongo_java_3 : Need MongoDB running and its Java Driver installed -tbl_thread : Bug MDEV-9844,10179,14214 03/01/2018 OB Option THREAD removed +infoschema-9739 : Crashes with MariaDB 10.0 +infoschema2-9739 : Temporary until recording result with MariaDB 10.0 +jdbc : Variable settings depend on machine configuration +jdbc_new : Variable settings depend on machine configuration +jdbc_oracle : Variable settings depend on machine configuration +jdbc_postgresql : Variable settings depend on machine configuration +json_mongo_c : Need MongoDB running and its C Driver installed +json_java_2 : Need MongoDB running and its Java Driver installed +json_java_3 : Need MongoDB running and its Java Driver installed +mongo_c : Need MongoDB running and its C Driver installed +mongo_java_2 : Need MongoDB running and its Java Driver installed +mongo_java_3 : Need MongoDB running and its Java Driver installed +tbl_thread : Bug MDEV-9844,10179,14214 03/01/2018 OB Option THREAD removed +vcol : Different error code on different versions diff --git a/storage/connect/mysql-test/connect/r/grant2.result b/storage/connect/mysql-test/connect/r/grant2.result index 0259dd74cdc..acefe6df659 100644 --- a/storage/connect/mysql-test/connect/r/grant2.result +++ b/storage/connect/mysql-test/connect/r/grant2.result @@ -622,7 +622,7 @@ test.t1 optimize status OK OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize Error Access denied for user 'user'@'localhost' (using password: NO) -test.t1 optimize Error Got error 122 'This operation requires the FILE privilege' from CONNECT +test.t1 optimize Error Can't lock file (errno: 122 "Internal (unspecified) error in handler") test.t1 optimize error Corrupt DROP TABLE t1; # Testing SQLCOM_ALTER_TABLE (adding columns) diff --git a/storage/connect/mysql-test/connect/r/infoschema2-9739.result b/storage/connect/mysql-test/connect/r/infoschema2-9739.result index 7d8a6839ea5..807d9a608f0 100644 --- a/storage/connect/mysql-test/connect/r/infoschema2-9739.result +++ b/storage/connect/mysql-test/connect/r/infoschema2-9739.result @@ -8,5 +8,5 @@ TABLE_CATALOG TABLE_SCHEMA TABLE_NAME TABLE_TYPE ENGINE VERSION ROW_FORMAT TABLE Warnings: Warning 1286 Unknown storage engine 'InnoDB' Warning 1286 Unknown storage engine 'InnoDB' -Warning 1296 Got error 174 'File t1.xml not found' from CONNECT +Warning 1105 drop table t1; diff --git a/storage/connect/mysql-test/connect/r/jdbc.result b/storage/connect/mysql-test/connect/r/jdbc.result index 895b4070d70..1bcd7b736bb 100644 --- a/storage/connect/mysql-test/connect/r/jdbc.result +++ b/storage/connect/mysql-test/connect/r/jdbc.result @@ -238,8 +238,7 @@ DROP TABLE t1, connect.emp; CREATE TABLE t2 (command varchar(128) not null,number int(5) not null flag=1,message varchar(255) flag=2) ENGINE=CONNECT TABLE_TYPE=JDBC CONNECTION='jdbc:mariadb://localhost:PORT/connect' OPTION_LIST='User=root,Execsrc=1'; SELECT * FROM t2 WHERE command='drop table tx1'; command number message -drop table tx1 0 Execute: java.sql.SQLSyntaxErrorException: Unknown table 'connect.tx1' -Query is : drop table tx1 +drop table tx1 0 Execute: java.sql.SQLSyntaxErrorException: (conn:23) Unknown table 'connect.tx1' SELECT * FROM t2 WHERE command = 'create table tx1 (a int not null, b char(32), c double(8,2))'; command number message create table tx1 (a int not null, b char(32), c double(8,2)) 0 Affected rows diff --git a/storage/connect/mysql-test/connect/r/json_java_2.result b/storage/connect/mysql-test/connect/r/json_java_2.result index 1a90132dede..ff87d8e2ad7 100644 --- a/storage/connect/mysql-test/connect/r/json_java_2.result +++ b/storage/connect/mysql-test/connect/r/json_java_2.result @@ -1,4 +1,3 @@ -SET GLOBAL connect_class_path='C:/MariaDB-10.2/MariaDB/storage/connect/mysql-test/connect/std_data/Mongo2.jar'; set connect_enable_mongo=1; # # Test the MONGO table type diff --git a/storage/connect/mysql-test/connect/r/json_java_3.result b/storage/connect/mysql-test/connect/r/json_java_3.result index 4c5fc94fca6..eb8bfc022d6 100644 --- a/storage/connect/mysql-test/connect/r/json_java_3.result +++ b/storage/connect/mysql-test/connect/r/json_java_3.result @@ -1,4 +1,3 @@ -SET GLOBAL connect_class_path='C:/MariaDB-10.2/MariaDB/storage/connect/mysql-test/connect/std_data/Mongo3.jar'; set connect_enable_mongo=1; # # Test the MONGO table type diff --git a/storage/connect/mysql-test/connect/r/mongo_java_2.result b/storage/connect/mysql-test/connect/r/mongo_java_2.result index 67c67653e88..bc186d7137e 100644 --- a/storage/connect/mysql-test/connect/r/mongo_java_2.result +++ b/storage/connect/mysql-test/connect/r/mongo_java_2.result @@ -1,4 +1,3 @@ -SET GLOBAL connect_class_path='C:/MariaDB-10.2/MariaDB/storage/connect/mysql-test/connect/std_data/Mongo2.jar'; set connect_enable_mongo=1; # # Test the MONGO table type diff --git a/storage/connect/mysql-test/connect/r/mongo_java_3.result b/storage/connect/mysql-test/connect/r/mongo_java_3.result index 665178bd3ea..30c696fc9eb 100644 --- a/storage/connect/mysql-test/connect/r/mongo_java_3.result +++ b/storage/connect/mysql-test/connect/r/mongo_java_3.result @@ -1,4 +1,3 @@ -SET GLOBAL connect_class_path='C:/MariaDB-10.2/MariaDB/storage/connect/mysql-test/connect/std_data/Mongo3.jar'; set connect_enable_mongo=1; # # Test the MONGO table type diff --git a/storage/connect/mysql-test/connect/r/mysql_exec.result b/storage/connect/mysql-test/connect/r/mysql_exec.result index 483fbd9e6a6..add98a6235d 100644 --- a/storage/connect/mysql-test/connect/r/mysql_exec.result +++ b/storage/connect/mysql-test/connect/r/mysql_exec.result @@ -30,6 +30,8 @@ insert ignore into t1(id) values(NULL) 1 1 Affected rows Warning 0 1364 Field 'msg' doesn't have a default value update t1 set msg = 'Four' where id = 4 0 1 Affected rows select * from t1 0 2 Result set columns +Warnings: +Warning 1105 Result set columns # # Checking Using Procedure # @@ -43,9 +45,13 @@ CALL p1('insert ignore into t1(id) values(NULL)'); command warnings number message insert ignore into t1(id) values(NULL) 1 1 Affected rows Warning 0 1364 Field 'msg' doesn't have a default value +Warnings: +Warning 1105 Affected rows CALL p1('update t1 set msg = "Five" where id = 5'); command warnings number message update t1 set msg = "Five" where id = 5 0 1 Affected rows +Warnings: +Warning 1105 Affected rows DROP PROCEDURE p1; DROP TABLE t1; SELECT * FROM t1; diff --git a/storage/connect/mysql-test/connect/r/odbc_postgresql.result b/storage/connect/mysql-test/connect/r/odbc_postgresql.result index 3426d23e29c..dc23dbdb990 100644 --- a/storage/connect/mysql-test/connect/r/odbc_postgresql.result +++ b/storage/connect/mysql-test/connect/r/odbc_postgresql.result @@ -99,9 +99,9 @@ Table_Cat Table_Schema Table_Name Column_Name Data_Type Type_Name Column_Size Bu mtr public t1 a 4 int4 10 4 0 10 0 mtr public t2 a 4 int4 10 4 0 10 0 mtr public v1 a 4 int4 10 4 0 10 1 -mtr schema1 t1 a 1 bpchar 10 60 NULL NULL 0 -mtr schema1 t2 a 1 bpchar 10 60 NULL NULL 0 -mtr schema1 v1 a 1 bpchar 10 60 NULL NULL 1 +mtr schema1 t1 a 1 bpchar 10 40 NULL NULL 0 +mtr schema1 t2 a 1 bpchar 10 40 NULL NULL 0 +mtr schema1 v1 a 1 bpchar 10 40 NULL NULL 1 DROP TABLE t1; # All columns in the schemas "public" and "schema1" CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=ODBC CONNECTION='DSN=ConnectEnginePostgresql;UID=mtr;PWD=mtr' CATFUNC=Columns TABNAME='%.%.%'; @@ -110,16 +110,16 @@ Table_Cat Table_Schema Table_Name Column_Name Data_Type Type_Name Column_Size Bu mtr public t1 a 4 int4 10 4 0 10 0 mtr public t2 a 4 int4 10 4 0 10 0 mtr public v1 a 4 int4 10 4 0 10 1 -mtr schema1 t1 a 1 bpchar 10 60 NULL NULL 0 -mtr schema1 t2 a 1 bpchar 10 60 NULL NULL 0 -mtr schema1 v1 a 1 bpchar 10 60 NULL NULL 1 +mtr schema1 t1 a 1 bpchar 10 40 NULL NULL 0 +mtr schema1 t2 a 1 bpchar 10 40 NULL NULL 0 +mtr schema1 v1 a 1 bpchar 10 40 NULL NULL 1 DROP TABLE t1; # All tables "t1" in all schemas CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=ODBC CONNECTION='DSN=ConnectEnginePostgresql;UID=mtr;PWD=mtr' CATFUNC=Columns TABNAME='%.%.t1'; SELECT * FROM t1 ORDER BY Table_Schema, Table_Name; Table_Cat Table_Schema Table_Name Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Radix Nullable Remarks mtr public t1 a 4 int4 10 4 0 10 0 -mtr schema1 t1 a 1 bpchar 10 60 NULL NULL 0 +mtr schema1 t1 a 1 bpchar 10 40 NULL NULL 0 DROP TABLE t1; # Table "t1" in the schema "public" CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=ODBC CONNECTION='DSN=ConnectEnginePostgresql;UID=mtr;PWD=mtr' CATFUNC=Columns TABNAME='%.public.t1'; @@ -131,14 +131,14 @@ DROP TABLE t1; CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=ODBC CONNECTION='DSN=ConnectEnginePostgresql;UID=mtr;PWD=mtr' CATFUNC=Columns TABNAME='%.schema1.t1'; SELECT * FROM t1 ORDER BY Table_Schema, Table_Name; Table_Cat Table_Schema Table_Name Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Radix Nullable Remarks -mtr schema1 t1 a 1 bpchar 10 60 NULL NULL 0 +mtr schema1 t1 a 1 bpchar 10 40 NULL NULL 0 DROP TABLE t1; # All tables "t1" in all schemas (Catalog name is ignored by PostgreSQL) CREATE TABLE t1 ENGINE=CONNECT TABLE_TYPE=ODBC CONNECTION='DSN=ConnectEnginePostgresql;UID=mtr;PWD=mtr' CATFUNC=Columns TABNAME='xxx.%.t1'; SELECT * FROM t1 ORDER BY Table_Schema, Table_Name; Table_Cat Table_Schema Table_Name Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Radix Nullable Remarks mtr public t1 a 4 int4 10 4 0 10 0 -mtr schema1 t1 a 1 bpchar 10 60 NULL NULL 0 +mtr schema1 t1 a 1 bpchar 10 40 NULL NULL 0 DROP TABLE t1; # # Checking tables diff --git a/storage/connect/mysql-test/connect/r/xml2.result b/storage/connect/mysql-test/connect/r/xml2.result index eea53bf55c7..b8075fa1928 100644 --- a/storage/connect/mysql-test/connect/r/xml2.result +++ b/storage/connect/mysql-test/connect/r/xml2.result @@ -333,37 +333,6 @@ DROP TABLE t1; # # Testing Cyrillic # -CREATE TABLE t1 -( -c CHAR(16) CHARACTER SET utf8 -) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' - OPTION_LIST='xmlsup=libxml2,rownode=b'; -SELECT * FROM t1; -c БВГДЕЖЗ -INSERT INTO t1 VALUES ('ИКЛМН'); -SELECT c, HEX(c) FROM t1; -c БВГДЕЖЗ -HEX(c) D091D092D093D094D095D096D097 -c ИКЛМН -HEX(c) D098D09AD09BD09CD09D -DROP TABLE t1; -CREATE TABLE t1 -( -c CHAR(16) CHARACTER SET cp1251 -) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' - OPTION_LIST='xmlsup=libxml2,rownode=b'; -SELECT * FROM t1; -c БВГДЕЖЗ -c ИКЛМН -INSERT INTO t1 VALUES ('ОПРСТ'); -SELECT c, HEX(c) FROM t1; -c БВГДЕЖЗ -HEX(c) C1C2C3C4C5C6C7 -c ИКЛМН -HEX(c) C8CACBCCCD -c ОПРСТ -HEX(c) CECFD0D1D2 -DROP TABLE t1; # # Testing that the underlying file is created with a proper Encoding # diff --git a/storage/connect/mysql-test/connect/t/json_java_2.test b/storage/connect/mysql-test/connect/t/json_java_2.test index bb32eff4e94..2f64d8e2eed 100644 --- a/storage/connect/mysql-test/connect/t/json_java_2.test +++ b/storage/connect/mysql-test/connect/t/json_java_2.test @@ -1,7 +1,9 @@ -- source jdbconn.inc -- source mongo.inc +--disable_query_log eval SET GLOBAL connect_class_path='$MTR_SUITE_DIR/std_data/Mongo2.jar'; +--enable_query_log let $DRV= Java; let $VERS= 2; let $TYPE= JSON; diff --git a/storage/connect/mysql-test/connect/t/json_java_3.test b/storage/connect/mysql-test/connect/t/json_java_3.test index 29e66cd5a1c..cee8343772a 100644 --- a/storage/connect/mysql-test/connect/t/json_java_3.test +++ b/storage/connect/mysql-test/connect/t/json_java_3.test @@ -1,7 +1,9 @@ -- source jdbconn.inc -- source mongo.inc +--disable_query_log eval SET GLOBAL connect_class_path='$MTR_SUITE_DIR/std_data/Mongo3.jar'; +--enable_query_log let $DRV= Java; let $VERS= 3; let $TYPE= JSON; diff --git a/storage/connect/mysql-test/connect/t/mongo_java_2.test b/storage/connect/mysql-test/connect/t/mongo_java_2.test index 21da5dce68f..7dcd028185e 100644 --- a/storage/connect/mysql-test/connect/t/mongo_java_2.test +++ b/storage/connect/mysql-test/connect/t/mongo_java_2.test @@ -1,7 +1,9 @@ -- source jdbconn.inc -- source mongo.inc +--disable_query_log eval SET GLOBAL connect_class_path='$MTR_SUITE_DIR/std_data/Mongo2.jar'; +--enable_query_log let $DRV= Java; let $VERS= 2; let $TYPE= MONGO; diff --git a/storage/connect/mysql-test/connect/t/mongo_java_3.test b/storage/connect/mysql-test/connect/t/mongo_java_3.test index b7584adcc7e..aab16d5e003 100644 --- a/storage/connect/mysql-test/connect/t/mongo_java_3.test +++ b/storage/connect/mysql-test/connect/t/mongo_java_3.test @@ -1,7 +1,9 @@ -- source jdbconn.inc -- source mongo.inc +--disable_query_log eval SET GLOBAL connect_class_path='$MTR_SUITE_DIR/std_data/Mongo3.jar'; +--enable_query_log let $DRV= Java; let $VERS= 3; let $TYPE= MONGO; diff --git a/storage/connect/mysql-test/connect/t/xml2.test b/storage/connect/mysql-test/connect/t/xml2.test index 0f26b8ab5c0..7bbc3dbd87c 100644 --- a/storage/connect/mysql-test/connect/t/xml2.test +++ b/storage/connect/mysql-test/connect/t/xml2.test @@ -240,24 +240,24 @@ DROP TABLE t1; --echo # --echo # Testing Cyrillic --echo # -CREATE TABLE t1 -( - c CHAR(16) CHARACTER SET utf8 -) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' - OPTION_LIST='xmlsup=libxml2,rownode=b'; -SELECT * FROM t1; -INSERT INTO t1 VALUES ('ИКЛМН'); -SELECT c, HEX(c) FROM t1; -DROP TABLE t1; -CREATE TABLE t1 -( - c CHAR(16) CHARACTER SET cp1251 -) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' - OPTION_LIST='xmlsup=libxml2,rownode=b'; -SELECT * FROM t1; -INSERT INTO t1 VALUES ('ОПРСТ'); -SELECT c, HEX(c) FROM t1; -DROP TABLE t1; +#CREATE TABLE t1 +#( +# c CHAR(16) CHARACTER SET utf8 +#) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' +# OPTION_LIST='xmlsup=libxml2,rownode=b'; +#SELECT * FROM t1; +#INSERT INTO t1 VALUES ('ИКЛМН'); +#SELECT c, HEX(c) FROM t1; +#DROP TABLE t1; +#CREATE TABLE t1 +#( +# c CHAR(16) CHARACTER SET cp1251 +#) ENGINE=CONNECT TABLE_TYPE=XML FILE_NAME='cp1251.xml' +# OPTION_LIST='xmlsup=libxml2,rownode=b'; +#SELECT * FROM t1; +#INSERT INTO t1 VALUES ('ОПРСТ'); +#SELECT c, HEX(c) FROM t1; +#DROP TABLE t1; --echo # diff --git a/storage/connect/odbconn.cpp b/storage/connect/odbconn.cpp index f7b1a43a95d..6687513fa6c 100644 --- a/storage/connect/odbconn.cpp +++ b/storage/connect/odbconn.cpp @@ -2354,11 +2354,11 @@ int ODBConn::GetCatInfo(CATPARM *cap) if (!Check(rc)) ThrowDBX(rc, fnc, hstmt); - rc = SQLNumResultCols(hstmt, &ncol); - - // n because we no more ignore the first column - if ((n = (UWORD)qrp->Nbcol) > (UWORD)ncol) - ThrowDBX(MSG(COL_NUM_MISM)); + // Some data source do not implement SQLNumResultCols + if (Check(SQLNumResultCols(hstmt, &ncol))) + // n because we no more ignore the first column + if ((n = (UWORD)qrp->Nbcol) > (UWORD)ncol) + ThrowDBX(MSG(COL_NUM_MISM)); // Unconditional to handle STRBLK's pval = (PVAL *)PlugSubAlloc(g, NULL, n * sizeof(PVAL)); diff --git a/storage/connect/tabext.cpp b/storage/connect/tabext.cpp index 64d401bef15..f2d5eb0e69d 100644 --- a/storage/connect/tabext.cpp +++ b/storage/connect/tabext.cpp @@ -125,6 +125,12 @@ EXTDEF::EXTDEF(void) /***********************************************************************/ bool EXTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { + if (g->Createas) { + strcpy(g->Message, + "Multiple-table UPDATE/DELETE commands are not supported"); + return true; + } // endif multi + Desc = NULL; Tabname = GetStringCatInfo(g, "Name", (Catfunc & (FNC_TABLE | FNC_COL)) ? NULL : Name); @@ -286,7 +292,7 @@ bool TDBEXT::MakeSrcdef(PGLOBAL g) char *catp = strstr(Srcdef, "%s"); if (catp) { - char *fil1, *fil2; + char *fil1 = 0, *fil2; PCSZ ph = ((EXTDEF*)To_Def)->Phpos; if (!ph) diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index 275b5edaeae..adb3fc4fb51 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -1157,8 +1157,9 @@ bool TDBXJDC::OpenDB(PGLOBAL g) /* Get the command to execute. */ /*********************************************************************/ if (!(Cmdlist = MakeCMD(g))) { - Jcp->Close(); - return true; + // Next lines commented out because of CHECK TABLE + //Jcp->Close(); + //return true; } // endif Query Rows = 1; @@ -1189,8 +1190,10 @@ int TDBXJDC::ReadDB(PGLOBAL g) Fpos++; // Used for progress info Cmdlist = (Nerr > Mxr) ? NULL : Cmdlist->Next; return RC_OK; - } else + } else { + PushWarning(g, this, 1); return RC_EF; + } // endif Cmdlist } // end of ReadDB diff --git a/storage/connect/tabjmg.cpp b/storage/connect/tabjmg.cpp index 4653973a4db..850d9e5fa9b 100644 --- a/storage/connect/tabjmg.cpp +++ b/storage/connect/tabjmg.cpp @@ -72,7 +72,7 @@ bool JMGDISC::Find(PGLOBAL g) bool JMGDISC::ColDesc(PGLOBAL g, jobject obj, char *pcn, char *pfmt, int ncol, int k) { - const char *key; + const char *key, *utf; char colname[65]; char fmt[129]; bool rc = true; @@ -101,7 +101,10 @@ bool JMGDISC::ColDesc(PGLOBAL g, jobject obj, char *pcn, char *pfmt, continue; jkey = (jstring)Jcp->env->CallObjectMethod(Jcp->job, bvnameid); - key = Jcp->env->GetStringUTFChars(jkey, NULL); + utf = Jcp->env->GetStringUTFChars(jkey, nullptr); + key = PlugDup(g, utf); + Jcp->env->ReleaseStringUTFChars(jkey, utf); + Jcp->env->DeleteLocalRef(jkey); if (pcn) { strncpy(colname, pcn, 64); @@ -457,8 +460,8 @@ PSZ JMGCOL::GetJpath(PGLOBAL g, bool proj) } else *p2++ = *p1; - *p2 = 0; - return projpath; + *p2 = 0; + return projpath; } else return Jpath; diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 0341c0f8aa0..2ff72905e86 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -16,6 +16,7 @@ typedef class JSONDEF *PJDEF; typedef class TDBJSON *PJTDB; typedef class JSONCOL *PJCOL; class TDBJSN; +PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info); /***********************************************************************/ /* The JSON tree node. Can be an Object or an Array. */ diff --git a/storage/connect/tabmysql.cpp b/storage/connect/tabmysql.cpp index 605b3822430..ceffafac02c 100644 --- a/storage/connect/tabmysql.cpp +++ b/storage/connect/tabmysql.cpp @@ -1587,8 +1587,9 @@ bool TDBMYEXC::OpenDB(PGLOBAL g) /* Get the command to execute. */ /*********************************************************************/ if (!(Cmdlist = MakeCMD(g))) { - Myc.Close(); - return true; + // Next lines commented out because of CHECK TABLE + //Myc.Close(); + //return true; } // endif Cmdlist return false; @@ -1647,8 +1648,10 @@ int TDBMYEXC::ReadDB(PGLOBAL g) ++N; return RC_OK; - } else - return RC_EF; + } else { + PushWarning(g, this, 1); + return RC_EF; + } // endif Cmdlist } // end of ReadDB diff --git a/storage/connect/tabodbc.cpp b/storage/connect/tabodbc.cpp index f7bc3934fbd..fddfb0c0420 100644 --- a/storage/connect/tabodbc.cpp +++ b/storage/connect/tabodbc.cpp @@ -1249,9 +1249,10 @@ bool TDBXDBC::OpenDB(PGLOBAL g) /* Get the command to execute. */ /*********************************************************************/ if (!(Cmdlist = MakeCMD(g))) { - Ocp->Close(); - return true; - } // endif Query + // Next lines commented out because of CHECK TABLE + //Ocp->Close(); + //return true; + } // endif Cmdlist Rows = 1; return false; @@ -1274,8 +1275,10 @@ int TDBXDBC::ReadDB(PGLOBAL g) Fpos++; // Used for progress info Cmdlist = (Nerr > Mxr) ? NULL : Cmdlist->Next; return RC_OK; - } else - return RC_EF; + } else { + PushWarning(g, this, 1); + return RC_EF; + } // endif Cmdlist } // end of ReadDB diff --git a/storage/connect/tabpivot.cpp b/storage/connect/tabpivot.cpp index da5d134f347..9121a0453e5 100644 --- a/storage/connect/tabpivot.cpp +++ b/storage/connect/tabpivot.cpp @@ -107,12 +107,12 @@ bool PIVAID::SkipColumn(PCOLRES crp, char *skc) /***********************************************************************/ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g) { - char *p, *query, *colname, *skc, buf[64]; - int ndif, nblin, w = 0; - bool b = false; - PVAL valp; - PQRYRES qrp; - PCOLRES *pcrp, crp, fncrp = NULL; + char *p, *query, *colname, *skc, buf[64]; + int ndif, nblin, w = 0; + bool b = false; + PVAL valp; + PQRYRES qrp; + PCOLRES *pcrp, crp, fncrp = NULL; try { // Are there columns to skip? @@ -186,7 +186,7 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g) } // endif picol - // Prepare the column list + // Prepare the column list for (pcrp = &Qryp->Colresp; crp = *pcrp; ) if (SkipColumn(crp, skc)) { // Ignore this column @@ -205,95 +205,95 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g) } else pcrp = &crp->Next; - if (!Rblkp) { - strcpy(g->Message, MSG(NO_DEF_PIVOTCOL)); - goto err; - } else if (!fncrp) { - strcpy(g->Message, MSG(NO_DEF_FNCCOL)); - goto err; - } // endif + if (!Rblkp) { + strcpy(g->Message, MSG(NO_DEF_PIVOTCOL)); + goto err; + } else if (!fncrp) { + strcpy(g->Message, MSG(NO_DEF_FNCCOL)); + goto err; + } // endif - if (Tabsrc) { - Myc.Close(); - b = false; + if (Tabsrc) { + Myc.Close(); + b = false; - // Before calling sort, initialize all - nblin = Qryp->Nblin; + // Before calling sort, initialize all + nblin = Qryp->Nblin; - Index.Size = nblin * sizeof(int); - Index.Sub = TRUE; // Should be small enough + Index.Size = nblin * sizeof(int); + Index.Sub = TRUE; // Should be small enough - if (!PlgDBalloc(g, NULL, Index)) - goto err; - - Offset.Size = (nblin + 1) * sizeof(int); - Offset.Sub = TRUE; // Should be small enough - - if (!PlgDBalloc(g, NULL, Offset)) - goto err; - - ndif = Qsort(g, nblin); - - if (ndif < 0) // error - goto err; - - } else { - // The query was limited, we must get pivot column values - // Returned values must be in their original character set - // if (Myc.ExecSQL(g, "SET character_set_results=NULL", &w) == RC_FX) - // goto err; - - query = (char*)PlugSubAlloc(g, NULL, 0); - sprintf(query, "SELECT DISTINCT `%s` FROM `%s`", Picol, Tabname); - PlugSubAlloc(g, NULL, strlen(query) + 1); - Myc.FreeResult(); - - // Send the source command to MySQL - if (Myc.ExecSQL(g, query, &w) == RC_FX) - goto err; - - // We must have a storage query to get pivot column values - if (!(qrp = Myc.GetResult(g, true))) - goto err; - - Myc.Close(); - b = false; - - // Get the column list - crp = qrp->Colresp; - Rblkp = crp->Kdata; - ndif = qrp->Nblin; - } // endif Tabsrc - - // Allocate the Value used to retieve column names - if (!(valp = AllocateValue(g, Rblkp->GetType(), - Rblkp->GetVlen(), - Rblkp->GetPrec()))) + if (!PlgDBalloc(g, NULL, Index)) goto err; - // Now make the functional columns - for (int i = 0; i < ndif; i++) { - if (i) { - crp = (PCOLRES)PlugSubAlloc(g, NULL, sizeof(COLRES)); - memcpy(crp, fncrp, sizeof(COLRES)); - } else - crp = fncrp; + Offset.Size = (nblin + 1) * sizeof(int); + Offset.Sub = TRUE; // Should be small enough - // Get the value that will be the generated column name - if (Tabsrc) - valp->SetValue_pvblk(Rblkp, Pex[Pof[i]]); - else - valp->SetValue_pvblk(Rblkp, i); + if (!PlgDBalloc(g, NULL, Offset)) + goto err; - colname = valp->GetCharString(buf); - crp->Name = PlugDup(g, colname); - crp->Flag = 1; + ndif = Qsort(g, nblin); - // Add this column - *pcrp = crp; - crp->Next = NULL; - pcrp = &crp->Next; - } // endfor i + if (ndif < 0) // error + goto err; + + } else { + // The query was limited, we must get pivot column values + // Returned values must be in their original character set + // if (Myc.ExecSQL(g, "SET character_set_results=NULL", &w) == RC_FX) + // goto err; + + query = (char*)PlugSubAlloc(g, NULL, 0); + sprintf(query, "SELECT DISTINCT `%s` FROM `%s`", Picol, Tabname); + PlugSubAlloc(g, NULL, strlen(query) + 1); + Myc.FreeResult(); + + // Send the source command to MySQL + if (Myc.ExecSQL(g, query, &w) == RC_FX) + goto err; + + // We must have a storage query to get pivot column values + if (!(qrp = Myc.GetResult(g, true))) + goto err; + + Myc.Close(); + b = false; + + // Get the column list + crp = qrp->Colresp; + Rblkp = crp->Kdata; + ndif = qrp->Nblin; + } // endif Tabsrc + + // Allocate the Value used to retieve column names + if (!(valp = AllocateValue(g, Rblkp->GetType(), + Rblkp->GetVlen(), + Rblkp->GetPrec()))) + goto err; + + // Now make the functional columns + for (int i = 0; i < ndif; i++) { + if (i) { + crp = (PCOLRES)PlugSubAlloc(g, NULL, sizeof(COLRES)); + memcpy(crp, fncrp, sizeof(COLRES)); + } else + crp = fncrp; + + // Get the value that will be the generated column name + if (Tabsrc) + valp->SetValue_pvblk(Rblkp, Pex[Pof[i]]); + else + valp->SetValue_pvblk(Rblkp, i); + + colname = valp->GetCharString(buf); + crp->Name = PlugDup(g, colname); + crp->Flag = 1; + + // Add this column + *pcrp = crp; + crp->Next = NULL; + pcrp = &crp->Next; + } // endfor i // We added ndif columns and removed 2 (picol and fncol) Qryp->Nbcol += (ndif - 2); @@ -306,10 +306,10 @@ PQRYRES PIVAID::MakePivotColumns(PGLOBAL g) } // end catch err: - if (b) - Myc.Close(); + if (b) + Myc.Close(); - return NULL; + return NULL; } // end of MakePivotColumns /***********************************************************************/ diff --git a/storage/connect/tabutil.cpp b/storage/connect/tabutil.cpp index 68b66aec31f..462a6fcd839 100644 --- a/storage/connect/tabutil.cpp +++ b/storage/connect/tabutil.cpp @@ -429,7 +429,7 @@ PTDB TDBPRX::GetSubTable(PGLOBAL g, PTABLE tabp, bool b) char buf[MAX_STR]; strcpy(buf, g->Message); - sprintf(g->Message, "Error accessing %s.%s: %s", db, name, buf); + snprintf(g->Message, MAX_STR, "Error accessing %s.%s: %s", db, name, buf); hc->tshp = NULL; goto err; } // endif Define diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 11b344ef652..40d020202ea 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -456,13 +456,11 @@ bool VCTCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) if (tdbp->Txfp->GetAmType() == TYPE_AM_VMP && ok) { Blk = AllocValBlock(g, (void*)1, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); Status |= BUF_MAPPED; // Will point into mapped file } else Blk = AllocValBlock(g, NULL, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); } // endif Mode return false; diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index 9532d7c2a8d..e2d3b664aeb 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -36,6 +36,7 @@ #define DONT_DEFINE_VOID #define MYSQL_SERVER +#include #include "sql_class.h" #undef OFFSET From cd0734d6bde05906b0dcecd2e90d0bbb056a88c7 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 28 Oct 2018 10:55:46 +0100 Subject: [PATCH 079/127] after-merge: enable tests --- storage/connect/mysql-test/connect/disabled.def | 2 -- storage/connect/mysql-test/connect/r/grant2.result | 2 +- storage/connect/mysql-test/connect/r/infoschema2-9739.result | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index bf1542f4b1d..827ed58b835 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -10,7 +10,6 @@ # ############################################################################## infoschema-9739 : Crashes with MariaDB 10.0 -infoschema2-9739 : Temporary until recording result with MariaDB 10.0 jdbc : Variable settings depend on machine configuration jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration @@ -22,4 +21,3 @@ mongo_c : Need MongoDB running and its C Driver installed mongo_java_2 : Need MongoDB running and its Java Driver installed mongo_java_3 : Need MongoDB running and its Java Driver installed tbl_thread : Bug MDEV-9844,10179,14214 03/01/2018 OB Option THREAD removed -vcol : Different error code on different versions diff --git a/storage/connect/mysql-test/connect/r/grant2.result b/storage/connect/mysql-test/connect/r/grant2.result index acefe6df659..0259dd74cdc 100644 --- a/storage/connect/mysql-test/connect/r/grant2.result +++ b/storage/connect/mysql-test/connect/r/grant2.result @@ -622,7 +622,7 @@ test.t1 optimize status OK OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize Error Access denied for user 'user'@'localhost' (using password: NO) -test.t1 optimize Error Can't lock file (errno: 122 "Internal (unspecified) error in handler") +test.t1 optimize Error Got error 122 'This operation requires the FILE privilege' from CONNECT test.t1 optimize error Corrupt DROP TABLE t1; # Testing SQLCOM_ALTER_TABLE (adding columns) diff --git a/storage/connect/mysql-test/connect/r/infoschema2-9739.result b/storage/connect/mysql-test/connect/r/infoschema2-9739.result index 807d9a608f0..7d8a6839ea5 100644 --- a/storage/connect/mysql-test/connect/r/infoschema2-9739.result +++ b/storage/connect/mysql-test/connect/r/infoschema2-9739.result @@ -8,5 +8,5 @@ TABLE_CATALOG TABLE_SCHEMA TABLE_NAME TABLE_TYPE ENGINE VERSION ROW_FORMAT TABLE Warnings: Warning 1286 Unknown storage engine 'InnoDB' Warning 1286 Unknown storage engine 'InnoDB' -Warning 1105 +Warning 1296 Got error 174 'File t1.xml not found' from CONNECT drop table t1; From d5f564a9960d1689c36eaab50144e6e26576a50e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 28 Oct 2018 12:42:06 +0100 Subject: [PATCH 080/127] rpm fixes: more %ignore'd files --- cmake/cpack_rpm.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index e80fb199f03..962c9f1f56d 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -83,12 +83,15 @@ SET(ignored "%ignore /etc" "%ignore /etc/init.d" "%ignore /etc/logrotate.d" + "%ignore /etc/systemd" + "%ignore /etc/systemd/system" "%ignore ${CMAKE_INSTALL_PREFIX}" "%ignore ${CMAKE_INSTALL_PREFIX}/bin" "%ignore ${CMAKE_INSTALL_PREFIX}/include" "%ignore ${CMAKE_INSTALL_PREFIX}/lib" "%ignore ${CMAKE_INSTALL_PREFIX}/lib/systemd" "%ignore ${CMAKE_INSTALL_PREFIX}/lib/systemd/system" + "%ignore ${CMAKE_INSTALL_PREFIX}/lib/tmpfiles.d" "%ignore ${CMAKE_INSTALL_PREFIX}/lib64" "%ignore ${CMAKE_INSTALL_PREFIX}/sbin" "%ignore ${CMAKE_INSTALL_PREFIX}/share" From b3009059d07651bfb6146353cf5838305c3d9f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 29 Oct 2018 11:26:23 +0200 Subject: [PATCH 081/127] Minor cleanup --- storage/innobase/buf/buf0dblwr.cc | 2 +- storage/innobase/fil/fil0fil.cc | 43 ------------------------------ storage/innobase/fsp/fsp0fsp.cc | 16 +++++++++++ storage/innobase/include/buf0buf.h | 20 ++++---------- storage/innobase/include/fil0fil.h | 39 --------------------------- storage/innobase/include/fsp0fsp.h | 38 ++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 98 deletions(-) diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index ab531dc775d..bed24fd7704 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -858,7 +858,7 @@ buf_dblwr_check_block( but just happens to have wrongly set FIL_PAGE_TYPE, such pages should never be modified to without also adjusting the page type during page allocation or - buf_flush_init_for_writing() or fil_page_reset_type(). */ + buf_flush_init_for_writing() or fil_block_reset_type(). */ break; case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_IBUF_BITMAP: diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d08e362af3b..38ad8caf137 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -5564,27 +5564,6 @@ fil_page_set_type( mach_write_to_2(page + FIL_PAGE_TYPE, type); } -/** Reset the page type. -Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE. -In MySQL 3.23.53, only undo log pages and index pages were tagged. -Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ -void -fil_page_reset_type( - const page_id_t page_id, - byte* page, - ulint type, - mtr_t* mtr) -{ - ib::info() - << "Resetting invalid page " << page_id << " type " - << fil_page_get_type(page) << " to " << type << "."; - mlog_write_ulint(page + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr); -} - /****************************************************************//** Closes the tablespace memory cache. */ void @@ -6311,25 +6290,3 @@ fil_space_set_punch_hole( { node->space->punch_hole = val; } - -/** Check (and if needed, reset) the page type. -Data files created before MySQL 5.1 may contain -garbage in the FIL_PAGE_TYPE field. -In MySQL 3.23.53, only undo log pages and index pages were tagged. -Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with possibly invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ -void -fil_block_check_type( - const buf_block_t& block, - ulint type, - mtr_t* mtr) -{ - ulint page_type = fil_page_get_type(block.frame); - - if (page_type != type) { - fil_page_reset_type(block.page.id, block.frame, type, mtr); - } -} diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2b44d7e60cc..7bf31504119 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1062,6 +1062,22 @@ fsp_get_pages_to_extend_ibd( return(size_increase); } +/** Reset the page type. +Data files created before MySQL 5.1.48 may contain garbage in FIL_PAGE_TYPE. +In MySQL 3.23.53, only undo log pages and index pages were tagged. +Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. +@param[in] block block with invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ +ATTRIBUTE_COLD +void fil_block_reset_type(const buf_block_t& block, ulint type, mtr_t* mtr) +{ + ib::info() + << "Resetting invalid page " << block.page.id << " type " + << fil_page_get_type(block.frame) << " to " << type << "."; + mlog_write_ulint(block.frame + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr); +} + /** Put new extents to the free list if there are free extents above the free limit. If an extent happens to contain an extent descriptor page, the extent is put to the FSP_FREE_FRAG list with the page marked as used. diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index b8bde3bd776..8e13b3876e4 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -223,8 +223,7 @@ public: @param[in] space tablespace id @param[in] page_no page number */ page_id_t(ulint space, ulint page_no) - : m_space(static_cast(space)), - m_page_no(static_cast(page_no)) + : m_space(uint32_t(space)), m_page_no(uint32(page_no)) { ut_ad(space <= 0xFFFFFFFFU); ut_ad(page_no <= 0xFFFFFFFFU); @@ -238,30 +237,21 @@ public: /** Retrieve the tablespace id. @return tablespace id */ - inline uint32_t space() const - { - return(m_space); - } + uint32_t space() const { return m_space; } /** Retrieve the page number. @return page number */ - inline uint32_t page_no() const - { - return(m_page_no); - } + uint32_t page_no() const { return m_page_no; } /** Retrieve the fold value. @return fold value */ - inline ulint fold() const - { - return (m_space << 20) + m_space + m_page_no; - } + ulint fold() const { return (m_space << 20) + m_space + m_page_no; } /** Reset the page number only. @param[in] page_no page number */ inline void set_page_no(ulint page_no) { - m_page_no = static_cast(page_no); + m_page_no = uint32_t(page_no); ut_ad(page_no <= 0xFFFFFFFFU); } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index d3ccac2057e..34bf216fd97 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1315,45 +1315,6 @@ fil_page_set_type( /*==============*/ byte* page, /*!< in/out: file page */ ulint type); /*!< in: type */ -/** Reset the page type. -Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE. -In MySQL 3.23.53, only undo log pages and index pages were tagged. -Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ -void -fil_page_reset_type( - const page_id_t page_id, - byte* page, - ulint type, - mtr_t* mtr); - -/** Get the file page type. -@param[in] page file page -@return page type */ -inline -uint16_t -fil_page_get_type(const byte* page) -{ - return(mach_read_from_2(page + FIL_PAGE_TYPE)); -} - -/** Check (and if needed, reset) the page type. -Data files created before MySQL 5.1 may contain -garbage in the FIL_PAGE_TYPE field. -In MySQL 3.23.53, only undo log pages and index pages were tagged. -Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with possibly invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ -void -fil_block_check_type( - const buf_block_t& block, - ulint type, - mtr_t* mtr); #ifdef UNIV_DEBUG /** Increase redo skipped of a tablespace. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 7b50f84eb92..3e4c0368b00 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -640,6 +640,44 @@ fseg_free_step_not_header_func( fseg_free_step_not_header_func(header, mtr) #endif /* BTR_CUR_HASH_ADAPT */ +/** Reset the page type. +Data files created before MySQL 5.1.48 may contain garbage in FIL_PAGE_TYPE. +In MySQL 3.23.53, only undo log pages and index pages were tagged. +Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. +@param[in] block block with invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ +ATTRIBUTE_COLD +void fil_block_reset_type(const buf_block_t& block, ulint type, mtr_t* mtr); + +/** Get the file page type. +@param[in] page file page +@return page type */ +inline uint16_t fil_page_get_type(const byte* page) +{ + return mach_read_from_2(page + FIL_PAGE_TYPE); +} + +/** Check (and if needed, reset) the page type. +Data files created before MySQL 5.1.48 may contain +garbage in the FIL_PAGE_TYPE field. +In MySQL 3.23.53, only undo log pages and index pages were tagged. +Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. +@param[in] page_id page number +@param[in,out] page page with possibly invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ +inline void +fil_block_check_type( + const buf_block_t& block, + ulint type, + mtr_t* mtr) +{ + if (UNIV_UNLIKELY(type != fil_page_get_type(block.frame))) { + fil_block_reset_type(block, type, mtr); + } +} + /** Checks if a page address is an extent descriptor page address. @param[in] page_id page id @param[in] page_size page size From cfd81de6d46970347a9ae4626770fe5ef22dda5b Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 29 Oct 2018 13:43:52 +0300 Subject: [PATCH 082/127] MDEV-17414: MyROCKS order desc limit 1 fails - Use the correct range bounds when doing a reverse-ordered range scan (this was already done for HA_READ_PREFIX_LAST_OR_PREV but not for HA_READ_BEFORE_KEY). --- storage/rocksdb/ha_rocksdb.cc | 10 +++++++++- .../mysql-test/rocksdb/r/rocksdb_range2.result | 17 +++++++++++++++++ .../mysql-test/rocksdb/t/rocksdb_range2.test | 12 ++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index af82e194d7b..56fe5df77cc 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8563,9 +8563,17 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, packed_size); uint end_key_packed_size = 0; + /* + In MariaDB, the end_key is always the bigger end of the range. + If we are doing a reverse-ordered scan (that is, walking from the bigger + key values to smaller), we should use the smaller end of range as end_key. + */ const key_range *cur_end_key= end_key; - if (find_flag == HA_READ_PREFIX_LAST_OR_PREV) + if (find_flag == HA_READ_PREFIX_LAST_OR_PREV || + find_flag == HA_READ_BEFORE_KEY) + { cur_end_key= m_start_range; + } const uint eq_cond_len = calc_eq_cond_len(kd, find_flag, slice, bytes_changed_by_succ, cur_end_key, diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result index 10a6a02008e..a925c21e188 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result @@ -9,3 +9,20 @@ explain select c1 from t1 where c1 > 5 limit 10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range i i 9 NULL # Using where; Using index drop table t1; +# +# MDEV-17414: MyROCKS order desc limit 1 fails +# +create table t1 (date date); +insert into t1 values ('2018-10-04'), ('2018-10-05'); +select * from t1 where date < '2018-10-09' order by date desc limit 1; +date +2018-10-05 +alter table t1 add index date_index (date); +select * from t1 where date < '2018-10-05' order by date desc limit 1; +date +2018-10-04 +# this should not produce an empty set: +select * from t1 where date < '2018-10-09' order by date desc limit 1; +date +2018-10-05 +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test index a7ac236451e..28010d13753 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test @@ -19,3 +19,15 @@ select count(*) from t1; explain select c1 from t1 where c1 > 5 limit 10; drop table t1; +--echo # +--echo # MDEV-17414: MyROCKS order desc limit 1 fails +--echo # +create table t1 (date date); +insert into t1 values ('2018-10-04'), ('2018-10-05'); +select * from t1 where date < '2018-10-09' order by date desc limit 1; # Works as expected +alter table t1 add index date_index (date); +select * from t1 where date < '2018-10-05' order by date desc limit 1; # Works as expected +--echo # this should not produce an empty set: +select * from t1 where date < '2018-10-09' order by date desc limit 1; +drop table t1; + From f8604ed9ffba0ef645280245cee89afe394c1fdd Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 29 Oct 2018 13:49:44 +0300 Subject: [PATCH 083/127] MDEV-17414: MyROCKS order desc limit 1 fails : Backport to 10.2 - Use the correct range bounds when doing a reverse-ordered range scan (this was already done for HA_READ_PREFIX_LAST_OR_PREV but not for HA_READ_BEFORE_KEY). --- storage/rocksdb/ha_rocksdb.cc | 10 +++++++++- .../mysql-test/rocksdb/r/rocksdb_range2.result | 17 +++++++++++++++++ .../mysql-test/rocksdb/t/rocksdb_range2.test | 12 ++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index d05f336ee18..03794291575 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8560,9 +8560,17 @@ int ha_rocksdb::index_read_map_impl(uchar *const buf, const uchar *const key, packed_size); uint end_key_packed_size = 0; + /* + In MariaDB, the end_key is always the bigger end of the range. + If we are doing a reverse-ordered scan (that is, walking from the bigger + key values to smaller), we should use the smaller end of range as end_key. + */ const key_range *cur_end_key= end_key; - if (find_flag == HA_READ_PREFIX_LAST_OR_PREV) + if (find_flag == HA_READ_PREFIX_LAST_OR_PREV || + find_flag == HA_READ_BEFORE_KEY) + { cur_end_key= m_start_range; + } const uint eq_cond_len = calc_eq_cond_len(kd, find_flag, slice, bytes_changed_by_succ, cur_end_key, diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result index 10a6a02008e..a925c21e188 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range2.result @@ -9,3 +9,20 @@ explain select c1 from t1 where c1 > 5 limit 10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range i i 9 NULL # Using where; Using index drop table t1; +# +# MDEV-17414: MyROCKS order desc limit 1 fails +# +create table t1 (date date); +insert into t1 values ('2018-10-04'), ('2018-10-05'); +select * from t1 where date < '2018-10-09' order by date desc limit 1; +date +2018-10-05 +alter table t1 add index date_index (date); +select * from t1 where date < '2018-10-05' order by date desc limit 1; +date +2018-10-04 +# this should not produce an empty set: +select * from t1 where date < '2018-10-09' order by date desc limit 1; +date +2018-10-05 +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test index a7ac236451e..28010d13753 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_range2.test @@ -19,3 +19,15 @@ select count(*) from t1; explain select c1 from t1 where c1 > 5 limit 10; drop table t1; +--echo # +--echo # MDEV-17414: MyROCKS order desc limit 1 fails +--echo # +create table t1 (date date); +insert into t1 values ('2018-10-04'), ('2018-10-05'); +select * from t1 where date < '2018-10-09' order by date desc limit 1; # Works as expected +alter table t1 add index date_index (date); +select * from t1 where date < '2018-10-05' order by date desc limit 1; # Works as expected +--echo # this should not produce an empty set: +select * from t1 where date < '2018-10-09' order by date desc limit 1; +drop table t1; + From 6a6cc8a653a374a60278dfdbd8bd702c0a692e35 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 29 Oct 2018 15:48:49 +0200 Subject: [PATCH 084/127] Remove not used table_flag HA_NO_VARCHAR --- sql/handler.h | 1 - sql/sql_table.cc | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/sql/handler.h b/sql/handler.h index d2d49b4601c..e52f9ddaefb 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -151,7 +151,6 @@ enum enum_alter_inplace_result { #define HA_HAS_OLD_CHECKSUM (1ULL << 24) /* Table data are stored in separate files (for lower_case_table_names) */ #define HA_FILE_BASED (1ULL << 26) -#define HA_NO_VARCHAR (1ULL << 27) /* unused */ #define HA_CAN_BIT_FIELD (1ULL << 28) /* supports bit fields */ #define HA_NEED_READ_RANGE_BUFFER (1ULL << 29) /* for read_multi_range */ #define HA_ANY_INDEX_MAY_BE_UNIQUE (1ULL << 30) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index bfeac0c3a57..c9676105dec 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2935,20 +2935,6 @@ uint Column_definition::pack_flag_numeric(uint dec) const bool Column_definition::prepare_stage2_varchar(ulonglong table_flags) { -#ifndef QQ_ALL_HANDLERS_SUPPORT_VARCHAR - if (table_flags & HA_NO_VARCHAR) - { - /* convert VARCHAR to CHAR because handler is not yet up to date */ - set_handler(&type_handler_var_string); - pack_length= type_handler()->calc_pack_length((uint) length); - if ((length / charset->mbmaxlen) > MAX_FIELD_CHARLENGTH) - { - my_error(ER_TOO_BIG_FIELDLENGTH, MYF(0), field_name.str, - static_cast(MAX_FIELD_CHARLENGTH)); - return true; - } - } -#endif pack_flag= (charset->state & MY_CS_BINSORT) ? FIELDFLAG_BINARY : 0; return false; } From d30124e844cdcd2eef4b13199e91aa5c304e907c Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 29 Oct 2018 16:12:52 +0200 Subject: [PATCH 085/127] MDEV-17503 CREATE SEQUENCE failed with innodb_force_primary_key =1 Fixed by adding table flag HA_WANTS_PRIMARY_KEY, which is like HA_REQUIRE_PRIMARY_KEY but tells SQL upper layer that the storage engine internally can handle tables without primary keys (for example for sequences or trough user variables) --- mysql-test/suite/sql_sequence/create.result | 9 +++++++++ mysql-test/suite/sql_sequence/create.test | 11 +++++++++++ sql/ha_partition.h | 4 ++++ sql/handler.h | 3 +++ sql/sql_table.cc | 4 +++- storage/innobase/handler/ha_innodb.cc | 2 +- 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/sql_sequence/create.result b/mysql-test/suite/sql_sequence/create.result index cb683d158d8..55d45a75abf 100644 --- a/mysql-test/suite/sql_sequence/create.result +++ b/mysql-test/suite/sql_sequence/create.result @@ -644,3 +644,12 @@ TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE DROP SEQUENCE seq1; CREATE TEMPORARY SEQUENCE seq1 ENGINE=InnoDB ROW_FORMAT=REDUNDANT; DROP TEMPORARY SEQUENCE seq1; +# +# MDEV-17503 CREATE SEQUENCE failed with innodb_force_primary_key =1 +# +set global innodb_force_primary_key =1; +CREATE SEQUENCE s1 START WITH 100 INCREMENT BY 10 ENGINE=innodb; +set global innodb_force_primary_key=default; +ALTER TABLE s1 ADD PRIMARY KEY (next_not_cached_value); +ERROR HY000: Sequence 'test.s1' table structure is invalid (Sequence tables cannot have any keys) +DROP SEQUENCE s1; diff --git a/mysql-test/suite/sql_sequence/create.test b/mysql-test/suite/sql_sequence/create.test index b2562058ca6..1bc62117526 100644 --- a/mysql-test/suite/sql_sequence/create.test +++ b/mysql-test/suite/sql_sequence/create.test @@ -462,3 +462,14 @@ SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME='test/seq1'; DROP SEQUENCE seq1; CREATE TEMPORARY SEQUENCE seq1 ENGINE=InnoDB ROW_FORMAT=REDUNDANT; DROP TEMPORARY SEQUENCE seq1; + +--echo # +--echo # MDEV-17503 CREATE SEQUENCE failed with innodb_force_primary_key =1 +--echo # + +set global innodb_force_primary_key =1; +CREATE SEQUENCE s1 START WITH 100 INCREMENT BY 10 ENGINE=innodb; +set global innodb_force_primary_key=default; +--error ER_SEQUENCE_INVALID_TABLE_STRUCTURE +ALTER TABLE s1 ADD PRIMARY KEY (next_not_cached_value); +DROP SEQUENCE s1; diff --git a/sql/ha_partition.h b/sql/ha_partition.h index 8a374fe87b7..82877806243 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -1041,6 +1041,10 @@ public: with hidden primary key) (No handler has this limitation currently) + HA_WANTS_PRIMARY_KEY: + Can't define a table without primary key except sequences + (Only InnoDB has this when using innodb_force_primary_key == ON) + HA_STATS_RECORDS_IS_EXACT: Does the counter of records after the info call specify an exact value or not. If it does this flag is set. diff --git a/sql/handler.h b/sql/handler.h index e52f9ddaefb..788ac4dd474 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -301,6 +301,9 @@ enum enum_alter_inplace_result { /* calling cmp_ref() on the engine is expensive */ #define HA_CMP_REF_IS_EXPENSIVE (1ULL << 54) +/* Engine wants primary keys for everything except sequences */ +#define HA_WANTS_PRIMARY_KEY (1ULL << 55) + /* bits in index_flags(index_number) for what you can do with index */ #define HA_READ_NEXT 1 /* TODO really use this flag */ #define HA_READ_PREV 2 /* supports ::index_prev */ diff --git a/sql/sql_table.cc b/sql/sql_table.cc index c9676105dec..b9fc431feb1 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4134,7 +4134,9 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, } if (!unique_key && !primary_key && - (file->ha_table_flags() & HA_REQUIRE_PRIMARY_KEY)) + ((file->ha_table_flags() & HA_REQUIRE_PRIMARY_KEY) || + ((file->ha_table_flags() & HA_WANTS_PRIMARY_KEY) && + !create_info->sequence))) { my_message(ER_REQUIRES_PRIMARY_KEY, ER_THD(thd, ER_REQUIRES_PRIMARY_KEY), MYF(0)); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index cfec257322e..20dc215ee7f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2918,7 +2918,7 @@ ha_innobase::ha_innobase( | HA_CAN_RTREEKEYS | HA_CAN_TABLES_WITHOUT_ROLLBACK | HA_CONCURRENT_OPTIMIZE - | (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0) + | (srv_force_primary_key ? HA_WANTS_PRIMARY_KEY : 0) ), m_start_of_scan(), m_mysql_has_locked() From f30148a7402a576c7133fcf1b157273b256869fe Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Oct 2018 14:51:29 +0100 Subject: [PATCH 086/127] CONNECT: bintar compilation failure on Mac OS X --- storage/connect/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index d985df9eaa7..7acc6c81837 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -126,7 +126,7 @@ IF(CONNECT_WITH_LIBXML2) FIND_PACKAGE(LibXml2) IF (LIBXML2_FOUND) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR}) - SET(ZLIB_LIBRARY "") # see ZLIB_INCLUDE_DIR below + SET(ZLIB_LIBRARY "z") # see ZLIB_INCLUDE_DIR below SET(XML_LIBRARY ${LIBXML2_LIBRARIES}) SET(CONNECT_SOURCES ${CONNECT_SOURCES} libdoc.cpp libdoc.h) add_definitions(-DLIBXML2_SUPPORT) From f4b8b6b9a3ad4ce5e5218e2ec2dfe6dd34112e45 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Oct 2018 21:44:38 +0100 Subject: [PATCH 087/127] MDEV-15919 lower_case_table_names does not behave as expected followup for e31e697f17f Fix the test not to fail on Mac OS X (lower_case_table_names=0 prevents mysqld from starting on case insensitive filesystem) --- mysql-test/suite/rpl/t/rpl_15919-master.opt | 1 - mysql-test/suite/rpl/t/rpl_15919.test | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 mysql-test/suite/rpl/t/rpl_15919-master.opt diff --git a/mysql-test/suite/rpl/t/rpl_15919-master.opt b/mysql-test/suite/rpl/t/rpl_15919-master.opt deleted file mode 100644 index 9b27aef9bf8..00000000000 --- a/mysql-test/suite/rpl/t/rpl_15919-master.opt +++ /dev/null @@ -1 +0,0 @@ ---lower_case_table_names=0 diff --git a/mysql-test/suite/rpl/t/rpl_15919.test b/mysql-test/suite/rpl/t/rpl_15919.test index 0462f7fd067..a5b25929ad0 100644 --- a/mysql-test/suite/rpl/t/rpl_15919.test +++ b/mysql-test/suite/rpl/t/rpl_15919.test @@ -1,5 +1,7 @@ ---source include/master-slave.inc +--source include/have_case_sensitive_file_system.inc --source include/have_innodb.inc +--source include/master-slave.inc + --connection master create table RPL(a int); insert into RPL values(1); From 8772824ce72a811ea92dd5cfb73602eb93237891 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 29 Oct 2018 23:21:55 +0100 Subject: [PATCH 088/127] Restore auto-switch to bundled ssl if WITH_SSL=yes if -DWITH_SSL=yes and system ssl cannot be used, bundled ssl should be auto-selected. That's how it worked in 10.0, and it was unintentionally broken in 10.1. --- cmake/ssl.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake index 18728c733db..9bbce599602 100644 --- a/cmake/ssl.cmake +++ b/cmake/ssl.cmake @@ -180,8 +180,7 @@ MACRO (MYSQL_CHECK_SSL) OPENSSL_MAJOR_VERSION "${OPENSSL_VERSION_NUMBER}" ) MESSAGE(STATUS "OPENSSL_MAJOR_VERSION = ${OPENSSL_MAJOR_VERSION}") - ENDIF() - IF(TRUE) #OPENSSL_MAJOR_VERSION GREATER 0) + SET(SSL_SOURCES "") SET(SSL_LIBRARIES ${OPENSSL_LIBRARIES} ${CRYPTO_LIBRARY}) IF(CMAKE_SYSTEM_NAME MATCHES "SunOS") From 6ced789186fabd7dce97b3d6d171ff9e5ddc5f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Oct 2018 13:29:19 +0200 Subject: [PATCH 089/127] MDEV-12023 Assertion failure sym_node->table != NULL on startup row_drop_table_for_mysql(): Avoid accessing non-existing dictionary tables. dict_create_or_check_foreign_constraint_tables(): Add debug instrumentation for creating and dropping a table before the creation of any non-core dictionary tables. trx_purge_add_update_undo_to_history(): Adjust a debug assertion, so that it will not fail due to the test instrumentation. --- mysql-test/include/maybe_debug.combinations | 5 + mysql-test/include/maybe_debug.inc | 3 + mysql-test/suite.pm | 3 + .../suite/innodb/r/table_flags,4k,debug.rdiff | 128 ++++++++++++++ ...,4k.rdiff => table_flags,4k,release.rdiff} | 0 .../suite/innodb/r/table_flags,8k,debug.rdiff | 128 ++++++++++++++ ...,8k.rdiff => table_flags,8k,release.rdiff} | 0 .../suite/innodb/r/table_flags,debug.rdiff | 128 ++++++++++++++ mysql-test/suite/innodb/t/table_flags.test | 4 + storage/innobase/dict/dict0crea.cc | 16 ++ storage/innobase/row/row0mysql.cc | 163 +++++++++--------- storage/innobase/trx/trx0purge.cc | 2 +- storage/xtradb/dict/dict0crea.cc | 17 +- storage/xtradb/row/row0mysql.cc | 163 +++++++++--------- storage/xtradb/trx/trx0purge.cc | 2 +- 15 files changed, 588 insertions(+), 174 deletions(-) create mode 100644 mysql-test/include/maybe_debug.combinations create mode 100644 mysql-test/include/maybe_debug.inc create mode 100644 mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,4k.rdiff => table_flags,4k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,8k.rdiff => table_flags,8k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,debug.rdiff diff --git a/mysql-test/include/maybe_debug.combinations b/mysql-test/include/maybe_debug.combinations new file mode 100644 index 00000000000..5ee57c0bfd8 --- /dev/null +++ b/mysql-test/include/maybe_debug.combinations @@ -0,0 +1,5 @@ +[debug] +--enable-gdb + +[release] +--disable-gdb diff --git a/mysql-test/include/maybe_debug.inc b/mysql-test/include/maybe_debug.inc new file mode 100644 index 00000000000..2f6c2848f9f --- /dev/null +++ b/mysql-test/include/maybe_debug.inc @@ -0,0 +1,3 @@ +# include file for test files that can be run with and without debug +# having debug and non-debug tests. +let $have_debug=`select version() like '%debug%'`; diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index 199cfba3b8f..52025801974 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -23,6 +23,9 @@ sub skip_combinations { # don't run tests for the wrong platform $skip{'include/platform.combinations'} = [ (IS_WINDOWS) ? 'unix' : 'win' ]; + $skip{'include/maybe_debug.combinations'} = + [ defined $::mysqld_variables{'debug-dbug'} ? 'release' : 'debug' ]; + # as a special case, disable certain include files as a whole $skip{'include/not_embedded.inc'} = 'Not run for embedded server' if $::opt_embedded_server; diff --git a/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff new file mode 100644 index 00000000000..83e40b2d3fd --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff @@ -0,0 +1,128 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,4k,debug.reject +@@ -10,81 +10,81 @@ + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + SYS_TABLES clustered index root page (8): + N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000003270194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x8000000320016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x80000003200260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501fc (NAME='SYS_TABLESPACES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++header=0x040028150209 (NAME='SYS_TABLESPACES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000003270110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000038150240 (NAME='test/tc', +- DB_TRX_ID=0x000000000303, +- DB_ROLL_PTR=0x83000001360110, +- ID=0x0000000000000010, ++header=0x00003815024d (NAME='test/tc', ++ DB_TRX_ID=0x000000000304, ++ DB_ROLL_PTR=0x830000032c0110, ++ ID=0x0000000000000011, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x0000401502c8 (NAME='test/td', +- DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x84000001370110, +- ID=0x0000000000000011, ++header=0x0000401502d5 (NAME='test/td', ++ DB_TRX_ID=0x000000000305, ++ DB_ROLL_PTR=0x840000032d0110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x0000501501b8 (NAME='test/tp', +- DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x86000001390110, +- ID=0x0000000000000013, ++header=0x00005015008d (NAME='test/tp', ++ DB_TRX_ID=0x000000000307, ++ DB_ROLL_PTR=0x86000003300110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x000030150284 (NAME='test/tr', +- DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++header=0x000010150291 (NAME='test/tr', ++ DB_TRX_ID=0x000000000303, ++ DB_ROLL_PTR=0x820000032b0110, ++ ID=0x0000000000000010, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -92,9 +92,9 @@ + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000001) + header=0x000048150074 (NAME='test/tz', +- DB_TRX_ID=0x000000000305, +- DB_ROLL_PTR=0x85000001380110, +- ID=0x0000000000000012, ++ DB_TRX_ID=0x000000000306, ++ DB_ROLL_PTR=0x850000032e0110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/r/table_flags,4k.rdiff b/mysql-test/suite/innodb/r/table_flags,4k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,4k.rdiff rename to mysql-test/suite/innodb/r/table_flags,4k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff new file mode 100644 index 00000000000..14fa7509c19 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff @@ -0,0 +1,128 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,8k,debug.reject +@@ -10,81 +10,81 @@ + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + SYS_TABLES clustered index root page (8): + N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000001d80194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x80000001d1016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x80000001d10260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501fc (NAME='SYS_TABLESPACES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++header=0x040028150209 (NAME='SYS_TABLESPACES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000001d80110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000038150240 (NAME='test/tc', +- DB_TRX_ID=0x000000000303, +- DB_ROLL_PTR=0x83000001360110, +- ID=0x0000000000000010, ++header=0x00003815024d (NAME='test/tc', ++ DB_TRX_ID=0x000000000304, ++ DB_ROLL_PTR=0x83000001dc0110, ++ ID=0x0000000000000011, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x0000401502c8 (NAME='test/td', +- DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x84000001370110, +- ID=0x0000000000000011, ++header=0x0000401502d5 (NAME='test/td', ++ DB_TRX_ID=0x000000000305, ++ DB_ROLL_PTR=0x84000001dd0110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x0000501501b8 (NAME='test/tp', +- DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x86000001390110, +- ID=0x0000000000000013, ++header=0x00005015008d (NAME='test/tp', ++ DB_TRX_ID=0x000000000307, ++ DB_ROLL_PTR=0x86000001df0110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x000030150284 (NAME='test/tr', +- DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++header=0x000010150291 (NAME='test/tr', ++ DB_TRX_ID=0x000000000303, ++ DB_ROLL_PTR=0x82000001db0110, ++ ID=0x0000000000000010, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -92,9 +92,9 @@ + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000001) + header=0x000048150074 (NAME='test/tz', +- DB_TRX_ID=0x000000000305, +- DB_ROLL_PTR=0x85000001380110, +- ID=0x0000000000000012, ++ DB_TRX_ID=0x000000000306, ++ DB_ROLL_PTR=0x85000001de0110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/r/table_flags,8k.rdiff b/mysql-test/suite/innodb/r/table_flags,8k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,8k.rdiff rename to mysql-test/suite/innodb/r/table_flags,8k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,debug.rdiff new file mode 100644 index 00000000000..d038515ffe5 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,debug.rdiff @@ -0,0 +1,128 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,debug.reject +@@ -10,81 +10,81 @@ + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + SYS_TABLES clustered index root page (8): + N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000001330194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012d016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012d0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501fc (NAME='SYS_TABLESPACES', +- DB_TRX_ID=0x000000000301, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++header=0x040028150209 (NAME='SYS_TABLESPACES', ++ DB_TRX_ID=0x000000000302, ++ DB_ROLL_PTR=0x81000001330110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000038150240 (NAME='test/tc', +- DB_TRX_ID=0x000000000303, +- DB_ROLL_PTR=0x83000001360110, +- ID=0x0000000000000010, ++header=0x00003815024d (NAME='test/tc', ++ DB_TRX_ID=0x000000000304, ++ DB_ROLL_PTR=0x83000001370110, ++ ID=0x0000000000000011, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x0000401502c8 (NAME='test/td', +- DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x84000001370110, +- ID=0x0000000000000011, ++header=0x0000401502d5 (NAME='test/td', ++ DB_TRX_ID=0x000000000305, ++ DB_ROLL_PTR=0x84000001380110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x0000501501b8 (NAME='test/tp', +- DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x86000001390110, +- ID=0x0000000000000013, ++header=0x00005015008d (NAME='test/tp', ++ DB_TRX_ID=0x000000000307, ++ DB_ROLL_PTR=0x860000013a0110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x000030150284 (NAME='test/tr', +- DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++header=0x000010150291 (NAME='test/tr', ++ DB_TRX_ID=0x000000000303, ++ DB_ROLL_PTR=0x82000001360110, ++ ID=0x0000000000000010, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -92,9 +92,9 @@ + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000001) + header=0x000048150074 (NAME='test/tz', +- DB_TRX_ID=0x000000000305, +- DB_ROLL_PTR=0x85000001380110, +- ID=0x0000000000000012, ++ DB_TRX_ID=0x000000000306, ++ DB_ROLL_PTR=0x85000001390110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/t/table_flags.test b/mysql-test/suite/innodb/t/table_flags.test index 4d3afcb2dec..59ae0a69ae6 100644 --- a/mysql-test/suite/innodb/t/table_flags.test +++ b/mysql-test/suite/innodb/t/table_flags.test @@ -1,6 +1,7 @@ --source include/innodb_page_size.inc # Embedded server tests do not support restarting --source include/not_embedded.inc +--source include/maybe_debug.inc --disable_query_log call mtr.add_suppression("InnoDB: New log files created, LSN="); @@ -22,6 +23,9 @@ let bugdir= $MYSQLTEST_VARDIR/tmp/table_flags; --let $d=--innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --let $d=$d --innodb-data-file-path=ibdata1:10M:autoextend --let $d=$d --innodb-undo-tablespaces=0 +if ($have_debug) { +--let $d=$d --debug=d,create_and_drop_garbage +} --let $restart_parameters=$d --innodb-stats-persistent=0 --innodb-file-format=1 --source include/restart_mysqld.inc diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 583b679428a..2388b50b72b 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1360,6 +1361,21 @@ dict_create_or_check_foreign_constraint_tables(void) row_mysql_lock_data_dictionary(trx); + DBUG_EXECUTE_IF( + "create_and_drop_garbage", + err = que_eval_sql( + NULL, + "PROCEDURE CREATE_GARBAGE_TABLE_PROC () IS\n" + "BEGIN\n" + "CREATE TABLE\n" + "\"test/#sql-ib-garbage\"(ID CHAR);\n" + "CREATE UNIQUE CLUSTERED INDEX PRIMARY" + " ON \"test/#sql-ib-garbage\"(ID);\n" + "END;\n", FALSE, trx); + ut_ad(err == DB_SUCCESS); + row_drop_table_for_mysql("test/#sql-ib-garbage", + trx, TRUE, TRUE);); + /* Check which incomplete table definition to drop. */ if (sys_foreign_err == DB_CORRUPTION) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index ae3f99fac68..d9e18a99201 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -45,6 +45,7 @@ Created 9/17/2000 Heikki Tuuri #include "dict0dict.h" #include "dict0crea.h" #include "dict0load.h" +#include "dict0priv.h" #include "dict0boot.h" #include "dict0stats.h" #include "dict0stats_bg.h" @@ -4137,95 +4138,87 @@ row_drop_table_for_mysql( info = pars_info_create(); - pars_info_add_str_literal(info, "table_name", name); + pars_info_add_str_literal(info, "name", name); - err = que_eval_sql(info, - "PROCEDURE DROP_TABLE_PROC () IS\n" - "sys_foreign_id CHAR;\n" - "table_id CHAR;\n" - "index_id CHAR;\n" - "foreign_id CHAR;\n" - "space_id INT;\n" - "found INT;\n" + if (strcmp(name, "SYS_FOREIGN") && strcmp(name, "SYS_FOREIGN_COLS") + && dict_table_get_low("SYS_FOREIGN") + && dict_table_get_low("SYS_FOREIGN_COLS")) { + err = que_eval_sql( + info, + "PROCEDURE DROP_FOREIGN_PROC () IS\n" + "fid CHAR;\n" - "DECLARE CURSOR cur_fk IS\n" - "SELECT ID FROM SYS_FOREIGN\n" - "WHERE FOR_NAME = :table_name\n" - "AND TO_BINARY(FOR_NAME)\n" - " = TO_BINARY(:table_name)\n" - "LOCK IN SHARE MODE;\n" + "DECLARE CURSOR fk IS\n" + "SELECT ID FROM SYS_FOREIGN\n" + "WHERE FOR_NAME = :name\n" + "AND TO_BINARY(FOR_NAME) = TO_BINARY(:name)\n" + "FOR UPDATE;\n" - "DECLARE CURSOR cur_idx IS\n" - "SELECT ID FROM SYS_INDEXES\n" - "WHERE TABLE_ID = table_id\n" - "LOCK IN SHARE MODE;\n" + "BEGIN\n" + "OPEN fk;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH fk INTO fid;\n" + " IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" + " DELETE FROM SYS_FOREIGN_COLS WHERE ID = fid;\n" + " DELETE FROM SYS_FOREIGN WHERE ID = fid;\n" + "END LOOP;\n" + "CLOSE fk;\n" + "END;\n", FALSE, trx); + if (err == DB_SUCCESS) { + info = pars_info_create(); + pars_info_add_str_literal(info, "name", name); + goto do_drop; + } + } else { +do_drop: + err = que_eval_sql( + info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "table_id CHAR;\n" + "index_id CHAR;\n" - "BEGIN\n" - "SELECT ID INTO table_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - "SELECT SPACE INTO space_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - "found := 1;\n" - "SELECT ID INTO sys_foreign_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = 'SYS_FOREIGN'\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN') THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN_COLS') THEN\n" - " found := 0;\n" - "END IF;\n" - "OPEN cur_fk;\n" - "WHILE found = 1 LOOP\n" - " FETCH cur_fk INTO foreign_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " ELSE\n" - " DELETE FROM SYS_FOREIGN_COLS\n" - " WHERE ID = foreign_id;\n" - " DELETE FROM SYS_FOREIGN\n" - " WHERE ID = foreign_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_fk;\n" - "found := 1;\n" - "OPEN cur_idx;\n" - "WHILE found = 1 LOOP\n" - " FETCH cur_idx INTO index_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " ELSE\n" - " DELETE FROM SYS_FIELDS\n" - " WHERE INDEX_ID = index_id;\n" - " DELETE FROM SYS_INDEXES\n" - " WHERE ID = index_id\n" - " AND TABLE_ID = table_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_idx;\n" - "DELETE FROM SYS_TABLESPACES\n" - "WHERE SPACE = space_id;\n" - "DELETE FROM SYS_DATAFILES\n" - "WHERE SPACE = space_id;\n" - "DELETE FROM SYS_COLUMNS\n" - "WHERE TABLE_ID = table_id;\n" - "DELETE FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" - "END;\n" - , FALSE, trx); + "DECLARE CURSOR cur_idx IS\n" + "SELECT ID FROM SYS_INDEXES\n" + "WHERE TABLE_ID = table_id\n" + "FOR UPDATE;\n" + + "BEGIN\n" + "SELECT ID INTO table_id\n" + "FROM SYS_TABLES WHERE NAME = :name FOR UPDATE;\n" + "IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" + "OPEN cur_idx;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH cur_idx INTO index_id;\n" + " IF (SQL % NOTFOUND) THEN EXIT; END IF;\n" + " DELETE FROM SYS_FIELDS\n" + " WHERE INDEX_ID = index_id;\n" + " DELETE FROM SYS_INDEXES\n" + " WHERE ID = index_id AND TABLE_ID = table_id;\n" + "END LOOP;\n" + "CLOSE cur_idx;\n" + + "DELETE FROM SYS_COLUMNS WHERE TABLE_ID = table_id;\n" + "DELETE FROM SYS_TABLES WHERE NAME = :name;\n" + + "END;\n", FALSE, trx); + + if (err == DB_SUCCESS && table->space + && dict_table_get_low("SYS_TABLESPACES") + && dict_table_get_low("SYS_DATAFILES")) { + info = pars_info_create(); + pars_info_add_int4_literal(info, "id", + lint(table->space)); + err = que_eval_sql( + info, + "PROCEDURE DROP_SPACE_PROC () IS\n" + "BEGIN\n" + "DELETE FROM SYS_TABLESPACES\n" + "WHERE SPACE = :id;\n" + "DELETE FROM SYS_DATAFILES\n" + "WHERE SPACE = :id;\n" + "END;\n", FALSE, trx); + } + } switch (err) { ibool is_temp; diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 2605bf29d25..acb916e1760 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -251,7 +251,7 @@ trx_purge_add_update_undo_to_history( in fast shutdown, we may roll back transactions (trx->undo_no==0) in THD::cleanup() invoked from unlink_thd(). */ ut_ad(srv_undo_sources - || ((srv_startup_is_before_trx_rollback_phase + || ((srv_is_being_started || trx_rollback_or_clean_is_active) && purge_sys->state == PURGE_STATE_INIT) || (trx->undo_no == 0 && srv_fast_shutdown)); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index 9a4040421d7..9d1312dc685 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1361,6 +1361,21 @@ dict_create_or_check_foreign_constraint_tables(void) row_mysql_lock_data_dictionary(trx); + DBUG_EXECUTE_IF( + "create_and_drop_garbage", + err = que_eval_sql( + NULL, + "PROCEDURE CREATE_GARBAGE_TABLE_PROC () IS\n" + "BEGIN\n" + "CREATE TABLE\n" + "\"test/#sql-ib-garbage\"(ID CHAR);\n" + "CREATE UNIQUE CLUSTERED INDEX PRIMARY" + " ON \"test/#sql-ib-garbage\"(ID);\n" + "END;\n", FALSE, trx); + ut_ad(err == DB_SUCCESS); + row_drop_table_for_mysql("test/#sql-ib-garbage", + trx, TRUE, TRUE);); + /* Check which incomplete table definition to drop. */ if (sys_foreign_err == DB_CORRUPTION) { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 6a32a988e30..cb6e6bbcb1f 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4162,95 +4162,87 @@ row_drop_table_for_mysql( info = pars_info_create(); - pars_info_add_str_literal(info, "table_name", name); + pars_info_add_str_literal(info, "name", name); - err = que_eval_sql(info, - "PROCEDURE DROP_TABLE_PROC () IS\n" - "sys_foreign_id CHAR;\n" - "table_id CHAR;\n" - "index_id CHAR;\n" - "foreign_id CHAR;\n" - "space_id INT;\n" - "found INT;\n" + if (strcmp(name, "SYS_FOREIGN") && strcmp(name, "SYS_FOREIGN_COLS") + && dict_table_get_low("SYS_FOREIGN") + && dict_table_get_low("SYS_FOREIGN_COLS")) { + err = que_eval_sql( + info, + "PROCEDURE DROP_FOREIGN_PROC () IS\n" + "fid CHAR;\n" - "DECLARE CURSOR cur_fk IS\n" - "SELECT ID FROM SYS_FOREIGN\n" - "WHERE FOR_NAME = :table_name\n" - "AND TO_BINARY(FOR_NAME)\n" - " = TO_BINARY(:table_name)\n" - "LOCK IN SHARE MODE;\n" + "DECLARE CURSOR fk IS\n" + "SELECT ID FROM SYS_FOREIGN\n" + "WHERE FOR_NAME = :name\n" + "AND TO_BINARY(FOR_NAME) = TO_BINARY(:name)\n" + "FOR UPDATE;\n" - "DECLARE CURSOR cur_idx IS\n" - "SELECT ID FROM SYS_INDEXES\n" - "WHERE TABLE_ID = table_id\n" - "LOCK IN SHARE MODE;\n" + "BEGIN\n" + "OPEN fk;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH fk INTO fid;\n" + " IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" + " DELETE FROM SYS_FOREIGN_COLS WHERE ID = fid;\n" + " DELETE FROM SYS_FOREIGN WHERE ID = fid;\n" + "END LOOP;\n" + "CLOSE fk;\n" + "END;\n", FALSE, trx); + if (err == DB_SUCCESS) { + info = pars_info_create(); + pars_info_add_str_literal(info, "name", name); + goto do_drop; + } + } else { +do_drop: + err = que_eval_sql( + info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "table_id CHAR;\n" + "index_id CHAR;\n" - "BEGIN\n" - "SELECT ID INTO table_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - "SELECT SPACE INTO space_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - "found := 1;\n" - "SELECT ID INTO sys_foreign_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = 'SYS_FOREIGN'\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN') THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN_COLS') THEN\n" - " found := 0;\n" - "END IF;\n" - "OPEN cur_fk;\n" - "WHILE found = 1 LOOP\n" - " FETCH cur_fk INTO foreign_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " ELSE\n" - " DELETE FROM SYS_FOREIGN_COLS\n" - " WHERE ID = foreign_id;\n" - " DELETE FROM SYS_FOREIGN\n" - " WHERE ID = foreign_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_fk;\n" - "found := 1;\n" - "OPEN cur_idx;\n" - "WHILE found = 1 LOOP\n" - " FETCH cur_idx INTO index_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " ELSE\n" - " DELETE FROM SYS_FIELDS\n" - " WHERE INDEX_ID = index_id;\n" - " DELETE FROM SYS_INDEXES\n" - " WHERE ID = index_id\n" - " AND TABLE_ID = table_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_idx;\n" - "DELETE FROM SYS_TABLESPACES\n" - "WHERE SPACE = space_id;\n" - "DELETE FROM SYS_DATAFILES\n" - "WHERE SPACE = space_id;\n" - "DELETE FROM SYS_COLUMNS\n" - "WHERE TABLE_ID = table_id;\n" - "DELETE FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" - "END;\n" - , FALSE, trx); + "DECLARE CURSOR cur_idx IS\n" + "SELECT ID FROM SYS_INDEXES\n" + "WHERE TABLE_ID = table_id\n" + "FOR UPDATE;\n" + + "BEGIN\n" + "SELECT ID INTO table_id\n" + "FROM SYS_TABLES WHERE NAME = :name FOR UPDATE;\n" + "IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" + "OPEN cur_idx;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH cur_idx INTO index_id;\n" + " IF (SQL % NOTFOUND) THEN EXIT; END IF;\n" + " DELETE FROM SYS_FIELDS\n" + " WHERE INDEX_ID = index_id;\n" + " DELETE FROM SYS_INDEXES\n" + " WHERE ID = index_id AND TABLE_ID = table_id;\n" + "END LOOP;\n" + "CLOSE cur_idx;\n" + + "DELETE FROM SYS_COLUMNS WHERE TABLE_ID = table_id;\n" + "DELETE FROM SYS_TABLES WHERE NAME = :name;\n" + + "END;\n", FALSE, trx); + + if (err == DB_SUCCESS && table->space + && dict_table_get_low("SYS_TABLESPACES") + && dict_table_get_low("SYS_DATAFILES")) { + info = pars_info_create(); + pars_info_add_int4_literal(info, "id", + lint(table->space)); + err = que_eval_sql( + info, + "PROCEDURE DROP_SPACE_PROC () IS\n" + "BEGIN\n" + "DELETE FROM SYS_TABLESPACES\n" + "WHERE SPACE = :id;\n" + "DELETE FROM SYS_DATAFILES\n" + "WHERE SPACE = :id;\n" + "END;\n", FALSE, trx); + } + } switch (err) { ibool is_temp; @@ -4395,7 +4387,6 @@ row_drop_table_for_mysql( case DB_OUT_OF_FILE_SPACE: err = DB_MUST_GET_MORE_FILE_SPACE; - trx->error_state = err; row_mysql_handle_errors(&err, trx, NULL, NULL); diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index 57bd11bacc3..09a5f0591b0 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -255,7 +255,7 @@ trx_purge_add_update_undo_to_history( in fast shutdown, we may roll back transactions (trx->undo_no==0) in THD::cleanup() invoked from unlink_thd(). */ ut_ad(srv_undo_sources - || ((srv_startup_is_before_trx_rollback_phase + || ((srv_is_being_started || trx_rollback_or_clean_is_active) && purge_sys->state == PURGE_STATE_INIT) || (trx->undo_no == 0 && srv_fast_shutdown)); From 3859273d048f0a4194c076f44aed4b45475ed158 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 10 Feb 2018 18:28:23 +1100 Subject: [PATCH 090/127] MDEV-14267: correct FSF address --- extra/mariabackup/backup_copy.cc | 4 ++-- extra/mariabackup/backup_mysql.cc | 4 ++-- extra/mariabackup/crc/crc-intel-pclmul.c | 2 +- extra/mariabackup/innobackupex.cc | 4 ++-- extra/mariabackup/xtrabackup.cc | 4 ++-- include/big_endian.h | 2 +- include/byte_order_generic.h | 2 +- include/byte_order_generic_x86.h | 2 +- include/byte_order_generic_x86_64.h | 2 +- include/little_endian.h | 2 +- include/my_cpu.h | 2 +- mysql-test/dgcov.pl | 2 +- plugin/auth_ed25519/ed25519-t.c | 2 +- plugin/user_variables/user_variables.cc | 2 +- sql/sql_get_diagnostics.cc | 2 +- sql/sql_get_diagnostics.h | 2 +- storage/rocksdb/event_listener.cc | 2 +- storage/rocksdb/event_listener.h | 2 +- storage/rocksdb/ha_rocksdb.cc | 2 +- storage/rocksdb/ha_rocksdb.h | 2 +- storage/rocksdb/ha_rocksdb_proto.h | 2 +- storage/rocksdb/logger.h | 2 +- storage/rocksdb/properties_collector.cc | 2 +- storage/rocksdb/properties_collector.h | 2 +- storage/rocksdb/rdb_buff.h | 2 +- storage/rocksdb/rdb_cf_manager.cc | 2 +- storage/rocksdb/rdb_cf_manager.h | 2 +- storage/rocksdb/rdb_cf_options.cc | 2 +- storage/rocksdb/rdb_cf_options.h | 2 +- storage/rocksdb/rdb_compact_filter.h | 2 +- storage/rocksdb/rdb_comparator.h | 2 +- storage/rocksdb/rdb_datadic.cc | 2 +- storage/rocksdb/rdb_datadic.h | 2 +- storage/rocksdb/rdb_i_s.cc | 2 +- storage/rocksdb/rdb_i_s.h | 2 +- storage/rocksdb/rdb_index_merge.cc | 2 +- storage/rocksdb/rdb_index_merge.h | 2 +- storage/rocksdb/rdb_io_watchdog.cc | 2 +- storage/rocksdb/rdb_io_watchdog.h | 2 +- storage/rocksdb/rdb_mutex_wrapper.cc | 2 +- storage/rocksdb/rdb_mutex_wrapper.h | 2 +- storage/rocksdb/rdb_perf_context.cc | 2 +- storage/rocksdb/rdb_perf_context.h | 2 +- storage/rocksdb/rdb_sst_info.cc | 2 +- storage/rocksdb/rdb_sst_info.h | 2 +- storage/rocksdb/rdb_threads.cc | 2 +- storage/rocksdb/rdb_threads.h | 2 +- storage/rocksdb/rdb_utils.cc | 2 +- storage/rocksdb/rdb_utils.h | 2 +- storage/rocksdb/unittest/test_properties_collector.cc | 2 +- 50 files changed, 54 insertions(+), 54 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index f4ed8726802..36cfd29f9e2 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -35,8 +35,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA *******************************************************/ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index aa9200f42fd..e542eb7e5e7 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -34,8 +34,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA *******************************************************/ #define MYSQL_CLIENT diff --git a/extra/mariabackup/crc/crc-intel-pclmul.c b/extra/mariabackup/crc/crc-intel-pclmul.c index d470c2bee43..a9e1c63d739 100644 --- a/extra/mariabackup/crc/crc-intel-pclmul.c +++ b/extra/mariabackup/crc/crc-intel-pclmul.c @@ -35,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * */ diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index 1e174c0fec8..3f74f25f5ba 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -34,8 +34,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d55eae6f743..e627826b8c4 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -36,8 +36,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ diff --git a/include/big_endian.h b/include/big_endian.h index 021b6abc383..5f6e623f6a0 100644 --- a/include/big_endian.h +++ b/include/big_endian.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* Data in big-endian format. diff --git a/include/byte_order_generic.h b/include/byte_order_generic.h index d4ac27eeb9c..14b00fc87eb 100644 --- a/include/byte_order_generic.h +++ b/include/byte_order_generic.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* Endianness-independent definitions for architectures other diff --git a/include/byte_order_generic_x86.h b/include/byte_order_generic_x86.h index a97dd0f43a3..5df208ff35c 100644 --- a/include/byte_order_generic_x86.h +++ b/include/byte_order_generic_x86.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* Optimized function-like macros for the x86 architecture (_WIN32 included). diff --git a/include/byte_order_generic_x86_64.h b/include/byte_order_generic_x86_64.h index fbc6e1f536b..42c1944ad21 100644 --- a/include/byte_order_generic_x86_64.h +++ b/include/byte_order_generic_x86_64.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* Optimized function-like macros for the x86 architecture (_WIN32 included). diff --git a/include/little_endian.h b/include/little_endian.h index 7223fea648f..eb704f4344f 100644 --- a/include/little_endian.h +++ b/include/little_endian.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* Data in little-endian format. diff --git a/include/my_cpu.h b/include/my_cpu.h index 026b92c1b74..a67c0e976d6 100644 --- a/include/my_cpu.h +++ b/include/my_cpu.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ /* instructions for specific cpu's */ diff --git a/mysql-test/dgcov.pl b/mysql-test/dgcov.pl index 27647eed056..05829c61bc8 100755 --- a/mysql-test/dgcov.pl +++ b/mysql-test/dgcov.pl @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA # Run gcov and report test coverage on only those code lines touched by # a given list of commits. diff --git a/plugin/auth_ed25519/ed25519-t.c b/plugin/auth_ed25519/ed25519-t.c index f7d58c48d7c..fa058f69325 100644 --- a/plugin/auth_ed25519/ed25519-t.c +++ b/plugin/auth_ed25519/ed25519-t.c @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include #include diff --git a/plugin/user_variables/user_variables.cc b/plugin/user_variables/user_variables.cc index 9190b2effb7..35378e0c8c8 100644 --- a/plugin/user_variables/user_variables.cc +++ b/plugin/user_variables/user_variables.cc @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #define MYSQL_SERVER #include diff --git a/sql/sql_get_diagnostics.cc b/sql/sql_get_diagnostics.cc index 1713cb04ebc..fcda2463e13 100644 --- a/sql/sql_get_diagnostics.cc +++ b/sql/sql_get_diagnostics.cc @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ #include "sql_list.h" // Sql_alloc, List, List_iterator #include "sql_cmd.h" // Sql_cmd diff --git a/sql/sql_get_diagnostics.h b/sql/sql_get_diagnostics.h index f34820757f5..6f1652bb146 100644 --- a/sql/sql_get_diagnostics.h +++ b/sql/sql_get_diagnostics.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA */ #ifndef SQL_GET_DIAGNOSTICS_H #define SQL_GET_DIAGNOSTICS_H diff --git a/storage/rocksdb/event_listener.cc b/storage/rocksdb/event_listener.cc index 1a621b5df4b..4313a718eda 100644 --- a/storage/rocksdb/event_listener.cc +++ b/storage/rocksdb/event_listener.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/event_listener.h b/storage/rocksdb/event_listener.h index 8772105de36..f666c876eac 100644 --- a/storage/rocksdb/event_listener.h +++ b/storage/rocksdb/event_listener.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #include "rocksdb/listener.h" diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 03794291575..4a1f19f45de 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index c4447109b0d..aae823fe4e1 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #ifdef USE_PRAGMA_INTERFACE diff --git a/storage/rocksdb/ha_rocksdb_proto.h b/storage/rocksdb/ha_rocksdb_proto.h index 08afd9780be..26417328194 100644 --- a/storage/rocksdb/ha_rocksdb_proto.h +++ b/storage/rocksdb/ha_rocksdb_proto.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ standard header files */ diff --git a/storage/rocksdb/logger.h b/storage/rocksdb/logger.h index ca75caf9df5..b2820127711 100644 --- a/storage/rocksdb/logger.h +++ b/storage/rocksdb/logger.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #include diff --git a/storage/rocksdb/properties_collector.cc b/storage/rocksdb/properties_collector.cc index 8bc4f34691f..417ef447a69 100644 --- a/storage/rocksdb/properties_collector.cc +++ b/storage/rocksdb/properties_collector.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include #ifdef _WIN32 diff --git a/storage/rocksdb/properties_collector.h b/storage/rocksdb/properties_collector.h index c4a48265a50..36d980d8f53 100644 --- a/storage/rocksdb/properties_collector.h +++ b/storage/rocksdb/properties_collector.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ system header files */ diff --git a/storage/rocksdb/rdb_buff.h b/storage/rocksdb/rdb_buff.h index c9647707232..23645324470 100644 --- a/storage/rocksdb/rdb_buff.h +++ b/storage/rocksdb/rdb_buff.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #include diff --git a/storage/rocksdb/rdb_cf_manager.cc b/storage/rocksdb/rdb_cf_manager.cc index 9e2594ee246..777c937797b 100644 --- a/storage/rocksdb/rdb_cf_manager.cc +++ b/storage/rocksdb/rdb_cf_manager.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation diff --git a/storage/rocksdb/rdb_cf_manager.h b/storage/rocksdb/rdb_cf_manager.h index 3f27747dce1..943b6f07c2b 100644 --- a/storage/rocksdb/rdb_cf_manager.h +++ b/storage/rocksdb/rdb_cf_manager.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_cf_options.cc b/storage/rocksdb/rdb_cf_options.cc index befcd3c0829..eaa3d07be4b 100644 --- a/storage/rocksdb/rdb_cf_options.cc +++ b/storage/rocksdb/rdb_cf_options.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation diff --git a/storage/rocksdb/rdb_cf_options.h b/storage/rocksdb/rdb_cf_options.h index 19e5da6a79e..349f7c42e32 100644 --- a/storage/rocksdb/rdb_cf_options.h +++ b/storage/rocksdb/rdb_cf_options.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_compact_filter.h b/storage/rocksdb/rdb_compact_filter.h index 20ae3c740c1..ecc78de91bf 100644 --- a/storage/rocksdb/rdb_compact_filter.h +++ b/storage/rocksdb/rdb_compact_filter.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #ifdef USE_PRAGMA_IMPLEMENTATION diff --git a/storage/rocksdb/rdb_comparator.h b/storage/rocksdb/rdb_comparator.h index 963a8615c6b..b43118eda36 100644 --- a/storage/rocksdb/rdb_comparator.h +++ b/storage/rocksdb/rdb_comparator.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ system header files */ diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 38da527a7ab..4ca37897edd 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 2f699a9cd70..d24ffeb0691 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ standard header files */ diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index d541922dfd6..2f41c1e8a9f 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/rdb_i_s.h b/storage/rocksdb/rdb_i_s.h index f4c4b04a68d..a0783f7b8c0 100644 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once namespace myrocks { diff --git a/storage/rocksdb/rdb_index_merge.cc b/storage/rocksdb/rdb_index_merge.cc index 66c7955abf9..f1245a205b5 100644 --- a/storage/rocksdb/rdb_index_merge.cc +++ b/storage/rocksdb/rdb_index_merge.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/rdb_index_merge.h b/storage/rocksdb/rdb_index_merge.h index 6e53663160a..e70923bbb0e 100644 --- a/storage/rocksdb/rdb_index_merge.h +++ b/storage/rocksdb/rdb_index_merge.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_io_watchdog.cc b/storage/rocksdb/rdb_io_watchdog.cc index f09efefcd2a..5b809dbf553 100644 --- a/storage/rocksdb/rdb_io_watchdog.cc +++ b/storage/rocksdb/rdb_io_watchdog.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ /* This C++ file's header */ #include "./rdb_io_watchdog.h" diff --git a/storage/rocksdb/rdb_io_watchdog.h b/storage/rocksdb/rdb_io_watchdog.h index 8900de5b32a..9c391eee3f3 100644 --- a/storage/rocksdb/rdb_io_watchdog.h +++ b/storage/rocksdb/rdb_io_watchdog.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_mutex_wrapper.cc b/storage/rocksdb/rdb_mutex_wrapper.cc index ed81bc0dc90..5d58b5709d0 100644 --- a/storage/rocksdb/rdb_mutex_wrapper.cc +++ b/storage/rocksdb/rdb_mutex_wrapper.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/rdb_mutex_wrapper.h b/storage/rocksdb/rdb_mutex_wrapper.h index 287a6112345..567e81e5ef6 100644 --- a/storage/rocksdb/rdb_mutex_wrapper.h +++ b/storage/rocksdb/rdb_mutex_wrapper.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_perf_context.cc b/storage/rocksdb/rdb_perf_context.cc index a35b410e8d1..34584484dd2 100644 --- a/storage/rocksdb/rdb_perf_context.cc +++ b/storage/rocksdb/rdb_perf_context.cc @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/rdb_perf_context.h b/storage/rocksdb/rdb_perf_context.h index 2aca3dc3bfd..036c497c2f5 100644 --- a/storage/rocksdb/rdb_perf_context.h +++ b/storage/rocksdb/rdb_perf_context.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ standard header files */ diff --git a/storage/rocksdb/rdb_sst_info.cc b/storage/rocksdb/rdb_sst_info.cc index 8186f497d2b..41cd3c51307 100644 --- a/storage/rocksdb/rdb_sst_info.cc +++ b/storage/rocksdb/rdb_sst_info.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ /* For PRIu64 use below: */ #define __STDC_FORMAT_MACROS diff --git a/storage/rocksdb/rdb_sst_info.h b/storage/rocksdb/rdb_sst_info.h index 42f6458e46b..f50645b1eeb 100644 --- a/storage/rocksdb/rdb_sst_info.h +++ b/storage/rocksdb/rdb_sst_info.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once diff --git a/storage/rocksdb/rdb_threads.cc b/storage/rocksdb/rdb_threads.cc index b4bba8d3087..276f228d7cc 100644 --- a/storage/rocksdb/rdb_threads.cc +++ b/storage/rocksdb/rdb_threads.cc @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation diff --git a/storage/rocksdb/rdb_threads.h b/storage/rocksdb/rdb_threads.h index 3c2dd680d41..b3331db1738 100644 --- a/storage/rocksdb/rdb_threads.h +++ b/storage/rocksdb/rdb_threads.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once /* C++ standard header files */ diff --git a/storage/rocksdb/rdb_utils.cc b/storage/rocksdb/rdb_utils.cc index 457a3312a03..9bc93dfecec 100644 --- a/storage/rocksdb/rdb_utils.cc +++ b/storage/rocksdb/rdb_utils.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #include diff --git a/storage/rocksdb/rdb_utils.h b/storage/rocksdb/rdb_utils.h index 3125941ee78..cdc7651e3c4 100644 --- a/storage/rocksdb/rdb_utils.h +++ b/storage/rocksdb/rdb_utils.h @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ #pragma once #include "rdb_mariadb_port.h" diff --git a/storage/rocksdb/unittest/test_properties_collector.cc b/storage/rocksdb/unittest/test_properties_collector.cc index 46a3badc6ee..6870cd20803 100644 --- a/storage/rocksdb/unittest/test_properties_collector.cc +++ b/storage/rocksdb/unittest/test_properties_collector.cc @@ -12,7 +12,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ /* MyRocks header files */ #include "../ha_rocksdb.h" From 6af46f610fbc6de10ec634154161522b727ff14b Mon Sep 17 00:00:00 2001 From: Shinnok Date: Tue, 30 Oct 2018 12:28:03 +0000 Subject: [PATCH 091/127] README: Add reporting security vulns. direction Also change Jira link to actual domain. The mariadb.org/jira web server redirect might not be maintained at some point. --- README | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README b/README index a9413f16b04..06c2f227fb4 100644 --- a/README +++ b/README @@ -44,10 +44,14 @@ and COPYING.thirdparty files. IMPORTANT: -Bug and/or error reports regarding MariaDB should be submitted at -http://mariadb.org/jira +Bug and/or error reports regarding MariaDB should be submitted at: +https://jira.mariadb.org -Bugs in the MySQL code can also be submitted at http://bugs.mysql.com +For reporting security vulnerabilities see: +https://mariadb.org/about/security-policy/ + +Bugs in the MySQL code can also be submitted at: +http://bugs.mysql.com The code for MariaDB, including all revision history, can be found at: https://github.com/MariaDB/server From dc91ea5bb762f8ef9712babd8661ed6ae31bf2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Oct 2018 13:29:19 +0200 Subject: [PATCH 092/127] MDEV-12023 Assertion failure sym_node->table != NULL on startup row_drop_table_for_mysql(): Avoid accessing non-existing dictionary tables. dict_create_or_check_foreign_constraint_tables(): Add debug instrumentation for creating and dropping a table before the creation of any non-core dictionary tables. trx_purge_add_update_undo_to_history(): Adjust a debug assertion, so that it will not fail due to the test instrumentation. --- .../innodb/r/table_flags,32k,debug.rdiff | 134 +++++++++++ ...2k.rdiff => table_flags,32k,release.rdiff} | 0 .../suite/innodb/r/table_flags,4k,debug.rdiff | 133 +++++++++++ ...,4k.rdiff => table_flags,4k,release.rdiff} | 0 .../innodb/r/table_flags,64k,debug.rdiff | 134 +++++++++++ ...4k.rdiff => table_flags,64k,release.rdiff} | 0 .../suite/innodb/r/table_flags,8k,debug.rdiff | 133 +++++++++++ ...,8k.rdiff => table_flags,8k,release.rdiff} | 0 .../suite/innodb/r/table_flags,debug.rdiff | 133 +++++++++++ mysql-test/suite/innodb/t/table_flags.test | 4 + storage/innobase/dict/dict0crea.cc | 15 ++ storage/innobase/row/row0mysql.cc | 208 ++++++++---------- storage/innobase/trx/trx0purge.cc | 2 +- 13 files changed, 781 insertions(+), 115 deletions(-) create mode 100644 mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,32k.rdiff => table_flags,32k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,4k.rdiff => table_flags,4k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,64k.rdiff => table_flags,64k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff rename mysql-test/suite/innodb/r/{table_flags,8k.rdiff => table_flags,8k,release.rdiff} (100%) create mode 100644 mysql-test/suite/innodb/r/table_flags,debug.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff new file mode 100644 index 00000000000..4527a2ebaf2 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff @@ -0,0 +1,134 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,32k,debug.reject +@@ -5,96 +5,98 @@ + SET innodb_strict_mode=OFF; + CREATE TABLE tz(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=COMPRESSED + KEY_BLOCK_SIZE=1; ++Warnings: ++Warning 1478 InnoDB: Cannot create a COMPRESSED table when innodb_page_size > 16k. Assuming ROW_FORMAT=DYNAMIC. + SET innodb_strict_mode=ON; + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=10; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', + DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012c016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012c0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501b8 (NAME='SYS_TABLESPACES', ++header=0x040028150209 (NAME='SYS_TABLESPACES', + DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000030150244 (NAME='SYS_VIRTUAL', ++header=0x000038150251 (NAME='SYS_VIRTUAL', + DB_TRX_ID=0x000000000304, + DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ ID=0x0000000000000010, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000040150288 (NAME='test/tc', ++header=0x000040150295 (NAME='test/tc', + DB_TRX_ID=0x000000000308, + DB_ROLL_PTR=0x84000001380110, +- ID=0x0000000000000011, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x000048150310 (NAME='test/td', ++header=0x00004815031d (NAME='test/td', + DB_TRX_ID=0x00000000030a, + DB_ROLL_PTR=0x85000001390110, +- ID=0x0000000000000012, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x000058150200 (NAME='test/tp', ++header=0x00005815008d (NAME='test/tp', + DB_TRX_ID=0x00000000030e, + DB_ROLL_PTR=0x870000013b0110, +- ID=0x0000000000000014, ++ ID=0x0000000000000015, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x0000381502cc (NAME='test/tr', ++header=0x0000101502d9 (NAME='test/tr', + DB_TRX_ID=0x000000000306, + DB_ROLL_PTR=0x83000001370110, +- ID=0x0000000000000010, ++ ID=0x0000000000000011, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -104,9 +106,9 @@ + header=0x000050150074 (NAME='test/tz', + DB_TRX_ID=0x00000000030c, + DB_ROLL_PTR=0x860000013a0110, +- ID=0x0000000000000013, ++ ID=0x0000000000000014, + N_COLS=0x80000001, +- TYPE=0x00000023, ++ TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), diff --git a/mysql-test/suite/innodb/r/table_flags,32k.rdiff b/mysql-test/suite/innodb/r/table_flags,32k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,32k.rdiff rename to mysql-test/suite/innodb/r/table_flags,32k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff new file mode 100644 index 00000000000..b7d24f1b2be --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff @@ -0,0 +1,133 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,4k,debug.reject +@@ -10,91 +10,91 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=10; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ DB_ROLL_PTR=0x81000003270194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x8000000320016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x80000003200260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501b8 (NAME='SYS_TABLESPACES', ++header=0x040028150209 (NAME='SYS_TABLESPACES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ DB_ROLL_PTR=0x81000003270110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000030150244 (NAME='SYS_VIRTUAL', ++header=0x000038150251 (NAME='SYS_VIRTUAL', + DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ DB_ROLL_PTR=0x820000032b0110, ++ ID=0x0000000000000010, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000040150288 (NAME='test/tc', ++header=0x000040150295 (NAME='test/tc', + DB_TRX_ID=0x000000000308, +- DB_ROLL_PTR=0x84000001380110, +- ID=0x0000000000000011, ++ DB_ROLL_PTR=0x840000032f0110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x000048150310 (NAME='test/td', ++header=0x00004815031d (NAME='test/td', + DB_TRX_ID=0x00000000030a, +- DB_ROLL_PTR=0x85000001390110, +- ID=0x0000000000000012, ++ DB_ROLL_PTR=0x85000003300110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x000058150200 (NAME='test/tp', ++header=0x00005815008d (NAME='test/tp', + DB_TRX_ID=0x00000000030e, +- DB_ROLL_PTR=0x870000013b0110, +- ID=0x0000000000000014, ++ DB_ROLL_PTR=0x87000003320110, ++ ID=0x0000000000000015, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x0000381502cc (NAME='test/tr', ++header=0x0000101502d9 (NAME='test/tr', + DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x83000001370110, +- ID=0x0000000000000010, ++ DB_ROLL_PTR=0x830000032d0110, ++ ID=0x0000000000000011, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -103,8 +103,8 @@ + SPACE=0x00000001) + header=0x000050150074 (NAME='test/tz', + DB_TRX_ID=0x00000000030c, +- DB_ROLL_PTR=0x860000013a0110, +- ID=0x0000000000000013, ++ DB_ROLL_PTR=0x86000003310110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/r/table_flags,4k.rdiff b/mysql-test/suite/innodb/r/table_flags,4k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,4k.rdiff rename to mysql-test/suite/innodb/r/table_flags,4k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff new file mode 100644 index 00000000000..54f3fd451be --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff @@ -0,0 +1,134 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,64k,debug.reject +@@ -5,96 +5,98 @@ + SET innodb_strict_mode=OFF; + CREATE TABLE tz(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=COMPRESSED + KEY_BLOCK_SIZE=1; ++Warnings: ++Warning 1478 InnoDB: Cannot create a COMPRESSED table when innodb_page_size > 16k. Assuming ROW_FORMAT=DYNAMIC. + SET innodb_strict_mode=ON; + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=10; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', + DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012c016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012c0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501b8 (NAME='SYS_TABLESPACES', ++header=0x040028150209 (NAME='SYS_TABLESPACES', + DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000030150244 (NAME='SYS_VIRTUAL', ++header=0x000038150251 (NAME='SYS_VIRTUAL', + DB_TRX_ID=0x000000000304, + DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ ID=0x0000000000000010, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000040150288 (NAME='test/tc', ++header=0x000040150295 (NAME='test/tc', + DB_TRX_ID=0x000000000308, + DB_ROLL_PTR=0x84000001380110, +- ID=0x0000000000000011, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x000048150310 (NAME='test/td', ++header=0x00004815031d (NAME='test/td', + DB_TRX_ID=0x00000000030a, + DB_ROLL_PTR=0x85000001390110, +- ID=0x0000000000000012, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x000058150200 (NAME='test/tp', ++header=0x00005815008d (NAME='test/tp', + DB_TRX_ID=0x00000000030e, + DB_ROLL_PTR=0x870000013b0110, +- ID=0x0000000000000014, ++ ID=0x0000000000000015, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x0000381502cc (NAME='test/tr', ++header=0x0000101502d9 (NAME='test/tr', + DB_TRX_ID=0x000000000306, + DB_ROLL_PTR=0x83000001370110, +- ID=0x0000000000000010, ++ ID=0x0000000000000011, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -104,9 +106,9 @@ + header=0x000050150074 (NAME='test/tz', + DB_TRX_ID=0x00000000030c, + DB_ROLL_PTR=0x860000013a0110, +- ID=0x0000000000000013, ++ ID=0x0000000000000014, + N_COLS=0x80000001, +- TYPE=0x00000023, ++ TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), diff --git a/mysql-test/suite/innodb/r/table_flags,64k.rdiff b/mysql-test/suite/innodb/r/table_flags,64k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,64k.rdiff rename to mysql-test/suite/innodb/r/table_flags,64k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff new file mode 100644 index 00000000000..3e45c3cfe84 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff @@ -0,0 +1,133 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,8k,debug.reject +@@ -10,91 +10,91 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=10; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ DB_ROLL_PTR=0x81000001d80194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x80000001d1016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x80000001d10260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501b8 (NAME='SYS_TABLESPACES', ++header=0x040028150209 (NAME='SYS_TABLESPACES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ DB_ROLL_PTR=0x81000001d80110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000030150244 (NAME='SYS_VIRTUAL', ++header=0x000038150251 (NAME='SYS_VIRTUAL', + DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ DB_ROLL_PTR=0x82000001db0110, ++ ID=0x0000000000000010, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000040150288 (NAME='test/tc', ++header=0x000040150295 (NAME='test/tc', + DB_TRX_ID=0x000000000308, +- DB_ROLL_PTR=0x84000001380110, +- ID=0x0000000000000011, ++ DB_ROLL_PTR=0x84000001de0110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x000048150310 (NAME='test/td', ++header=0x00004815031d (NAME='test/td', + DB_TRX_ID=0x00000000030a, +- DB_ROLL_PTR=0x85000001390110, +- ID=0x0000000000000012, ++ DB_ROLL_PTR=0x85000001df0110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x000058150200 (NAME='test/tp', ++header=0x00005815008d (NAME='test/tp', + DB_TRX_ID=0x00000000030e, +- DB_ROLL_PTR=0x870000013b0110, +- ID=0x0000000000000014, ++ DB_ROLL_PTR=0x87000001e10110, ++ ID=0x0000000000000015, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x0000381502cc (NAME='test/tr', ++header=0x0000101502d9 (NAME='test/tr', + DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x83000001370110, +- ID=0x0000000000000010, ++ DB_ROLL_PTR=0x83000001dd0110, ++ ID=0x0000000000000011, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -103,8 +103,8 @@ + SPACE=0x00000001) + header=0x000050150074 (NAME='test/tz', + DB_TRX_ID=0x00000000030c, +- DB_ROLL_PTR=0x860000013a0110, +- ID=0x0000000000000013, ++ DB_ROLL_PTR=0x86000001e00110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/r/table_flags,8k.rdiff b/mysql-test/suite/innodb/r/table_flags,8k,release.rdiff similarity index 100% rename from mysql-test/suite/innodb/r/table_flags,8k.rdiff rename to mysql-test/suite/innodb/r/table_flags,8k,release.rdiff diff --git a/mysql-test/suite/innodb/r/table_flags,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,debug.rdiff new file mode 100644 index 00000000000..fef75cb3193 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,debug.rdiff @@ -0,0 +1,133 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,debug.reject +@@ -10,91 +10,91 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=10; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ DB_ROLL_PTR=0x81000001330194, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012d016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012d0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501b8 (NAME='SYS_TABLESPACES', ++header=0x040028150209 (NAME='SYS_TABLESPACES', + DB_TRX_ID=0x000000000302, +- DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ DB_ROLL_PTR=0x81000001330110, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000030150244 (NAME='SYS_VIRTUAL', ++header=0x000038150251 (NAME='SYS_VIRTUAL', + DB_TRX_ID=0x000000000304, +- DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ DB_ROLL_PTR=0x82000001360110, ++ ID=0x0000000000000010, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000040150288 (NAME='test/tc', ++header=0x000040150295 (NAME='test/tc', + DB_TRX_ID=0x000000000308, +- DB_ROLL_PTR=0x84000001380110, +- ID=0x0000000000000011, ++ DB_ROLL_PTR=0x84000001390110, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x000048150310 (NAME='test/td', ++header=0x00004815031d (NAME='test/td', + DB_TRX_ID=0x00000000030a, +- DB_ROLL_PTR=0x85000001390110, +- ID=0x0000000000000012, ++ DB_ROLL_PTR=0x850000013a0110, ++ ID=0x0000000000000013, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x000058150200 (NAME='test/tp', ++header=0x00005815008d (NAME='test/tp', + DB_TRX_ID=0x00000000030e, +- DB_ROLL_PTR=0x870000013b0110, +- ID=0x0000000000000014, ++ DB_ROLL_PTR=0x870000013c0110, ++ ID=0x0000000000000015, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x0000381502cc (NAME='test/tr', ++header=0x0000101502d9 (NAME='test/tr', + DB_TRX_ID=0x000000000306, +- DB_ROLL_PTR=0x83000001370110, +- ID=0x0000000000000010, ++ DB_ROLL_PTR=0x83000001380110, ++ ID=0x0000000000000011, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -103,8 +103,8 @@ + SPACE=0x00000001) + header=0x000050150074 (NAME='test/tz', + DB_TRX_ID=0x00000000030c, +- DB_ROLL_PTR=0x860000013a0110, +- ID=0x0000000000000013, ++ DB_ROLL_PTR=0x860000013b0110, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x00000023, + MIX_ID=0x0000000000000000, diff --git a/mysql-test/suite/innodb/t/table_flags.test b/mysql-test/suite/innodb/t/table_flags.test index 02d5a744e2c..07fe39664f2 100644 --- a/mysql-test/suite/innodb/t/table_flags.test +++ b/mysql-test/suite/innodb/t/table_flags.test @@ -1,6 +1,7 @@ --source include/innodb_page_size.inc # Embedded server tests do not support restarting --source include/not_embedded.inc +--source include/maybe_debug.inc --disable_query_log call mtr.add_suppression("InnoDB: Table `mysql`\\.`innodb_table_stats` not found"); @@ -28,6 +29,9 @@ let bugdir= $MYSQLTEST_VARDIR/tmp/table_flags; --let $d=--innodb-data-home-dir=$bugdir --innodb-log-group-home-dir=$bugdir --let $d=$d --innodb-data-file-path=ibdata1:1M:autoextend --let $d=$d --innodb-undo-tablespaces=0 +if ($have_debug) { +--let $d=$d --debug=d,create_and_drop_garbage +} --let $restart_parameters=$d --innodb-stats-persistent=0 --innodb-file-format=1 --source include/restart_mysqld.inc diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 2f9419efc62..ece81caafd8 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -1653,6 +1653,21 @@ dict_create_or_check_foreign_constraint_tables(void) row_mysql_lock_data_dictionary(trx); + DBUG_EXECUTE_IF( + "create_and_drop_garbage", + err = que_eval_sql( + NULL, + "PROCEDURE CREATE_GARBAGE_TABLE_PROC () IS\n" + "BEGIN\n" + "CREATE TABLE\n" + "\"test/#sql-ib-garbage\"(ID CHAR);\n" + "CREATE UNIQUE CLUSTERED INDEX PRIMARY" + " ON \"test/#sql-ib-garbage\"(ID);\n" + "END;\n", FALSE, trx); + ut_ad(err == DB_SUCCESS); + row_drop_table_for_mysql("test/#sql-ib-garbage", trx, + SQLCOM_DROP_DB, true);); + /* Check which incomplete table definition to drop. */ if (sys_foreign_err == DB_CORRUPTION) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 02a71d2fa63..434d538a7ad 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -37,6 +37,8 @@ Created 9/17/2000 Heikki Tuuri #include #include "dict0dict.h" #include "dict0load.h" +#include "dict0priv.h" +#include "dict0boot.h" #include "dict0stats.h" #include "dict0stats_bg.h" #include "dict0defrag_bg.h" @@ -3682,132 +3684,110 @@ defer: info = pars_info_create(); - pars_info_add_str_literal(info, "table_name", name); + pars_info_add_str_literal(info, "name", name); - err = (sqlcom == SQLCOM_TRUNCATE) ? DB_SUCCESS : que_eval_sql( - info, - "PROCEDURE DROP_FOREIGN_PROC () IS\n" - "sys_foreign_id CHAR;\n" - "table_id CHAR;\n" - "foreign_id CHAR;\n" - "space_id INT;\n" - "found INT;\n" + if (sqlcom != SQLCOM_TRUNCATE + && strchr(name, '/') + && dict_table_get_low("SYS_FOREIGN") + && dict_table_get_low("SYS_FOREIGN_COLS")) { + err = que_eval_sql( + info, + "PROCEDURE DROP_FOREIGN_PROC () IS\n" + "fid CHAR;\n" - "DECLARE CURSOR cur_fk IS\n" - "SELECT ID FROM SYS_FOREIGN\n" - "WHERE FOR_NAME = :table_name\n" - "AND TO_BINARY(FOR_NAME)\n" - " = TO_BINARY(:table_name)\n" - "LOCK IN SHARE MODE;\n" + "DECLARE CURSOR fk IS\n" + "SELECT ID FROM SYS_FOREIGN\n" + "WHERE FOR_NAME = :name\n" + "AND TO_BINARY(FOR_NAME) = TO_BINARY(:name)\n" + "FOR UPDATE;\n" - "BEGIN\n" - - "SELECT ID INTO table_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - - "SELECT SPACE INTO space_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" - - "found := 1;\n" - "SELECT ID INTO sys_foreign_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = 'SYS_FOREIGN'\n" - "LOCK IN SHARE MODE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN') THEN\n" - " found := 0;\n" - "END IF;\n" - "IF (:table_name = 'SYS_FOREIGN_COLS') \n" - "THEN\n" - " found := 0;\n" - "END IF;\n" - - "OPEN cur_fk;\n" - "WHILE found = 1 LOOP\n" - " FETCH cur_fk INTO foreign_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " ELSE\n" - " DELETE FROM \n" - " SYS_FOREIGN_COLS\n" - " WHERE ID = foreign_id;\n" - " DELETE FROM SYS_FOREIGN\n" - " WHERE ID = foreign_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_fk;\n" - - "END;\n", - FALSE, trx); - - if (err == DB_SUCCESS) { - if (sqlcom != SQLCOM_TRUNCATE) { + "BEGIN\n" + "OPEN fk;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH fk INTO fid;\n" + " IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" + " DELETE FROM SYS_FOREIGN_COLS WHERE ID=fid;\n" + " DELETE FROM SYS_FOREIGN WHERE ID=fid;\n" + "END LOOP;\n" + "CLOSE fk;\n" + "END;\n", FALSE, trx); + if (err == DB_SUCCESS) { info = pars_info_create(); - pars_info_add_str_literal(info, "table_name", - name); + pars_info_add_str_literal(info, "name", name); + goto do_drop; + } + } else { +do_drop: + if (dict_table_get_low("SYS_VIRTUAL")) { + err = que_eval_sql( + info, + "PROCEDURE DROP_VIRTUAL_PROC () IS\n" + "tid CHAR;\n" + + "BEGIN\n" + "SELECT ID INTO tid FROM SYS_TABLES\n" + "WHERE NAME = :name FOR UPDATE;\n" + "IF (SQL % NOTFOUND) THEN RETURN;" + " END IF;\n" + "DELETE FROM SYS_VIRTUAL" + " WHERE TABLE_ID = tid;\n" + "END;\n", FALSE, trx); + if (err == DB_SUCCESS) { + info = pars_info_create(); + pars_info_add_str_literal( + info, "name", name); + } + } else { + err = DB_SUCCESS; } - err = que_eval_sql( - info, - "PROCEDURE DROP_TABLE_PROC () IS\n" - "table_id CHAR;\n" - "space_id INT;\n" - "index_id CHAR;\n" + err = err == DB_SUCCESS ? que_eval_sql( + info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "tid CHAR;\n" + "iid CHAR;\n" - "DECLARE CURSOR cur_idx IS\n" - "SELECT ID FROM SYS_INDEXES\n" - "WHERE TABLE_ID = table_id\n" - "FOR UPDATE;\n" + "DECLARE CURSOR cur_idx IS\n" + "SELECT ID FROM SYS_INDEXES\n" + "WHERE TABLE_ID = tid FOR UPDATE;\n" - "BEGIN\n" - "SELECT ID, SPACE INTO table_id,space_id\n" - "FROM SYS_TABLES\n" - "WHERE NAME = :table_name FOR UPDATE;\n" - "IF (SQL % NOTFOUND) THEN\n" - " RETURN;\n" - "END IF;\n" + "BEGIN\n" + "SELECT ID INTO tid FROM SYS_TABLES\n" + "WHERE NAME = :name FOR UPDATE;\n" + "IF (SQL % NOTFOUND) THEN RETURN; END IF;\n" - "DELETE FROM SYS_COLUMNS\n" - "WHERE TABLE_ID = table_id;\n" - "DELETE FROM SYS_TABLES\n" - "WHERE NAME = :table_name;\n" + "OPEN cur_idx;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH cur_idx INTO iid;\n" + " IF (SQL % NOTFOUND) THEN EXIT; END IF;\n" + " DELETE FROM SYS_FIELDS\n" + " WHERE INDEX_ID = iid;\n" + " DELETE FROM SYS_INDEXES\n" + " WHERE ID = iid AND TABLE_ID = tid;\n" + "END LOOP;\n" + "CLOSE cur_idx;\n" - "DELETE FROM SYS_TABLESPACES\n" - "WHERE SPACE = space_id;\n" - "DELETE FROM SYS_DATAFILES\n" - "WHERE SPACE = space_id;\n" + "DELETE FROM SYS_COLUMNS WHERE TABLE_ID=tid;\n" + "DELETE FROM SYS_TABLES WHERE NAME=:name;\n" - "DELETE FROM SYS_VIRTUAL\n" - "WHERE TABLE_ID = table_id;\n" + "END;\n", FALSE, trx) : err; - "OPEN cur_idx;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH cur_idx INTO index_id;\n" - " IF (SQL % NOTFOUND) THEN\n" - " EXIT;\n" - " ELSE\n" - " DELETE FROM SYS_FIELDS\n" - " WHERE INDEX_ID = index_id;\n" - " DELETE FROM SYS_INDEXES\n" - " WHERE ID = index_id\n" - " AND TABLE_ID = table_id;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE cur_idx;\n" - - "END;\n", - FALSE, trx); + if (err == DB_SUCCESS && table->space + && dict_table_get_low("SYS_TABLESPACES") + && dict_table_get_low("SYS_DATAFILES")) { + info = pars_info_create(); + pars_info_add_int4_literal(info, "id", + lint(table->space)); + err = que_eval_sql( + info, + "PROCEDURE DROP_SPACE_PROC () IS\n" + "BEGIN\n" + "DELETE FROM SYS_TABLESPACES\n" + "WHERE SPACE = :id;\n" + "DELETE FROM SYS_DATAFILES\n" + "WHERE SPACE = :id;\n" + "END;\n", FALSE, trx); + } } } else { page_no = page_nos; diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 6a88b335f17..825b9eab3a6 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -296,7 +296,7 @@ trx_purge_add_update_undo_to_history( user transactions. */ ut_ad(srv_undo_sources || trx->undo_no == 0 - || ((srv_startup_is_before_trx_rollback_phase + || ((srv_is_being_started || trx_rollback_or_clean_is_active) && purge_sys->state == PURGE_STATE_INIT) || (srv_force_recovery >= SRV_FORCE_NO_BACKGROUND From 57898316b6fb8920cb68a473c33edbb583da4d89 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Tue, 30 Oct 2018 18:15:41 +0400 Subject: [PATCH 093/127] MDEV-17256 Decimal field multiplication bug. We should clear trailing zeroes in frac part. Otherwise that tail is growing quickly and forces unnecessary truncating of arguments. --- mysql-test/r/type_newdecimal.result | 42 +++++++++++++++++++++++++++ mysql-test/t/type_newdecimal.test | 44 +++++++++++++++++++++++++++++ strings/decimal.c | 35 ++++++++++++----------- 3 files changed, 105 insertions(+), 16 deletions(-) diff --git a/mysql-test/r/type_newdecimal.result b/mysql-test/r/type_newdecimal.result index 3cd5028f858..81d1637947c 100644 --- a/mysql-test/r/type_newdecimal.result +++ b/mysql-test/r/type_newdecimal.result @@ -1997,3 +1997,45 @@ select 0.0000000001 mod 1; select 0.01 mod 1; 0.01 mod 1 0.01 +CREATE TABLE t1 ( +`FLD1` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD2` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD3` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD4` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD5` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD6` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD7` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD8` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD9` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD10` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD11` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD12` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD13` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD14` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD15` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD16` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD17` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD18` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD19` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD20` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD21` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD22` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, +`FLD23` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000 +); +INSERT INTO t1 VALUES (001.0760,000.9500,001.0000,001.0000,001.0000, +001.0000,001.0000,001.0000,001.0000,001.0000,001.0000,000.5949,001.0194, +001.0000,001.0000,001.0000,001.0000,001.0000,001.0000,000.9220,001.1890,001.2130,327.2690); +select FLD1*FLD2*FLD3*FLD4*FLD5*FLD6*FLD7*FLD8*FLD9*FLD10*FLD11*FLD12*FLD13*FLD14*FLD15*FLD16*FLD17*FLD18*FLD19*FLD20*FLD21*FLD22*FLD23 as calc1 from t1; +calc1 +269.775757576440530322187032000000 +select FLD23*FLD2*FLD1*FLD4*FLD5*FLD11*FLD12*FLD13*FLD3*FLD15*FLD16*FLD17*FLD18*FLD19*FLD20*FLD21*FLD22*FLD14*FLD6*FLD7*FLD8*FLD9*FLD10 as calc2 from t1; +calc2 +269.775757576440530322187032000000 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT 1.0 * 2.000; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `1.0 * 2.000` decimal(6,4) NOT NULL DEFAULT '0.0000' +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; diff --git a/mysql-test/t/type_newdecimal.test b/mysql-test/t/type_newdecimal.test index bb5a61d84ec..adc8eb77198 100644 --- a/mysql-test/t/type_newdecimal.test +++ b/mysql-test/t/type_newdecimal.test @@ -1581,3 +1581,47 @@ select 0.000000000000000000000000000000000000000000000000001 mod 1; select 0.0000000001 mod 1; select 0.01 mod 1; +# +# MDEV-17256 Decimal field multiplication bug +# + +CREATE TABLE t1 ( + `FLD1` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD2` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD3` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD4` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD5` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD6` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD7` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD8` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD9` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD10` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD11` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD12` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD13` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD14` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD15` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD16` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD17` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD18` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD19` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD20` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD21` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD22` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000, + `FLD23` decimal(7,4) unsigned zerofill NOT NULL DEFAULT 001.0000 +); + +INSERT INTO t1 VALUES (001.0760,000.9500,001.0000,001.0000,001.0000, + 001.0000,001.0000,001.0000,001.0000,001.0000,001.0000,000.5949,001.0194, + 001.0000,001.0000,001.0000,001.0000,001.0000,001.0000,000.9220,001.1890,001.2130,327.2690); + +select FLD1*FLD2*FLD3*FLD4*FLD5*FLD6*FLD7*FLD8*FLD9*FLD10*FLD11*FLD12*FLD13*FLD14*FLD15*FLD16*FLD17*FLD18*FLD19*FLD20*FLD21*FLD22*FLD23 as calc1 from t1; +select FLD23*FLD2*FLD1*FLD4*FLD5*FLD11*FLD12*FLD13*FLD3*FLD15*FLD16*FLD17*FLD18*FLD19*FLD20*FLD21*FLD22*FLD14*FLD6*FLD7*FLD8*FLD9*FLD10 as calc2 from t1; + +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT 1.0 * 2.000; +SHOW CREATE TABLE t1; +DROP TABLE t1; + + diff --git a/strings/decimal.c b/strings/decimal.c index 3d90a58ea8a..05546aa77a3 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -2077,26 +2077,21 @@ int decimal_mul(const decimal_t *from1, const decimal_t *from2, decimal_t *to) } } - /* Now we have to check for -0.000 case */ - if (to->sign) + /* Remove trailing zero words in frac part */ + frac0= ROUND_UP(to->frac); + + if (frac0 > 0 && to->buf[intg0 + frac0 - 1] == 0) { - dec1 *buf= to->buf; - dec1 *end= to->buf + intg0 + frac0; - DBUG_ASSERT(buf != end); - for (;;) + do { - if (*buf) - break; - if (++buf == end) - { - /* We got decimal zero */ - decimal_make_zero(to); - break; - } - } + frac0--; + } while (frac0 > 0 && to->buf[intg0 + frac0 - 1] == 0); + to->frac= DIG_PER_DEC1 * frac0; } + + /* Remove heading zero words in intg part */ buf1= to->buf; - d_to_move= intg0 + ROUND_UP(to->frac); + d_to_move= intg0 + frac0; while (!*buf1 && (to->intg > DIG_PER_DEC1)) { buf1++; @@ -2109,6 +2104,14 @@ int decimal_mul(const decimal_t *from1, const decimal_t *from2, decimal_t *to) for (; d_to_move--; cur_d++, buf1++) *cur_d= *buf1; } + + /* Now we have to check for -0.000 case */ + if (to->sign && to->frac == 0 && to->buf[0] == 0) + { + DBUG_ASSERT(to->intg <= DIG_PER_DEC1); + /* We got decimal zero */ + decimal_make_zero(to); + } return error; } From a737135ae39dafe8b1136386ce23dfa8bed877f9 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Tue, 30 Oct 2018 18:15:58 +0200 Subject: [PATCH 094/127] List of unstable tests for 10.0.37 release --- mysql-test/unstable-tests | 82 +++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index e8bb02f51e0..6725e2ae9ff 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,9 +23,9 @@ # ############################################################################## -# Based on 10.0 e52315a4a2c7e752e786f9fcf63b0b5685b0d474 +# Based on 10.0 6ced789186fabd7dce97b3d6d171ff9e5ddc5f48 -main.alter_table : Modified in 10.0.36 +main.alter_table : Modified in 10.0.37 main.assign_key_cache : Added in 10.0.36 main.assign_key_cache_debug : Added in 10.0.36 main.auto_increment : Modified in 10.0.36 @@ -33,39 +33,45 @@ main.bootstrap : Modified in 10.0.36 main.connect_debug : Added in 10.0.36 main.count_distinct2 : MDEV-11768 - timeout main.create_delayed : MDEV-10605 - failed with timeout -main.create_or_replace : Modified in 10.0.36 -ctype_binary : Modified in 10.0.36 -ctype_eucjpms : Modified in 10.0.36 -ctype_euckr : Modified in 10.0.36 -ctype_gbk : Modified in 10.0.36 -ctype_latin1 : Modified in 10.0.36 -ctype_ucs : Modified in 10.0.36 -ctype_ujis : Modified in 10.0.36 -ctype_utf16le : Modified in 10.0.36 -ctype_utf16 : Modified in 10.0.36 -ctype_utf32 : Modified in 10.0.36 -ctype_utf8mb4 : Modified in 10.0.36 -ctype_utf8 : Modified in 10.0.36 +main.create_or_replace : Modified in 10.0.37 +main.ctype_binary : Modified in 10.0.36 +main.ctype_eucjpms : Modified in 10.0.36 +main.ctype_euckr : Modified in 10.0.36 +main.ctype_gbk : Modified in 10.0.36 +main.ctype_latin1 : Modified in 10.0.36 +main.ctype_uca : Modified in 10.0.37 +main.ctype_ucs : Modified in 10.0.36 +main.ctype_ujis : Modified in 10.0.36 +main.ctype_utf16le : Modified in 10.0.36 +main.ctype_utf16 : Modified in 10.0.36 +main.ctype_utf32 : Modified in 10.0.36 +main.ctype_utf8mb4 : Modified in 10.0.36 +main.ctype_utf8 : Modified in 10.0.36 main.debug_sync : MDEV-10607 - internal error main.derived : Modified in 10.0.36 -main.derived_opt : MDEV-11768 - timeout +main.derived_opt : MDEV-11768 - timeout; modified in 10.0.37 main.events_slowlog : MDEV-12821 - wrong result -main.gis : MDEV-13411 - wrong result on P8 -main.grant : Modified in 10.0.36 +main.func_concat : Modified in 10.0.37 +main.func_isnull : Modified in 10.0.37 +main.func_time : Modified in 10.0.37 +main.gis : MDEV-13411 - wrong result on P8; modified in 10.0.37 +main.grant : Modified in 10.0.37 main.grant2 : Modified in 10.0.36 main.grant_not_windows : Added in 10.0.36 +main.group_min_max : Modified in 10.0.37 main.having : Modified in 10.0.36 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - wrong result main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.insert_select : Modified in 10.0.36 -main.join : Modified in 10.0.36 +main.join : Modified in 10.0.37 main.join_cache : Modified in 10.0.36 main.join_outer : Modified in 10.0.36 main.kill_processlist-6619 : MDEV-10793 - wrong result main.limit : Modified in 10.0.36 main.log_tables-big : MDEV-13408 - wrong result +main.lowercase_fs_off : Modified in 10.0.37 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" @@ -77,26 +83,32 @@ main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqlslap : Modified in 10.0.36 main.mysqltest : MDEV-9269 - fails on Alpha main.mysql_client_test_nonblock : MDEV-15096 - exec failed +main.order_by_zerolength-4285 : Modified in 10.0.37 +main.partition_explicit_prune : Modified in 10.0.37 main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.rename : Modified in 10.0.36 main.query_cache_debug : MDEV-15281 - resize or similar command in progress -main.selectivity : Modified in 10.0.36 +main.selectivity : Modified in 10.0.37 main.show_explain : MDEV-10674 - wrong result +main.sp : Modified in 10.0.37 main.sp-innodb : Modified in 10.0.36 main.sp_notembedded : MDEV-10607 - internal error -main.sp-security : MDEV-10607 - sporadic "can't connect" +main.sp-security : MDEV-10607 - sporadic "can't connect"; modified in 10.0.37 main.statistics : Modified in 10.0.36 main.statistics_close : Added in 10.0.36 -main.stat_tables : Modified in 10.0.36 +main.stat_tables : Modified in 10.0.37 main.stat_tables_par_innodb : MDEV-14155 - wrong rounding main.subselect : Modified in 10.0.36 +main.subselect_extra_no_semijoin : Modified in 10.0.37 main.subselect_innodb : MDEV-10614 - sporadic wrong results main.subselect_sj : Modified in 10.0.36 main.subselect_sj_mat : Modified in 10.0.36 main.subselect_sj2_mat : Modified in 10.0.36 main.subselect4 : Modified in 10.0.36 main.tc_heuristic_recover : MDEV-15200 - wrong error on mysqld_stub_cmd -main.type_datetime : MDEV-14322 - wrong result +main.type_datetime : Modified in 10.0.37 +main.type_float : Modified in 10.0.37 +main.type_year : Modified in 10.0.37 main.union : Modified in 10.0.36 main.xa : MDEV-11769 - lock wait timeout @@ -148,14 +160,19 @@ handler.ps : Added in 10.0.36 #---------------------------------------------------------------- heap.heap_auto_increment : Modified in 10.0.36 +heap.heap_btree : Modified in 10.0.37 #---------------------------------------------------------------- +innodb.alter_inplace_perfschema : Added in 10.0.37 innodb.alter_partitioned_xa : Added in 10.0.36 innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.foreign-keys : Modified in 10.0.37 +innodb.foreign_key : Added in 10.0.37 innodb.group_commit_crash : MDEV-11770 - checksum mismatch innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch innodb.innodb-alter : Modified in 10.0.36 +innodb.innodb-alter-debug : Modified in 10.0.37 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan innodb.innodb_bug48024 : MDEV-14352 - Assertion failure @@ -167,9 +184,11 @@ innodb.log_file_size : MDEV-15668 - Not found pattern innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption innodb.rename_table : Added in 10.0.36 innodb.table_definition_cache_debug : MDEV-14206 - Unexpected warning +innodb.table_flags : Modified in 10.0.37 innodb.xa_recovery : MDEV-15279 - mysqld got exception innodb_fts.basic : Added in 10.0.36 +innodb_fts.fts_kill_query : Modified in 10.0.37 innodb_fts.innodb-fts-fic : MDEV-14154 - Assertion failure innodb_fts.innodb_fts_misc_debug : MDEV-14156 - Unexpected warning innodb_fts.sync_ddl : Added in 10.0.36 @@ -177,8 +196,10 @@ innodb_fts.sync_ddl : Added in 10.0.36 #---------------------------------------------------------------- maria.alter : Modified in 10.0.36 +maria.create : Added in 10.0.37 +maria.fulltext2 : Added in 10.0.37 maria.lock : Modified in 10.0.36 -maria.maria : MDEV-14430 - Wrong result; modified in 10.0.36 +maria.maria : MDEV-14430 - Wrong result; modified in 10.0.37 #---------------------------------------------------------------- @@ -205,9 +226,11 @@ parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed parts.partition_auto_increment_maria : MDEV-14430 - wrong result parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.truncate_locked : Added in 10.0.36 +parts.update_and_cache : Added in 10.0.37 #---------------------------------------------------------------- +perfschema.connect_attrs : MDEV-17283 - Wrong result perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown @@ -219,7 +242,6 @@ perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result -plugins.processlist : MDEV-16818 - Added in 10.0.36 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.0.36 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc, MDEV-14295 - wrong result @@ -231,11 +253,13 @@ roles.create_and_grant_role : MDEV-11772 - wrong result rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rename : Added in 10.0.36 +rpl.rpl_15919 : Added in 10.0.37 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering on master rpl.rpl_ddl : MDEV-10417 - Fails on Mips +rpl.rpl_foreign_key_innodb : Modified in 10.0.37 rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log @@ -244,6 +268,7 @@ rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_id_pk : MDEV-16567 - Assertion failure rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips +rpl.rpl_lcase_tblnames_rewrite_db : Added in 10.0.37 rpl.rpl_mdev6020 : MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mixed_implicit_commit_binlog : Included file modified in 10.0.36 rpl.rpl_parallel : MDEV-10653 - Timeouts @@ -254,13 +279,16 @@ rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_row_basic_11bugs : MDEV-12171 - Server failed to start rpl.rpl_row_implicit_commit_binlog : Included file modified in 10.0.36 rpl.rpl_row_index_choice : MDEV-13409 - Server crash +rpl.rpl_row_lcase_tblnames : Added in 10.0.37 rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x +rpl.rpl_row_spatial : Added in 10.0.37 rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_show_slave_hosts : MDEV-12171 - Server failed to start rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock rpl.rpl_stm_implicit_commit_binlog : Included file modified in 10.0.36 +rpl.rpl_stm_lcase_tblnames : Added in 10.0.37 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries @@ -294,6 +322,8 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x +sys_vars.innodb_ft_result_cache_limit_32 : Added in 10.0.37 +sys_vars.innodb_ft_result_cache_limit_64 : Added in 10.0.37 sys_vars.thread_cache_size_func : MDEV-11775 - wrong result #---------------------------------------------------------------- @@ -314,6 +344,8 @@ tokudb.savepoint-5 : MDEV-15280 - wrong result tokudb_backup.* : MDEV-11001 - tests don't work +tokudb_bugs.PS-3773 : Added in 10.0.37 +tokudb_bugs.alter_table_comment_rebuild_data : Added in 10.0.37 tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output tokudb_bugs.frm_store : MDEV-12823 - Valgrind From 250c5aa02cd6d0dd68b5ae3f1ddc35be951c1349 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 30 Oct 2018 20:13:15 +0100 Subject: [PATCH 095/127] ./mtr --gdb='b mysql_parse;r' --gdb now accepts an argument, it will be passed to gdb as a command. multiple commands can be separated by a (non-standard and not escapable) delimiter - semicolon (;). Old usage with a bare --gdb continues to work too, of course. Cherry-picked c47c0ca50c4 5441bbd3b1f 339b9055791 --- mysql-test/mysql-test-run.pl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index ade3f99104e..8253af5fbbd 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1140,7 +1140,7 @@ sub command_line_setup { 'debug' => \$opt_debug, 'debug-common' => \$opt_debug_common, 'debug-server' => \$opt_debug_server, - 'gdb' => \$opt_gdb, + 'gdb=s' => \$opt_gdb, 'client-gdb' => \$opt_client_gdb, 'manual-gdb' => \$opt_manual_gdb, 'manual-lldb' => \$opt_manual_lldb, @@ -1234,6 +1234,9 @@ sub command_line_setup { 'skip-test-list=s' => \@opt_skip_test_list ); + # fix options (that take an optional argument and *only* after = sign + my %fixopt = ( '--gdb' => '--gdb=#' ); + @ARGV = map { $fixopt{$_} or $_ } @ARGV; GetOptions(%options) or usage("Can't read options"); usage("") if $opt_usage; list_options(\%options) if $opt_list_options; @@ -5992,7 +5995,9 @@ sub gdb_arguments { # Put $args into a single string $input = $input ? "< $input" : ""; - if ($type ne 'client' and $opt_valgrind_mysqld) { + if ($type eq 'client') { + mtr_tofile($gdb_init_file, "set args @$$args $input"); + } elsif ($opt_valgrind_mysqld) { my $v = $$exe; my $vargs = []; valgrind_arguments($vargs, \$v); @@ -6002,7 +6007,11 @@ shell sleep 1 target remote | /usr/lib64/valgrind/../../bin/vgdb EOF } else { - mtr_tofile($gdb_init_file, "set args @$$args $input\n"); + mtr_tofile($gdb_init_file, + join("\n", + "set args @$$args $input", + split /;/, $opt_gdb || "" + )); } if ( $opt_manual_gdb ) From 31f1fe223e3ca0f2a44f7e55df05f096552ae61e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 30 Oct 2018 20:19:56 +0100 Subject: [PATCH 096/127] don't try to build with OpenSSL 1.1+ --- cmake/ssl.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake index 6f7bd92c777..0c6cde31299 100644 --- a/cmake/ssl.cmake +++ b/cmake/ssl.cmake @@ -79,7 +79,8 @@ MACRO (MYSQL_CHECK_SSL) HAVE_ERR_remove_thread_state) SET(CMAKE_REQUIRED_INCLUDES) SET(CMAKE_REQUIRED_LIBRARIES) - IF(OPENSSL_FOUND AND CRYPTO_LIBRARY AND HAVE_SHA512_DIGEST_LENGTH) + IF(OPENSSL_FOUND AND OPENSSL_VERSION VERSION_LESS "1.1.0" AND + CRYPTO_LIBRARY AND HAVE_SHA512_DIGEST_LENGTH) SET(SSL_SOURCES "") SET(SSL_LIBRARIES ${OPENSSL_LIBRARIES} ${CRYPTO_LIBRARY}) SET(SSL_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) From b92ff45089eb21ede352f5fd18dc69baa4bcb13f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 31 Oct 2018 10:35:15 +0100 Subject: [PATCH 097/127] disabling a crashing test --- mysql-test/suite/innodb/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/disabled.def b/mysql-test/suite/innodb/disabled.def index 8cae44a3607..e8991f6b851 100644 --- a/mysql-test/suite/innodb/disabled.def +++ b/mysql-test/suite/innodb/disabled.def @@ -9,4 +9,4 @@ # Do not use any TAB characters for whitespace. # ############################################################################## - +table_flags: crashes on assert, Marko to fix From a5cbdd63bc2ff25a52e2e10f84b6aaf59837dbae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 31 Oct 2018 12:08:28 +0200 Subject: [PATCH 098/127] Fix innodb.table_flags,debug --- mysql-test/suite/innodb/disabled.def | 2 +- .../innodb/r/table_flags,32k,debug.rdiff | 130 ++++++++++++++++++ .../suite/innodb/r/table_flags,4k,debug.rdiff | 8 +- .../innodb/r/table_flags,64k,debug.rdiff | 130 ++++++++++++++++++ .../suite/innodb/r/table_flags,8k,debug.rdiff | 8 +- .../suite/innodb/r/table_flags,debug.rdiff | 8 +- storage/innobase/row/row0mysql.cc | 4 +- storage/xtradb/row/row0mysql.cc | 4 +- 8 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff create mode 100644 mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff diff --git a/mysql-test/suite/innodb/disabled.def b/mysql-test/suite/innodb/disabled.def index e8991f6b851..8cae44a3607 100644 --- a/mysql-test/suite/innodb/disabled.def +++ b/mysql-test/suite/innodb/disabled.def @@ -9,4 +9,4 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -table_flags: crashes on assert, Marko to fix + diff --git a/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff new file mode 100644 index 00000000000..13fdc5657c4 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,32k,debug.rdiff @@ -0,0 +1,130 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,32k,debug.reject +@@ -4,85 +4,87 @@ + CREATE TABLE td(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + CREATE TABLE tz(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=COMPRESSED + KEY_BLOCK_SIZE=1; ++Warnings: ++Warning 1478 InnoDB: Cannot create a COMPRESSED table when innodb_page_size > 16k. Assuming ROW_FORMAT=COMPACT. + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', +- DB_TRX_ID=0x000000000301, ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', ++ DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012c016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012c0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501fc (NAME='SYS_TABLESPACES', +- DB_TRX_ID=0x000000000301, ++header=0x040028150209 (NAME='SYS_TABLESPACES', ++ DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000038150240 (NAME='test/tc', +- DB_TRX_ID=0x000000000303, ++header=0x00003815024d (NAME='test/tc', ++ DB_TRX_ID=0x000000000304, + DB_ROLL_PTR=0x83000001360110, +- ID=0x0000000000000010, ++ ID=0x0000000000000011, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x0000401502c8 (NAME='test/td', +- DB_TRX_ID=0x000000000304, ++header=0x0000401502d5 (NAME='test/td', ++ DB_TRX_ID=0x000000000305, + DB_ROLL_PTR=0x84000001370110, +- ID=0x0000000000000011, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x0000501501b8 (NAME='test/tp', +- DB_TRX_ID=0x000000000306, ++header=0x00005015008d (NAME='test/tp', ++ DB_TRX_ID=0x000000000307, + DB_ROLL_PTR=0x86000001390110, +- ID=0x0000000000000013, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x000030150284 (NAME='test/tr', +- DB_TRX_ID=0x000000000302, ++header=0x000010150291 (NAME='test/tr', ++ DB_TRX_ID=0x000000000303, + DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ ID=0x0000000000000010, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -90,11 +92,11 @@ + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000001) + header=0x000048150074 (NAME='test/tz', +- DB_TRX_ID=0x000000000305, ++ DB_TRX_ID=0x000000000306, + DB_ROLL_PTR=0x85000001380110, +- ID=0x0000000000000012, ++ ID=0x0000000000000013, + N_COLS=0x80000001, +- TYPE=0x00000023, ++ TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), diff --git a/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff index 83e40b2d3fd..b49aba8f661 100644 --- a/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff +++ b/mysql-test/suite/innodb/r/table_flags,4k,debug.rdiff @@ -1,7 +1,7 @@ --- suite/innodb/r/table_flags.result +++ suite/innodb/r/table_flags,4k,debug.reject -@@ -10,81 +10,81 @@ - CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +@@ -8,81 +8,81 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; SYS_TABLES clustered index root page (8): N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 -header=0x01000003016e (NAME=0x696e66696d756d00) @@ -97,7 +97,7 @@ + DB_ROLL_PTR=0x86000003300110, + ID=0x0000000000000014, N_COLS=0x80000001, - TYPE=0x00000021, + TYPE=0x000009a1, MIX_ID=0x0000000000000000, MIX_LEN=0x00000050, CLUSTER_NAME=NULL(0 bytes), @@ -113,7 +113,7 @@ N_COLS=0x00000001, TYPE=0x00000001, MIX_ID=0x0000000000000000, -@@ -92,9 +92,9 @@ +@@ -90,9 +90,9 @@ CLUSTER_NAME=NULL(0 bytes), SPACE=0x00000001) header=0x000048150074 (NAME='test/tz', diff --git a/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff new file mode 100644 index 00000000000..dcd5f2329c7 --- /dev/null +++ b/mysql-test/suite/innodb/r/table_flags,64k,debug.rdiff @@ -0,0 +1,130 @@ +--- suite/innodb/r/table_flags.result ++++ suite/innodb/r/table_flags,64k,debug.reject +@@ -4,85 +4,87 @@ + CREATE TABLE td(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; + CREATE TABLE tz(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=COMPRESSED + KEY_BLOCK_SIZE=1; ++Warnings: ++Warning 1478 InnoDB: Cannot create a COMPRESSED table when innodb_page_size > 16k. Assuming ROW_FORMAT=COMPACT. + CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; + SYS_TABLES clustered index root page (8): + N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 +-header=0x01000003016e (NAME=0x696e66696d756d00) +-header=0x00002815008d (NAME='SYS_DATAFILES', +- DB_TRX_ID=0x000000000301, ++header=0x0100000301bf (NAME=0x696e66696d756d00) ++header=0x0000301500de (NAME='SYS_DATAFILES', ++ DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320194, +- ID=0x000000000000000e, ++ ID=0x000000000000000f, + N_COLS=0x00000002, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0000101500d5 (NAME='SYS_FOREIGN', ++header=0x000018150126 (NAME='SYS_FOREIGN', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0110, +- ID=0x000000000000000b, ++ DB_ROLL_PTR=0x800000012c016f, ++ ID=0x000000000000000c, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000018150122 (NAME='SYS_FOREIGN_COLS', ++header=0x000020150173 (NAME='SYS_FOREIGN_COLS', + DB_TRX_ID=0x000000000300, +- DB_ROLL_PTR=0x800000012d0201, +- ID=0x000000000000000c, ++ DB_ROLL_PTR=0x800000012c0260, ++ ID=0x000000000000000d, + N_COLS=0x00000004, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x0400201501fc (NAME='SYS_TABLESPACES', +- DB_TRX_ID=0x000000000301, ++header=0x040028150209 (NAME='SYS_TABLESPACES', ++ DB_TRX_ID=0x000000000302, + DB_ROLL_PTR=0x81000001320110, +- ID=0x000000000000000d, ++ ID=0x000000000000000e, + N_COLS=0x00000003, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000040, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000000) +-header=0x000038150240 (NAME='test/tc', +- DB_TRX_ID=0x000000000303, ++header=0x00003815024d (NAME='test/tc', ++ DB_TRX_ID=0x000000000304, + DB_ROLL_PTR=0x83000001360110, +- ID=0x0000000000000010, ++ ID=0x0000000000000011, + N_COLS=0x80000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000002) +-header=0x0000401502c8 (NAME='test/td', +- DB_TRX_ID=0x000000000304, ++header=0x0000401502d5 (NAME='test/td', ++ DB_TRX_ID=0x000000000305, + DB_ROLL_PTR=0x84000001370110, +- ID=0x0000000000000011, ++ ID=0x0000000000000012, + N_COLS=0x80000001, + TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000003) +-header=0x0000501501b8 (NAME='test/tp', +- DB_TRX_ID=0x000000000306, ++header=0x00005015008d (NAME='test/tp', ++ DB_TRX_ID=0x000000000307, + DB_ROLL_PTR=0x86000001390110, +- ID=0x0000000000000013, ++ ID=0x0000000000000014, + N_COLS=0x80000001, + TYPE=0x000009a1, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000005) +-header=0x000030150284 (NAME='test/tr', +- DB_TRX_ID=0x000000000302, ++header=0x000010150291 (NAME='test/tr', ++ DB_TRX_ID=0x000000000303, + DB_ROLL_PTR=0x82000001350110, +- ID=0x000000000000000f, ++ ID=0x0000000000000010, + N_COLS=0x00000001, + TYPE=0x00000001, + MIX_ID=0x0000000000000000, +@@ -90,11 +92,11 @@ + CLUSTER_NAME=NULL(0 bytes), + SPACE=0x00000001) + header=0x000048150074 (NAME='test/tz', +- DB_TRX_ID=0x000000000305, ++ DB_TRX_ID=0x000000000306, + DB_ROLL_PTR=0x85000001380110, +- ID=0x0000000000000012, ++ ID=0x0000000000000013, + N_COLS=0x80000001, +- TYPE=0x00000023, ++ TYPE=0x00000021, + MIX_ID=0x0000000000000000, + MIX_LEN=0x00000050, + CLUSTER_NAME=NULL(0 bytes), diff --git a/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff index 14fa7509c19..0ce96c8eb10 100644 --- a/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff +++ b/mysql-test/suite/innodb/r/table_flags,8k,debug.rdiff @@ -1,7 +1,7 @@ --- suite/innodb/r/table_flags.result +++ suite/innodb/r/table_flags,8k,debug.reject -@@ -10,81 +10,81 @@ - CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +@@ -8,81 +8,81 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; SYS_TABLES clustered index root page (8): N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 -header=0x01000003016e (NAME=0x696e66696d756d00) @@ -97,7 +97,7 @@ + DB_ROLL_PTR=0x86000001df0110, + ID=0x0000000000000014, N_COLS=0x80000001, - TYPE=0x00000021, + TYPE=0x000009a1, MIX_ID=0x0000000000000000, MIX_LEN=0x00000050, CLUSTER_NAME=NULL(0 bytes), @@ -113,7 +113,7 @@ N_COLS=0x00000001, TYPE=0x00000001, MIX_ID=0x0000000000000000, -@@ -92,9 +92,9 @@ +@@ -90,9 +90,9 @@ CLUSTER_NAME=NULL(0 bytes), SPACE=0x00000001) header=0x000048150074 (NAME='test/tz', diff --git a/mysql-test/suite/innodb/r/table_flags,debug.rdiff b/mysql-test/suite/innodb/r/table_flags,debug.rdiff index d038515ffe5..acc796a0a7f 100644 --- a/mysql-test/suite/innodb/r/table_flags,debug.rdiff +++ b/mysql-test/suite/innodb/r/table_flags,debug.rdiff @@ -1,7 +1,7 @@ --- suite/innodb/r/table_flags.result +++ suite/innodb/r/table_flags,debug.reject -@@ -10,81 +10,81 @@ - CREATE TABLE tp(a INT PRIMARY KEY)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +@@ -8,81 +8,81 @@ + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9; SYS_TABLES clustered index root page (8): N_RECS=9; LEVEL=0; INDEX_ID=0x0000000000000001 -header=0x01000003016e (NAME=0x696e66696d756d00) @@ -97,7 +97,7 @@ + DB_ROLL_PTR=0x860000013a0110, + ID=0x0000000000000014, N_COLS=0x80000001, - TYPE=0x00000021, + TYPE=0x000009a1, MIX_ID=0x0000000000000000, MIX_LEN=0x00000050, CLUSTER_NAME=NULL(0 bytes), @@ -113,7 +113,7 @@ N_COLS=0x00000001, TYPE=0x00000001, MIX_ID=0x0000000000000000, -@@ -92,9 +92,9 @@ +@@ -90,9 +90,9 @@ CLUSTER_NAME=NULL(0 bytes), SPACE=0x00000001) header=0x000048150074 (NAME='test/tz', diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 7a737cd10e8..bb815de3138 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -4049,7 +4049,9 @@ row_drop_table_for_mysql( dict_stats_recalc_pool_del(table); dict_stats_defrag_pool_del(table, NULL); - btr_defragment_remove_table(table); + if (btr_defragment_thread_active) { + btr_defragment_remove_table(table); + } /* Remove stats for this table and all of its indexes from the persistent storage if it exists and if there are stats for this diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 450813bef6d..88ebe24f3bb 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4059,7 +4059,9 @@ row_drop_table_for_mysql( dict_stats_recalc_pool_del(table); dict_stats_defrag_pool_del(table, NULL); - btr_defragment_remove_table(table); + if (btr_defragment_thread_active) { + btr_defragment_remove_table(table); + } /* Remove stats for this table and all of its indexes from the persistent storage if it exists and if there are stats for this From 75ceb6ff130cdac67fadc1d45cd64578a19b4835 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 31 Oct 2018 14:25:26 +0400 Subject: [PATCH 099/127] MDEV-17298 ASAN unknown-crash / READ of size 1 in my_strntoul_8bit upon INSERT .. SELECT --- mysql-test/r/ctype_latin1.result | 9 +++++++++ mysql-test/t/ctype_latin1.test | 12 ++++++++++++ strings/ctype-simple.c | 4 ++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/ctype_latin1.result b/mysql-test/r/ctype_latin1.result index 8913c8082c8..d2e4cc31097 100644 --- a/mysql-test/r/ctype_latin1.result +++ b/mysql-test/r/ctype_latin1.result @@ -7963,5 +7963,14 @@ ABCDEFGHI-ABCDEFGHI DROP TABLE t1; SET optimizer_switch=@save_optimizer_switch; # +# MDEV-17298 ASAN unknown-crash / READ of size 1 in my_strntoul_8bit upon INSERT .. SELECT +# +SET NAMES latin1; +CREATE TABLE t1 (a CHAR); +CREATE TABLE t2 (b ENUM('foo','bar')); +INSERT INTO t1 VALUES ('1'); +INSERT INTO t2 SELECT * FROM t1; +DROP TABLE t1, t2; +# # End of 10.0 tests # diff --git a/mysql-test/t/ctype_latin1.test b/mysql-test/t/ctype_latin1.test index da4d76ee2cf..0ec983b8b74 100644 --- a/mysql-test/t/ctype_latin1.test +++ b/mysql-test/t/ctype_latin1.test @@ -264,6 +264,18 @@ SET NAMES latin1; --source include/ctype_mdev13118.inc +--echo # +--echo # MDEV-17298 ASAN unknown-crash / READ of size 1 in my_strntoul_8bit upon INSERT .. SELECT +--echo # + +SET NAMES latin1; +CREATE TABLE t1 (a CHAR); +CREATE TABLE t2 (b ENUM('foo','bar')); +INSERT INTO t1 VALUES ('1'); +INSERT INTO t2 SELECT * FROM t1; +DROP TABLE t1, t2; + + --echo # --echo # End of 10.0 tests --echo # diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c index 61b14b84820..8d12c05db18 100644 --- a/strings/ctype-simple.c +++ b/strings/ctype-simple.c @@ -454,7 +454,6 @@ ulong my_strntoul_8bit(CHARSET_INFO *cs, register uint cutlim; register uint32 i; register const char *s; - register uchar c; const char *save, *e; int overflow; @@ -489,8 +488,9 @@ ulong my_strntoul_8bit(CHARSET_INFO *cs, overflow = 0; i = 0; - for (c = *s; s != e; c = *++s) + for ( ; s != e; ++s) { + register uchar c= *s; if (c>='0' && c<='9') c -= '0'; else if (c>='A' && c<='Z') From f8268f3cce4577c28ab62e53293556d05a74fb1a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 31 Oct 2018 17:19:25 +0100 Subject: [PATCH 100/127] MDEV-17195 Speedup lock-ddl-per-table, if there is a large number of tables Query INFORMATION_SCHEMA.INNODB_SYS_TABLES only once, and cache results. As a small cleanup, remove mdl_lock_con_mutex, the MDL handling connection is never accessed by multiple threads at the same time. --- extra/mariabackup/backup_mysql.cc | 59 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index e542eb7e5e7..94b3baf5c3d 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1758,62 +1758,63 @@ backup_cleanup() } -static pthread_mutex_t mdl_lock_con_mutex; static MYSQL *mdl_con = NULL; +std::map spaceid_to_tablename; + void mdl_lock_init() { - pthread_mutex_init(&mdl_lock_con_mutex, NULL); mdl_con = xb_mysql_connect(); - if (mdl_con) + if (!mdl_con) { - xb_mysql_query(mdl_con, "BEGIN", false, true); + msg("FATAL: cannot create connection for MDL locks"); + exit(1); } + const char *query = + "SELECT NAME, SPACE FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE '%%/%%'"; + + MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, query, true, true); + while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) { + int err; + ulint id = (ulint)my_strtoll10(row[1], 0, &err); + spaceid_to_tablename[id] = ut_get_name(0, row[0]); + } + mysql_free_result(mysql_result); + + xb_mysql_query(mdl_con, "BEGIN", false, true); } void mdl_lock_table(ulint space_id) { - std::ostringstream oss; - oss << "SELECT NAME " - "FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES " - "WHERE SPACE = " << space_id << " AND NAME LIKE '%%/%%'"; + if (space_id == 0) + return; - pthread_mutex_lock(&mdl_lock_con_mutex); + std::string full_table_name = spaceid_to_tablename[space_id]; - MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, oss.str().c_str(), true, true); + DBUG_EXECUTE_IF("rename_during_mdl_lock_table", + if (full_table_name == "`test`.`t1`") + xb_mysql_query(mysql_connection, "RENAME TABLE test.t1 to test.t2", false, true); + ); - while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) { - - DBUG_EXECUTE_IF("rename_during_mdl_lock_table", - if (strcmp(row[0], "test/t1") == 0) - xb_mysql_query(mysql_connection, "RENAME TABLE test.t1 to test.t2", false, true - );); - - std::string full_table_name = ut_get_name(0,row[0]); - std::ostringstream lock_query; - lock_query << "SELECT 1 FROM " << full_table_name << " LIMIT 0"; - msg_ts("Locking MDL for %s\n", full_table_name.c_str()); - if (mysql_query(mdl_con, lock_query.str().c_str())) { + std::ostringstream lock_query; + lock_query << "SELECT 1 FROM " << full_table_name << " LIMIT 0"; + msg_ts("Locking MDL for %s\n", full_table_name.c_str()); + if (mysql_query(mdl_con, lock_query.str().c_str())) { msg_ts("Warning : locking MDL failed for space id %zu, name %s\n", space_id, full_table_name.c_str()); - } else { + } else { MYSQL_RES *r = mysql_store_result(mdl_con); mysql_free_result(r); - } } - - pthread_mutex_unlock(&mdl_lock_con_mutex); - mysql_free_result(mysql_result); } - void mdl_unlock_all() { msg_ts("Unlocking MDL for all tables\n"); xb_mysql_query(mdl_con, "COMMIT", false, true); mysql_close(mdl_con); - pthread_mutex_destroy(&mdl_lock_con_mutex); + spaceid_to_tablename.clear(); } From bf28ba67b6936cebc18e21ec386faf40ec2e4571 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 31 Oct 2018 22:06:15 +0100 Subject: [PATCH 101/127] update rdiffs for 32bit --- .../r/sysvars_innodb,32bit,xtradb.rdiff | 10 +- .../sys_vars/r/sysvars_innodb,32bit.rdiff | 96 ++++++++++--------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff index 746d9fffd0e..73dd57ed1fb 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit,xtradb.rdiff @@ -475,7 +475,7 @@ VARIABLE_COMMENT InnoDB Fulltext search number of words to optimize for each optimize table call NUMERIC_MIN_VALUE 1000 NUMERIC_MAX_VALUE 10000 -@@ -1173,7 +1355,7 @@ +@@ -1173,10 +1355,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000000000 VARIABLE_SCOPE GLOBAL @@ -483,7 +483,11 @@ +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT InnoDB Fulltext search query result cache limit in bytes NUMERIC_MIN_VALUE 1000000 - NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 18446744073709551615 ++NUMERIC_MAX_VALUE 4294967295 + NUMERIC_BLOCK_SIZE 0 + ENUM_VALUE_LIST NULL + READ_ONLY NO @@ -1201,7 +1383,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2 @@ -1211,7 +1215,7 @@ VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL -GLOBAL_VALUE 5.6.42 -+GLOBAL_VALUE 5.6.41-83.1 ++GLOBAL_VALUE 5.6.41-84.1 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff index 46148b1d6ba..55101ef8ef8 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff @@ -125,7 +125,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -767,7 +767,7 @@ +@@ -781,7 +781,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 120 VARIABLE_SCOPE GLOBAL @@ -134,7 +134,7 @@ VARIABLE_COMMENT Number of pages reserved in doublewrite buffer for batch flushing NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 127 -@@ -851,7 +851,7 @@ +@@ -865,7 +865,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -143,7 +143,7 @@ VARIABLE_COMMENT Speeds up the shutdown process of the InnoDB storage engine. Possible values are 0, 1 (faster) or 2 (fastest - crash-like). NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -865,7 +865,7 @@ +@@ -879,7 +879,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 600 VARIABLE_SCOPE GLOBAL @@ -152,7 +152,7 @@ VARIABLE_COMMENT Maximum number of seconds that semaphore times out in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -935,7 +935,7 @@ +@@ -949,7 +949,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -161,7 +161,7 @@ VARIABLE_COMMENT Make the first page of the given tablespace dirty. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -949,7 +949,7 @@ +@@ -963,7 +963,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -170,7 +170,7 @@ VARIABLE_COMMENT Number of iterations over which the background flushing is averaged. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1000 -@@ -977,7 +977,7 @@ +@@ -991,7 +991,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -179,7 +179,7 @@ VARIABLE_COMMENT Controls the durability/speed trade-off for commits. Set to 0 (write and flush redo log to disk only once per second), 1 (flush to disk at each commit), 2 (write to log at commit but flush to disk only once per second) or 3 (flush to disk at prepare and at commit, slower and usually redundant). 1 and 3 guarantees that after a crash, committed transactions will not be lost and will be consistent with the binlog and other transactional engines. 2 can get inconsistent and lose transactions if there is a power failure or kernel crash but not if mysqld crashes. 0 has no guarantees in case of crash. 0 and 2 can be faster than 1 or 3. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3 -@@ -1005,7 +1005,7 @@ +@@ -1019,7 +1019,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -188,7 +188,7 @@ VARIABLE_COMMENT Set to 0 (don't flush neighbors from buffer pool), 1 (flush contiguous neighbors from buffer pool) or 2 (flush neighbors from buffer pool), when flushing a block NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -1047,7 +1047,7 @@ +@@ -1061,7 +1061,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -197,7 +197,7 @@ VARIABLE_COMMENT Helps to save your data in case the disk image of the database becomes corrupt. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6 -@@ -1075,7 +1075,7 @@ +@@ -1089,7 +1089,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8000000 VARIABLE_SCOPE GLOBAL @@ -206,7 +206,7 @@ VARIABLE_COMMENT InnoDB Fulltext search cache size in bytes NUMERIC_MIN_VALUE 1600000 NUMERIC_MAX_VALUE 80000000 -@@ -1117,7 +1117,7 @@ +@@ -1131,7 +1131,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 84 VARIABLE_SCOPE GLOBAL @@ -215,7 +215,7 @@ VARIABLE_COMMENT InnoDB Fulltext search maximum token size in characters NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 84 -@@ -1131,7 +1131,7 @@ +@@ -1145,7 +1145,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 3 VARIABLE_SCOPE GLOBAL @@ -224,7 +224,7 @@ VARIABLE_COMMENT InnoDB Fulltext search minimum token size in characters NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16 -@@ -1145,7 +1145,7 @@ +@@ -1159,7 +1159,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000 VARIABLE_SCOPE GLOBAL @@ -233,7 +233,7 @@ VARIABLE_COMMENT InnoDB Fulltext search number of words to optimize for each optimize table call NUMERIC_MIN_VALUE 1000 NUMERIC_MAX_VALUE 10000 -@@ -1159,7 +1159,7 @@ +@@ -1173,10 +1173,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000000000 VARIABLE_SCOPE GLOBAL @@ -241,8 +241,12 @@ +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT InnoDB Fulltext search query result cache limit in bytes NUMERIC_MIN_VALUE 1000000 - NUMERIC_MAX_VALUE 4294967295 -@@ -1187,7 +1187,7 @@ +-NUMERIC_MAX_VALUE 18446744073709551615 ++NUMERIC_MAX_VALUE 4294967295 + NUMERIC_BLOCK_SIZE 0 + ENUM_VALUE_LIST NULL + READ_ONLY NO +@@ -1201,7 +1201,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -251,7 +255,7 @@ VARIABLE_COMMENT InnoDB Fulltext search parallel sort degree, will round up to nearest power of 2 number NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 16 -@@ -1201,7 +1201,7 @@ +@@ -1215,7 +1215,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 640000000 VARIABLE_SCOPE GLOBAL @@ -260,7 +264,7 @@ VARIABLE_COMMENT Total memory allocated for InnoDB Fulltext Search cache NUMERIC_MIN_VALUE 32000000 NUMERIC_MAX_VALUE 1600000000 -@@ -1229,7 +1229,7 @@ +@@ -1243,7 +1243,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 100 VARIABLE_SCOPE GLOBAL @@ -269,7 +273,7 @@ VARIABLE_COMMENT Up to what percentage of dirty pages should be flushed when innodb finds it has spare resources to do so. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -1271,10 +1271,10 @@ +@@ -1285,10 +1285,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 200 VARIABLE_SCOPE GLOBAL @@ -282,7 +286,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1283,12 +1283,12 @@ +@@ -1297,12 +1297,12 @@ SESSION_VALUE NULL GLOBAL_VALUE 2000 GLOBAL_VALUE_ORIGIN COMPILE-TIME @@ -298,7 +302,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1355,7 +1355,7 @@ +@@ -1369,7 +1369,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 50 VARIABLE_SCOPE SESSION @@ -307,7 +311,7 @@ VARIABLE_COMMENT Timeout in seconds an InnoDB transaction may wait for a lock before being rolled back. Values above 100000000 disable the timeout. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1073741824 -@@ -1369,10 +1369,10 @@ +@@ -1383,10 +1383,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 16777216 VARIABLE_SCOPE GLOBAL @@ -320,7 +324,7 @@ NUMERIC_BLOCK_SIZE 1024 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1411,7 +1411,7 @@ +@@ -1425,7 +1425,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -329,7 +333,7 @@ VARIABLE_COMMENT Number of log files in the log group. InnoDB writes to the files in a circular fashion. NUMERIC_MIN_VALUE 2 NUMERIC_MAX_VALUE 100 -@@ -1453,10 +1453,10 @@ +@@ -1467,10 +1467,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 1024 VARIABLE_SCOPE GLOBAL @@ -342,7 +346,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1495,10 +1495,10 @@ +@@ -1509,10 +1509,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -355,7 +359,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1509,7 +1509,7 @@ +@@ -1523,7 +1523,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -364,7 +368,7 @@ VARIABLE_COMMENT Maximum delay of user threads in micro-seconds NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 10000000 -@@ -1523,7 +1523,7 @@ +@@ -1537,7 +1537,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -373,7 +377,7 @@ VARIABLE_COMMENT Number of identical copies of log groups we keep for the database. Currently this should be set to 1. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 10 -@@ -1593,7 +1593,7 @@ +@@ -1607,7 +1607,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL @@ -382,7 +386,7 @@ VARIABLE_COMMENT Number of multi-threaded flush threads NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1649,10 +1649,10 @@ +@@ -1663,10 +1663,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -395,7 +399,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1677,7 +1677,7 @@ +@@ -1691,7 +1691,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16 VARIABLE_SCOPE GLOBAL @@ -404,7 +408,7 @@ VARIABLE_COMMENT Number of rw_locks protecting buffer pool page_hash. Rounded up to the next power of 2 NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -1691,7 +1691,7 @@ +@@ -1705,7 +1705,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16384 VARIABLE_SCOPE GLOBAL @@ -413,7 +417,7 @@ VARIABLE_COMMENT Page size to use for all InnoDB tablespaces. NUMERIC_MIN_VALUE 4096 NUMERIC_MAX_VALUE 65536 -@@ -1733,7 +1733,7 @@ +@@ -1747,7 +1747,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 300 VARIABLE_SCOPE GLOBAL @@ -422,7 +426,7 @@ VARIABLE_COMMENT Number of UNDO log pages to purge in one batch from the history list. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 5000 -@@ -1775,7 +1775,7 @@ +@@ -1789,7 +1789,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -431,7 +435,7 @@ VARIABLE_COMMENT Purge threads can be from 1 to 32. Default is 1. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 32 -@@ -1803,7 +1803,7 @@ +@@ -1817,7 +1817,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 56 VARIABLE_SCOPE GLOBAL @@ -440,7 +444,7 @@ VARIABLE_COMMENT Number of pages that must be accessed sequentially for InnoDB to trigger a readahead. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 64 -@@ -1817,7 +1817,7 @@ +@@ -1831,7 +1831,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -449,7 +453,7 @@ VARIABLE_COMMENT Number of background read I/O threads in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1845,10 +1845,10 @@ +@@ -1859,10 +1859,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -462,7 +466,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1873,7 +1873,7 @@ +@@ -1887,7 +1887,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -471,7 +475,7 @@ VARIABLE_COMMENT Number of undo logs to use (deprecated). NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -1887,7 +1887,7 @@ +@@ -1901,7 +1901,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -480,7 +484,7 @@ VARIABLE_COMMENT An InnoDB page number. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -1943,7 +1943,7 @@ +@@ -1957,7 +1957,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1048576 VARIABLE_SCOPE GLOBAL @@ -489,7 +493,7 @@ VARIABLE_COMMENT Memory buffer size for index creation NUMERIC_MIN_VALUE 65536 NUMERIC_MAX_VALUE 67108864 -@@ -1957,10 +1957,10 @@ +@@ -1971,10 +1971,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 6 VARIABLE_SCOPE GLOBAL @@ -502,7 +506,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2167,7 +2167,7 @@ +@@ -2181,7 +2181,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -511,7 +515,7 @@ VARIABLE_COMMENT Size of the mutex/lock wait array. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -2181,10 +2181,10 @@ +@@ -2195,10 +2195,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -524,7 +528,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2209,7 +2209,7 @@ +@@ -2223,7 +2223,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -533,7 +537,7 @@ VARIABLE_COMMENT Helps in performance tuning in heavily concurrent environments. Sets the maximum number of threads allowed inside InnoDB. Value 0 will disable the thread throttling. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -2223,7 +2223,7 @@ +@@ -2237,7 +2237,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 10000 VARIABLE_SCOPE GLOBAL @@ -542,7 +546,7 @@ VARIABLE_COMMENT Time of innodb thread sleeping before joining InnoDB queue (usec). Value 0 disable a sleep NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000000 -@@ -2293,7 +2293,7 @@ +@@ -2307,7 +2307,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -551,7 +555,7 @@ VARIABLE_COMMENT Number of undo logs to use. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2307,7 +2307,7 @@ +@@ -2321,7 +2321,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -560,7 +564,7 @@ VARIABLE_COMMENT Number of undo tablespaces to use. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 126 -@@ -2405,7 +2405,7 @@ +@@ -2419,7 +2419,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL From 8d834cd0f370b306f63c2364552d187fc388e59e Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 31 Oct 2018 23:48:29 +0200 Subject: [PATCH 102/127] Updated list of unstable tests for 10.1.37 release --- mysql-test/unstable-tests | 95 +++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 50 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index b52f45815fa..d26ce111375 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,51 +23,38 @@ # ############################################################################## -# Based on bb-10.1-release f95d26b4d3ec0e4b039e80439f85e4f22bd79fe7 +# Based on 10.1 a5cbdd63bc2ff25a52e2e10f84b6aaf59837dbae +main.alter_table : Modified in 10.1.37 main.alter_table_trans : MDEV-12084 - timeout main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result main.auth_named_pipe : MDEV-14724 - System error 2 -main.auto_increment : Modified in 10.1.35 -main.bootstrap : Modified in 10.1.35 main.count_distinct2 : MDEV-11768 - timeout main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_binary : Modified in 10.1.35 -main.ctype_eucjpms : Modified in 10.1.35 -main.ctype_euckr : Modified in 10.1.35 -main.ctype_gbk : Modified in 10.1.35 -main.ctype_latin1 : Modified in 10.1.35 -main.ctype_ucs : Modified in 10.1.35 -main.ctype_ujis : Modified in 10.1.35 -main.ctype_utf16le : Modified in 10.1.35 -main.ctype_utf16 : MDEV-10675: timeout or extra warnings; modified in 10.1.35 -main.ctype_utf32 : Modified in 10.1.35 -main.ctype_utf8 : Modified in 10.1.35 -main.ctype_utf8mb4 : Modified in 10.1.35 +main.create_or_replace : Modified in 10.1.37 +main.ctype_uca : Modified in 10.1.37 +main.ctype_utf16 : MDEV-10675: timeout or extra warnings main.debug_sync : MDEV-10607 - internal error -main.derived : Modified in 10.1.35 -main.derived_opt : MDEV-11768 - timeout +main.derived_opt : MDEV-11768 - timeout; modified in 10.1.37 main.events_2 : MDEV-13277 - Server crash main.events_bugs : MDEV-12892 - Crash in fill_schema_processlist main.events_slowlog : MDEV-12821 - Wrong result main.events_restart : MDEV-12236 - Server shutdown problem main.flush : Modified in 10.1.36 +main.func_concat : Modified in 10.1.37 main.func_isnull : Modified in 10.1.36 -main.func_time : Modified in 10.1.36 +main.func_time : Modified in 10.1.37 main.gis : MDEV-13411 - wrong result on P8; modified in 10.1.36 -main.grant : Modified in 10.1.35 +main.grant : Modified in 10.1.37 main.group_min_max : Modified in 10.1.36 -main.having : Modified in 10.1.35 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - Wrong execution plan, timeout with valgrind main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure main.join : Modified in 10.1.36 -main.join_cache : Modified in 10.1.35 -main.join_outer : Modified in 10.1.35 main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist main.log_tables-big : MDEV-13408 - wrong result -main.max_statement_time : Modified in 10.1.35 +main.lowercase_fs_off : Modified in 10.1.37 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" @@ -79,33 +66,32 @@ main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqlslap : MDEV-11801 - timeout main.mysqltest : MDEV-9269 - fails on Alpha main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan +main.order_by_zerolength-4285 : Modified in 10.1.37 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock +main.partition_explicit_prune : Modified in 10.1.37 main.partition_innodb_plugin : MDEV-12901 - Valgrind warnings main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.query_cache : MDEV-12895 - Wrong result main.query_cache_debug : MDEV-15281 - Resize or similar command in progress main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away -main.rename : Modified in 10.1.35 main.selectivity : Modified in 10.1.36 main.set_statement : MDEV-13183 - Wrong result main.show_explain : MDEV-10674 - sporadic failure main.sp : Modified in 10.1.36 main.sp_notembedded : MDEV-10607 - internal error -main.sp-security : MDEV-10607 - sporadic "can't connect" -main.stat_tables : Modified in 10.1.36 +main.sp-security : MDEV-10607 - sporadic "can't connect"; modified in 10.1.37 +main.stat_tables : Modified in 10.1.37 main.stat_tables_par_innodb : MDEV-14155 - wrong rounding -main.statistics : Modified in 10.1.35 main.status : MDEV-8510 - sporadic wrong result -main.subselect : Modified in 10.1.35 main.subselect_extra_no_semijoin : Modified in 10.1.36 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_sj_mat : Modified in 10.1.35 -main.subselect_sj2_mat : Modified in 10.1.35 main.tc_heuristic_recover : MDEV-15200 - wrong error on mysqld_stub_cmd main.type_blob : MDEV-15195 - Wrong result -main.type_datetime : MDEV-14322 - wrong result +main.type_datetime : Modified in 10.1.37 main.type_datetime_hires : MDEV-10687 - timeout -main.union : Modified in 10.1.35 +main.type_float : Modified in 10.1.37 +main.type_newdecimal : Modified in 10.1.37 +main.type_year : Modified in 10.1.37 main.xa : MDEV-11769 - lock wait timeout #---------------------------------------------------------------- @@ -119,11 +105,14 @@ archive-test_sql_discovery.discover : MDEV-16817 - Table marked as crashed #---------------------------------------------------------------- +auth_gssapi.basic : Modified in 10.1.37 + +#---------------------------------------------------------------- + binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed -binlog.binlog_flush_binlogs_delete_domain : MDEV-14431 - Wrong error code +binlog.binlog_flush_binlogs_delete_domain : Modified in 10.1.37 binlog.binlog_incident : Modified in 10.1.36 binlog.binlog_killed : MDEV-12925 - Wrong result -binlog.binlog_tmp_table_row : Added in 10.1.35 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- @@ -141,6 +130,7 @@ binlog.load_data_stm_view : MDEV-16948 - Wrong result #---------------------------------------------------------------- connect.pivot : MDEV-14803 - failed to discover table +connect.xml2 : Modified in 10.1.37 connect.zip : MDEV-13884 - Wrong result #---------------------------------------------------------------- @@ -196,18 +186,19 @@ galera_3nodes.* : The suite has not been stabilized yet #---------------------------------------------------------------- -heap.heap_auto_increment : Modified in 10.1.35 +heap.heap_btree : Modified in 10.1.37 #---------------------------------------------------------------- +innodb.alter_inplace_perfschema : Added in 10.1.37 innodb.binlog_consistent : MDEV-10618 - Server fails to start innodb.doublewrite : MDEV-12905 - Lost connection to MySQL server innodb.foreign-keys : Modified in 10.1.36 -innodb.foreign_key : Added in 10.1.36 +innodb.foreign_key : Modified in 10.1.37 innodb.group_commit_crash : MDEV-11770 - checksum mismatch innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup -innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS +innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS; modified in 10.1.37 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout innodb.innodb-blob : MDEV-12053 - Client crash innodb.innodb_bug14147491 : MDEV-11808 - wrong error codes @@ -222,26 +213,25 @@ innodb.innodb-page_compression_lzma : MDEV-14353 - wrong result on Fedora 25 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result -innodb.innodb-wl5522 : Modified in 10.1.35 innodb.innodb_zip_innochecksum2 : MDEV-13882 - Warning: difficult to find free blocks innodb.log_file_size : MDEV-15668 - Not found pattern innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace innodb.table_definition_cache_debug : MDEV-14206 - Extra warning -innodb.table_flags : MDEV-14363 - Operating system error number 2 +innodb.table_flags : Modified in 10.1.37 innodb.xa_recovery : MDEV-15279 - mysqld got exception #---------------------------------------------------------------- -innodb_fts.fts_kill_query : Added in 10.1.35 +innodb_fts.fts_kill_query : Modified in 10.0.37 innodb_fts.innodb-fts-fic : MDEV-14154 - Assertion failure innodb_fts.innodb_fts_misc_debug : MDEV-14156 - Unexpected warning -innodb_fts.sync_ddl : Added in 10.1.35 #---------------------------------------------------------------- maria.concurrent : Added in 10.1.36 maria.create : Added in 10.1.36 +maria.fulltext2 : Added in 10.1.37 maria.insert_select : MDEV-12757 - Timeout maria.insert_select-7314 : MDEV-16492 - Timeout maria.maria : MDEV-14430 - Wrong result; modified in 10.1.36 @@ -251,6 +241,7 @@ maria.maria : MDEV-14430 - Wrong result; modified in 1 mariabackup.incremental_encrypted : MDEV-15667 - Timeout mariabackup.mdev-14447 : MDEV-15201 - Timeout mariabackup.xb_compressed_encrypted : MDEV-14812 - Segfault +mariabackup.xb_partition : MDEV-17584 - Crash on shutdown #---------------------------------------------------------------- @@ -276,16 +267,16 @@ multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_h #---------------------------------------------------------------- -parts.alter_data_directory_innodb : Added in 10.1.35 parts.partition_alter2_2_maria : MDEV-14364 - Lost connection to MySQL server during query parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed parts.partition_auto_increment_maria : MDEV-14430 - Wrong result parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.partition_innodb_status_file : MDEV-12901 - Valgrind -parts.truncate_locked : Added in 10.1.35 +parts.update_and_cache : Added in 10.1.37 #---------------------------------------------------------------- +perfschema.connect_attrs : MDEV-17283 - Wrong result perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown @@ -302,7 +293,6 @@ perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.processlist : Added in 10.1.35 plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc @@ -313,6 +303,7 @@ roles.create_and_grant_role : MDEV-11772 - wrong result #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rpl_15919 : Added in 10.1.37 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -337,6 +328,7 @@ rpl.rpl_insert_id : MDEV-15197 - Wrong result rpl.rpl_insert_id_pk : MDEV-16567 - Assertion failure rpl.rpl_insert_ignore : MDEV-14365 - Lost connection to MySQL server during query rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips +rpl.rpl_lcase_tblnames_rewrite_db : Added in 10.1.37 rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog rpl.rpl_mdev6020 : MDEV-10417 - Fails on Mips rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed @@ -346,7 +338,8 @@ rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Sync slave with master fail rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_mdev6589 : MDEV-12979 - Assertion failure rpl.rpl_parallel_multilevel2 : MDEV-14723 - Timeout -rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.35 +rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.37 +rpl.rpl_parallel_optimistic_nobinlog : Config file modified in 10.1.37 rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips @@ -358,6 +351,7 @@ rpl.rpl_row_img_blobs : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_min : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_noblob : MDEV-13875 - command "diff_files" failed rpl.rpl_row_index_choice : MDEV-15196 - Slave crash +rpl.rpl_row_lcase_tblnames : Added in 10.1.37 rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_row_spatial : Added in 10.1.36 rpl.rpl_semi_sync : MDEV-11220 - Wrong result @@ -370,6 +364,7 @@ rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock rpl.rpl_start_stop_slave : MDEV-13567 - Replication failure +rpl.rpl_stm_lcase_tblnames : Added in 10.1.37 rpl.rpl_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_stm_relay_ign_space : MDEV-14360 - Test assertion rpl.rpl_sync : MDEV-10633 - Database page corruption @@ -408,6 +403,8 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x +sys_vars.innodb_ft_result_cache_limit_32 : Added in 10.1.37 +sys_vars.innodb_ft_result_cache_limit_64 : Added in 10.1.37 sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout sys_vars.log_slow_admin_statements_func : MDEV-12235 - Server crash sys_vars.rpl_init_slave_func : MDEV-10149 - wrong results @@ -455,23 +452,21 @@ rpl-tokudb.* : MDEV-14354 - Tests fail with tcmalloc #---------------------------------------------------------------- -unit.* : suite.pm modified in 10.1.35 - unit.lf : MDEV-12897 - Signal 11 thrown unit.ma_test_loghandler : MDEV-10638 - record read not ok +unit.mf_iocache : MDEV-17586 - total matches eof, cache is read up to eof unit.my_atomic : MDEV-15670 - Signal 11 thrown #---------------------------------------------------------------- vcol.not_supported : MDEV-10639 - Testcase timeout vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout -vcol.vcol_misc : MDEV-16651 - Wrong error message; modified in 10.1.35 +vcol.vcol_misc : MDEV-16651 - Wrong error message #---------------------------------------------------------------- wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Failure upon check-testcase -wsrep.pool_of_threads : MDEV-12234 - Library problem on Power -wsrep.variables : Modified in 10.1.36 +wsrep.variables : MDEV-17585 - Deadlock; modified in 10.1.37 wsrep_info.plugin : MDEV-12909 - Wrong result From b2f39a5f567d006000aad8b431267298cf81b569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 1 Nov 2018 09:15:41 +0200 Subject: [PATCH 103/127] Add missing wsrep.cnf.sh --- support-files/CMakeLists.txt | 7 ++ support-files/wsrep.cnf.sh | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 support-files/wsrep.cnf.sh diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index cc9eed6c7be..d02da51194c 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -50,6 +50,13 @@ FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small) ENDFOREACH() ENDIF() +IF(WITH_WSREP) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/wsrep.cnf.sh + ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} @ONLY) + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} + DESTINATION ${inst_location} COMPONENT IniFiles) +ENDIF() + IF(UNIX) SET(prefix ${CMAKE_INSTALL_PREFIX}) FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure) diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh new file mode 100644 index 00000000000..51ce3dca2dd --- /dev/null +++ b/support-files/wsrep.cnf.sh @@ -0,0 +1,125 @@ +# This file contains wsrep-related mysqld options. It should be included +# in the main MySQL configuration file. +# +# Options that need to be customized: +# - wsrep_provider +# - wsrep_cluster_address +# - wsrep_sst_auth +# The rest of defaults should work out of the box. + +## +## mysqld options _MANDATORY_ for correct opration of the cluster +## +[mysqld] + +# (This must be substituted by wsrep_format) +binlog_format=ROW + +# Currently only InnoDB storage engine is supported +default-storage-engine=innodb + +# to avoid issues with 'bulk mode inserts' using autoinc +innodb_autoinc_lock_mode=2 + +# Override bind-address +# In some systems bind-address defaults to 127.0.0.1, and with mysqldump SST +# it will have (most likely) disastrous consequences on donor node +bind-address=0.0.0.0 + +## +## WSREP options +## + +# Enable wsrep +wsrep_on=1 + +# Full path to wsrep provider library or 'none' +wsrep_provider=none + +# Provider specific configuration options +#wsrep_provider_options= + +# Logical cluster name. Should be the same for all nodes. +wsrep_cluster_name="my_wsrep_cluster" + +# Group communication system handle +#wsrep_cluster_address="dummy://" + +# Human-readable node name (non-unique). Hostname by default. +#wsrep_node_name= + +# Base replication [:port] of the node. +# The values supplied will be used as defaults for state transfer receiving, +# listening ports and so on. Default: address of the first network interface. +#wsrep_node_address= + +# Address for incoming client connections. Autodetect by default. +#wsrep_node_incoming_address= + +# How many threads will process writesets from other nodes +wsrep_slave_threads=1 + +# DBUG options for wsrep provider +#wsrep_dbug_option + +# Generate fake primary keys for non-PK tables (required for multi-master +# and parallel applying operation) +wsrep_certify_nonPK=1 + +# Maximum number of rows in write set +wsrep_max_ws_rows=131072 + +# Maximum size of write set +wsrep_max_ws_size=1073741824 + +# to enable debug level logging, set this to 1 +wsrep_debug=0 + +# convert locking sessions into transactions +wsrep_convert_LOCK_to_trx=0 + +# how many times to retry deadlocked autocommits +wsrep_retry_autocommit=1 + +# change auto_increment_increment and auto_increment_offset automatically +wsrep_auto_increment_control=1 + +# retry autoinc insert, which failed for duplicate key error +wsrep_drupal_282555_workaround=0 + +# enable "strictly synchronous" semantics for read operations +wsrep_causal_reads=0 + +# Command to call when node status or cluster membership changes. +# Will be passed all or some of the following options: +# --status - new status of this node +# --uuid - UUID of the cluster +# --primary - whether the component is primary or not ("yes"/"no") +# --members - comma-separated list of members +# --index - index of this node in the list +wsrep_notify_cmd= + +## +## WSREP State Transfer options +## + +# State Snapshot Transfer method +wsrep_sst_method=rsync + +# Address which donor should send State Snapshot to. +# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!! +# (SST method dependent. Defaults to the first IP of the first interface) +#wsrep_sst_receive_address= + +# SST authentication string. This will be used to send SST to joining nodes. +# Depends on SST method. For mysqldump method it is root: +wsrep_sst_auth=root: + +# Desired SST donor name. +#wsrep_sst_donor= + +# Reject client queries when donating SST (false) +#wsrep_sst_donor_rejects_queries=0 + +# Protocol version to use +# wsrep_protocol_version= From c4c738e1ef983a0a4353957cf5d488a3f2cb0a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 1 Nov 2018 09:27:59 +0200 Subject: [PATCH 104/127] Revert commit b2f39a5f567d006000aad8b431267298cf81b569 wrong branch. --- support-files/CMakeLists.txt | 7 -- support-files/wsrep.cnf.sh | 125 ----------------------------------- 2 files changed, 132 deletions(-) delete mode 100644 support-files/wsrep.cnf.sh diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index d02da51194c..cc9eed6c7be 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -50,13 +50,6 @@ FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small) ENDFOREACH() ENDIF() -IF(WITH_WSREP) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/wsrep.cnf.sh - ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} @ONLY) - INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} - DESTINATION ${inst_location} COMPONENT IniFiles) -ENDIF() - IF(UNIX) SET(prefix ${CMAKE_INSTALL_PREFIX}) FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure) diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh deleted file mode 100644 index 51ce3dca2dd..00000000000 --- a/support-files/wsrep.cnf.sh +++ /dev/null @@ -1,125 +0,0 @@ -# This file contains wsrep-related mysqld options. It should be included -# in the main MySQL configuration file. -# -# Options that need to be customized: -# - wsrep_provider -# - wsrep_cluster_address -# - wsrep_sst_auth -# The rest of defaults should work out of the box. - -## -## mysqld options _MANDATORY_ for correct opration of the cluster -## -[mysqld] - -# (This must be substituted by wsrep_format) -binlog_format=ROW - -# Currently only InnoDB storage engine is supported -default-storage-engine=innodb - -# to avoid issues with 'bulk mode inserts' using autoinc -innodb_autoinc_lock_mode=2 - -# Override bind-address -# In some systems bind-address defaults to 127.0.0.1, and with mysqldump SST -# it will have (most likely) disastrous consequences on donor node -bind-address=0.0.0.0 - -## -## WSREP options -## - -# Enable wsrep -wsrep_on=1 - -# Full path to wsrep provider library or 'none' -wsrep_provider=none - -# Provider specific configuration options -#wsrep_provider_options= - -# Logical cluster name. Should be the same for all nodes. -wsrep_cluster_name="my_wsrep_cluster" - -# Group communication system handle -#wsrep_cluster_address="dummy://" - -# Human-readable node name (non-unique). Hostname by default. -#wsrep_node_name= - -# Base replication [:port] of the node. -# The values supplied will be used as defaults for state transfer receiving, -# listening ports and so on. Default: address of the first network interface. -#wsrep_node_address= - -# Address for incoming client connections. Autodetect by default. -#wsrep_node_incoming_address= - -# How many threads will process writesets from other nodes -wsrep_slave_threads=1 - -# DBUG options for wsrep provider -#wsrep_dbug_option - -# Generate fake primary keys for non-PK tables (required for multi-master -# and parallel applying operation) -wsrep_certify_nonPK=1 - -# Maximum number of rows in write set -wsrep_max_ws_rows=131072 - -# Maximum size of write set -wsrep_max_ws_size=1073741824 - -# to enable debug level logging, set this to 1 -wsrep_debug=0 - -# convert locking sessions into transactions -wsrep_convert_LOCK_to_trx=0 - -# how many times to retry deadlocked autocommits -wsrep_retry_autocommit=1 - -# change auto_increment_increment and auto_increment_offset automatically -wsrep_auto_increment_control=1 - -# retry autoinc insert, which failed for duplicate key error -wsrep_drupal_282555_workaround=0 - -# enable "strictly synchronous" semantics for read operations -wsrep_causal_reads=0 - -# Command to call when node status or cluster membership changes. -# Will be passed all or some of the following options: -# --status - new status of this node -# --uuid - UUID of the cluster -# --primary - whether the component is primary or not ("yes"/"no") -# --members - comma-separated list of members -# --index - index of this node in the list -wsrep_notify_cmd= - -## -## WSREP State Transfer options -## - -# State Snapshot Transfer method -wsrep_sst_method=rsync - -# Address which donor should send State Snapshot to. -# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!! -# (SST method dependent. Defaults to the first IP of the first interface) -#wsrep_sst_receive_address= - -# SST authentication string. This will be used to send SST to joining nodes. -# Depends on SST method. For mysqldump method it is root: -wsrep_sst_auth=root: - -# Desired SST donor name. -#wsrep_sst_donor= - -# Reject client queries when donating SST (false) -#wsrep_sst_donor_rejects_queries=0 - -# Protocol version to use -# wsrep_protocol_version= From 38b3e52c3c40b35339b1d18c09053be416a93420 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 31 Oct 2018 23:30:34 +0530 Subject: [PATCH 105/127] MDEV-16695: Estimate for rows of derived tables is very high when we are using index_merge union For index merge union[or sort union], the estimates are not taken into account while calculating the selectivity of a condition. So instead of showing the estimates of the index merge union[or sort union], it shows estimates equal to all the records of the table. The fix for the issue is to include the selectivity of index merge union[or sort union] while calculating the selectivity of a condition. --- mysql-test/r/index_merge_myisam.result | 50 ++++++++++++++++++++++++++ mysql-test/t/index_merge_myisam.test | 35 ++++++++++++++++++ sql/opt_range.cc | 6 ++++ 3 files changed, 91 insertions(+) diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result index b3beff5a967..51b764213ca 100644 --- a/mysql-test/r/index_merge_myisam.result +++ b/mysql-test/r/index_merge_myisam.result @@ -1712,3 +1712,53 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL PRIMARY,c1,i,c2 NULL NULL NULL 69 Using where DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +# +# MDEV-16695: Estimate for rows of derived tables is very high when we are using index_merge union +# +create table t0 +( +key1 int not null, +INDEX i1(key1) +); +insert into t0 values (1),(2),(3),(4),(5),(6),(7),(8); +set @d=8; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +insert into t0 select key1+ @d from t0; +set @d=@d*2; +alter table t0 add key2 int not null, add index i2(key2); +alter table t0 add key3 int not null, add index i3(key3); +alter table t0 add key8 int not null, add index i8(key8); +update t0 set key2=key1,key3=key1,key8=1024-key1; +analyze table t0; +Table Op Msg_type Msg_text +test.t0 analyze status OK +set @optimizer_switch_save=@@optimizer_switch; +set optimizer_switch='derived_merge=off,derived_with_keys=off'; +explain select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 2 Using where +2 DERIVED t0 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where +select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +key1 key2 key3 key8 +3 3 3 1021 +set optimizer_use_condition_selectivity=2; +explain select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 2 Using where +2 DERIVED t0 index_merge i1,i2 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where +select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +key1 key2 key3 key8 +3 3 3 1021 +set @@optimizer_switch= @optimizer_switch_save; +drop table t0; diff --git a/mysql-test/t/index_merge_myisam.test b/mysql-test/t/index_merge_myisam.test index d265007431e..75beb9bd883 100644 --- a/mysql-test/t/index_merge_myisam.test +++ b/mysql-test/t/index_merge_myisam.test @@ -243,3 +243,38 @@ DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +--echo # +--echo # MDEV-16695: Estimate for rows of derived tables is very high when we are using index_merge union +--echo # + +create table t0 +( + key1 int not null, + INDEX i1(key1) +); + +insert into t0 values (1),(2),(3),(4),(5),(6),(7),(8); +let $1=7; +set @d=8; +while ($1) +{ + eval insert into t0 select key1+ @d from t0; + eval set @d=@d*2; + dec $1; +} +alter table t0 add key2 int not null, add index i2(key2); +alter table t0 add key3 int not null, add index i3(key3); +alter table t0 add key8 int not null, add index i8(key8); + +update t0 set key2=key1,key3=key1,key8=1024-key1; +analyze table t0; + +set @optimizer_switch_save=@@optimizer_switch; +set optimizer_switch='derived_merge=off,derived_with_keys=off'; +explain select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +set optimizer_use_condition_selectivity=2; +explain select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +select * from (select * from t0 where key1 = 3 or key2 =3) as Z where Z.key8 > 5; +set @@optimizer_switch= @optimizer_switch_save; +drop table t0; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 0fd2cd267fc..3bcaa72e32f 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3725,6 +3725,12 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond) } + if (quick && (quick->get_type() == QUICK_SELECT_I::QS_TYPE_ROR_UNION || + quick->get_type() == QUICK_SELECT_I::QS_TYPE_INDEX_MERGE)) + { + table->cond_selectivity*= (quick->records/table_records); + } + bitmap_union(used_fields, &handled_columns); /* Check if we can improve selectivity estimates by using sampling */ From d355be887714c726e4a312e1947dc8f8632539bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 1 Nov 2018 10:40:14 +0200 Subject: [PATCH 106/127] Remove dead code in dict_build_table_def_step() At the start of the function, we already dereferenced node, so the checks for NULL are unnecessary and redundant. --- storage/innobase/dict/dict0crea.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index ece81caafd8..758a45de943 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -431,9 +431,7 @@ dict_build_table_def_step( dberr_t err = fil_ibd_create( space, table->name.m_name, filepath, fsp_flags, - FIL_IBD_FILE_INITIAL_SIZE, - node ? node->mode : FIL_ENCRYPTION_DEFAULT, - node ? node->key_id : FIL_DEFAULT_ENCRYPTION_KEY); + FIL_IBD_FILE_INITIAL_SIZE, node->mode, node->key_id); ut_free(filepath); From abcd09c95a49d02bf136a21d62d38a2d03672f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 1 Nov 2018 10:48:56 +0200 Subject: [PATCH 107/127] mtr_t::start(): Remove unused parameters The parameters bool sync=true, bool read_only=false of mtr_t::start() were added in https://github.com/mysql/mysql-server/commit/eca5b0fc17a5bd6d4833d35a0d08c8549dd3b5ec (MySQL 5.7.3). The parameter read_only was never used anywhere. The parameter sync was only copied around, and would be returned by the unused function mtr_t::is_async(). We do not need this dead code in MariaDB. --- storage/innobase/include/mtr0mtr.h | 29 +++-------------------------- storage/innobase/include/trx0rec.h | 3 +-- storage/innobase/mtr/mtr0mtr.cc | 28 +++++----------------------- storage/innobase/trx/trx0rec.cc | 7 +++---- storage/innobase/trx/trx0trx.cc | 7 ++----- 5 files changed, 14 insertions(+), 60 deletions(-) diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index bdd3a6a67b9..4c00b07c0cc 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -37,12 +37,6 @@ Created 11/26/1995 Heikki Tuuri /** Start a mini-transaction. */ #define mtr_start(m) (m)->start() -/** Start a synchronous mini-transaction */ -#define mtr_start_sync(m) (m)->start(true) - -/** Start an asynchronous read-only mini-transaction */ -#define mtr_start_ro(m) (m)->start(true, true) - /** Commit a mini-transaction. */ #define mtr_commit(m) (m)->commit() @@ -223,22 +217,8 @@ struct mtr_t { @param[in] n_reserved number of reserved extents */ void release_free_extents(ulint n_reserved); - /** Start a mini-transaction. - @param sync true if it is a synchronous mini-transaction - @param read_only true if read only mini-transaction */ - void start(bool sync = true, bool read_only = false); - - /** @return whether this is an asynchronous mini-transaction. */ - bool is_async() const - { - return(!m_sync); - } - - /** Request a future commit to be synchronous. */ - void set_sync() - { - m_sync = true; - } + /** Start a mini-transaction. */ + void start(); /** Commit the mini-transaction. */ void commit(); @@ -590,9 +570,6 @@ private: /** LSN at commit time */ volatile lsn_t m_commit_lsn; - - /** true if it is synchronous mini-transaction */ - bool m_sync; }; #include "mtr0mtr.ic" diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index 2551d5759ae..80a12a48453 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -183,8 +183,7 @@ trx_undo_rec_get_partial_row( @param[in,out] trx transaction @param[in] table table that is being renamed @return DB_SUCCESS or error code */ -dberr_t -trx_undo_report_rename(trx_t* trx, const dict_table_t* table) +dberr_t trx_undo_report_rename(trx_t* trx, const dict_table_t* table) MY_ATTRIBUTE((nonnull, warn_unused_result)); /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 72b24b08871..9367d537424 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -383,18 +383,8 @@ public: /** Constructor. Takes ownership of the mtr->m_impl, is responsible for deleting it. @param[in,out] mtr mini-transaction */ - explicit Command(mtr_t* mtr) - : - m_locks_released() - { - init(mtr); - } - - void init(mtr_t* mtr) - { - m_impl = &mtr->m_impl; - m_sync = mtr->m_sync; - } + explicit Command(mtr_t* mtr) : m_impl(&mtr->m_impl), m_locks_released() + {} /** Destructor */ ~Command() @@ -427,9 +417,6 @@ private: @return number of bytes to write in finish_write() */ ulint prepare_write(); - /** true if it is a sync mini-transaction. */ - bool m_sync; - /** The mini-transaction state. */ mtr_t::Impl* m_impl; @@ -488,18 +475,13 @@ mtr_write_log( log_close(); } -/** Start a mini-transaction. -@param sync true if it is a synchronous mini-transaction -@param read_only true if read only mini-transaction */ -void -mtr_t::start(bool sync, bool read_only) +/** Start a mini-transaction. */ +void mtr_t::start() { UNIV_MEM_INVALID(this, sizeof(*this)); UNIV_MEM_INVALID(&m_impl, sizeof(m_impl)); - m_sync = sync; - m_commit_lsn = 0; new(&m_impl.m_log) mtr_buf_t(); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 9f47fd89945..9c0008831a8 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1932,8 +1932,7 @@ trx_undo_page_report_rename(trx_t* trx, const dict_table_t* table, @param[in,out] trx transaction @param[in] table table that is being renamed @return DB_SUCCESS or error code */ -dberr_t -trx_undo_report_rename(trx_t* trx, const dict_table_t* table) +dberr_t trx_undo_report_rename(trx_t* trx, const dict_table_t* table) { ut_ad(!trx->read_only); ut_ad(trx->id); @@ -1949,7 +1948,7 @@ trx_undo_report_rename(trx_t* trx, const dict_table_t* table) ut_ad((err == DB_SUCCESS) == (*pundo != NULL)); if (trx_undo_t* undo = *pundo) { mtr_t mtr; - mtr.start(trx); + mtr.start(); buf_block_t* block = buf_page_get_gen( page_id_t(undo->space, undo->last_page_no), @@ -1978,7 +1977,7 @@ trx_undo_report_rename(trx_t* trx, const dict_table_t* table) break; } else { mtr.commit(); - mtr.start(trx); + mtr.start(); block = trx_undo_add_page(trx, undo, &mtr); if (!block) { err = DB_OUT_OF_FILE_SPACE; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index c3dd7e29c37..4aba7a64ed8 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1838,9 +1838,6 @@ trx_commit_low( bool serialised; if (mtr != NULL) { - - mtr->set_sync(); - serialised = trx_write_serialisation_history(trx, mtr); /* The following call commits the mini-transaction, making the @@ -1905,7 +1902,7 @@ trx_commit( if (trx->has_logged()) { mtr = &local_mtr; - mtr_start_sync(mtr); + mtr->start(); } else { mtr = NULL; @@ -2566,7 +2563,7 @@ trx_prepare_low(trx_t* trx) trx_rseg_t* rseg = trx->rsegs.m_redo.rseg; - mtr.start(true); + mtr.start(); /* Change the undo log segment states from TRX_UNDO_ACTIVE to TRX_UNDO_PREPARED: these modifications to the file data From dafbd50e8a09340022e5207d2223372666fdabc8 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Thu, 1 Nov 2018 16:06:03 +0200 Subject: [PATCH 108/127] MDEV-17133 follow-up patch to fix mf_iocache-t unittest which did not always correctly simulated io-cache::end_of_file. The error was caused by implicit cast to unsigned of an intemediate term in a formula. --- unittest/sql/mf_iocache-t.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc index 95451d52f86..1eef8365074 100644 --- a/unittest/sql/mf_iocache-t.cc +++ b/unittest/sql/mf_iocache-t.cc @@ -326,7 +326,7 @@ void mdev17133() MY_MIN(sizeof(buf_o), info.end_of_file + eof_block_size + // plus 25% of block for randomization to the average - (eof_block_size/4 - rand() % (eof_block_size/2))); + eof_block_size/4 - rand() % (eof_block_size/2)); // read a chunk by blocks of variable size read_iter times // the last block completes the current chunk From 4acfc6ecd9085ff3307aa741351924287fb9593c Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Thu, 27 Sep 2018 23:34:24 +0300 Subject: [PATCH 109/127] MDEV-17038 ALTER TABLE CHANGE COLUMN c1 c1 bigint NOT NULL - generates error if table uses SYSTEM VERSIONING * Fine-grained inplace skipping by INNOBASE_ALTER_VERSIONED_REBUILD; * Fixed column WITHOUT SYSTEM VERSIONING + ADD COLUMN; * Fixed instant field change (MDEV-16330); * Revisited test versioning.online; * Merged the test versioning.trx_id_versioning_attribute_persistence to versioning.online; * Renamed some versioning functions: ** change_fields_versioning_cache() -> vers_change_fields_cache() ** change_field_versioning_try() -> vers_change_field_try() Skip condition moved out of func. Closes tempesta-tech/mariadb#414 Closes tempesta-tech/mariadb#540 Related to tempesta-tech/mariadb#281 --- .../suite/versioning/innodb.combinations | 5 + mysql-test/suite/versioning/innodb.inc | 1 + mysql-test/suite/versioning/r/alter.result | 2 +- mysql-test/suite/versioning/r/online.result | 136 +++++++++++++--- ...id_versioning_attribute_persistence.result | 86 ---------- mysql-test/suite/versioning/t/online.test | 154 ++++++++++++++---- ...x_id_versioning_attribute_persistence.test | 87 ---------- storage/innobase/handler/handler0alter.cc | 62 ++++--- 8 files changed, 277 insertions(+), 256 deletions(-) create mode 100644 mysql-test/suite/versioning/innodb.combinations create mode 100644 mysql-test/suite/versioning/innodb.inc delete mode 100644 mysql-test/suite/versioning/r/trx_id_versioning_attribute_persistence.result delete mode 100644 mysql-test/suite/versioning/t/trx_id_versioning_attribute_persistence.test diff --git a/mysql-test/suite/versioning/innodb.combinations b/mysql-test/suite/versioning/innodb.combinations new file mode 100644 index 00000000000..1a0812cfafe --- /dev/null +++ b/mysql-test/suite/versioning/innodb.combinations @@ -0,0 +1,5 @@ +[timestamp] +default-storage-engine=innodb + +[trx_id] +default-storage-engine=innodb diff --git a/mysql-test/suite/versioning/innodb.inc b/mysql-test/suite/versioning/innodb.inc new file mode 100644 index 00000000000..c841fece702 --- /dev/null +++ b/mysql-test/suite/versioning/innodb.inc @@ -0,0 +1 @@ +--source include/have_innodb.inc diff --git a/mysql-test/suite/versioning/r/alter.result b/mysql-test/suite/versioning/r/alter.result index 8db23ef6d39..f1637d79543 100644 --- a/mysql-test/suite/versioning/r/alter.result +++ b/mysql-test/suite/versioning/r/alter.result @@ -370,7 +370,7 @@ a b 1 NULL 2 NULL 3 1 -4 4 +4 2 create or replace table t (a int) with system versioning; insert into t values (1), (2), (3); delete from t where a<3; diff --git a/mysql-test/suite/versioning/r/online.result b/mysql-test/suite/versioning/r/online.result index b2a34481d63..21441acb6b2 100644 --- a/mysql-test/suite/versioning/r/online.result +++ b/mysql-test/suite/versioning/r/online.result @@ -1,34 +1,116 @@ set system_versioning_alter_history=keep; -create or replace table t (a int, b int) engine=innodb; +create or replace table t (a int); alter table t add system versioning, lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED +ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned timestamp tables. Try LOCK=SHARED +alter table t add system versioning, algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Not implemented for system-versioned timestamp tables. Try ALGORITHM=COPY alter table t add system versioning, lock=shared; -alter table t drop column b, lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED -alter table t drop column b, algorithm=inplace; -ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Not implemented for system-versioned tables. Try ALGORITHM=COPY -alter table t add index idx(a), lock=none; +alter table t add column b int, change column a a int without system versioning, lock=none; alter table t drop system versioning, lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED +ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned operations. Try LOCK=SHARED alter table t drop system versioning, algorithm=inplace; -ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Not implemented for system-versioned tables. Try ALGORITHM=COPY -create or replace table t (a int, b int) engine=innodb; -alter table t -add s bigint unsigned as row start, -add e bigint unsigned as row end, -add period for system_time(s, e), -add system versioning, -lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED -alter table t -add s bigint unsigned as row start, -add e bigint unsigned as row end, -add period for system_time(s, e), -add system versioning; +create or replace table t ( +a int, b int, +row_start SYS_DATATYPE as row start invisible, +row_end SYS_DATATYPE as row end invisible, +period for system_time (row_start, row_end) +) with system versioning; +insert into t values (1, 0); +insert into t values (2, 0); +delete from t where a = 2; alter table t drop column b, lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED alter table t add index idx(a), lock=none; -alter table t drop column s, drop column e; -alter table t drop system versioning, lock=none; -ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED -drop table t; +select a, check_row(row_start, row_end) from t for system_time all order by a; +a check_row(row_start, row_end) +1 CURRENT ROW +2 HISTORICAL ROW +# MDEV-17038 ALTER TABLE CHANGE COLUMN c1 c1 bigint NOT NULL - +# generates error if table uses SYSTEM VERSIONING [tempesta-tech/mariadb#540] +create or replace table t1 (a int, key(a)) with system versioning; +create or replace table t2 like t; +alter table t2 add foreign key(a) references t1(a); +alter table t2 modify column a int not null, lock=none; +drop table t2; +drop table t1; +# MDEV-16330 Allow instant change of WITH SYSTEM VERSIONING column attribute +create or replace table t1 ( +a int, +b int, +row_start SYS_DATATYPE as row start invisible, +row_end SYS_DATATYPE as row end invisible, +period for system_time(row_start, row_end) +) with system versioning; +create or replace table t2 ( +a int without system versioning, +b int, +row_start SYS_DATATYPE as row start invisible, +row_end SYS_DATATYPE as row end invisible, +period for system_time(row_start, row_end) +) with system versioning; +insert into t1 values (1,1); +insert into t2 values (1,1); +set @@system_versioning_alter_history=keep; +# without rebuild +alter table t1 +change a a int without system versioning, +algorithm=instant; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +alter table t2 +change a a int with system versioning, +add primary key pk (a), +algorithm=instant; +ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE +# with rebuild +alter table t2 +change a a int with system versioning, +add primary key pk (a); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +update t1 set a=2; +select count(*) from t1 for system_time all; +count(*) +1 +update t2 set a=2; +select count(*) from t2 for system_time all; +count(*) +2 +drop table t1, t2; +# rollback ALTER TABLE: nothing should change +create or replace table t ( +a int, +b int, +row_start SYS_DATATYPE as row start invisible, +row_end SYS_DATATYPE as row end invisible, +period for system_time(row_start, row_end) +) with system versioning; +insert into t values (1, 1); +select c.prtype from information_schema.innodb_sys_columns as c +join information_schema.innodb_sys_tables as t on c.table_id=t.table_id +where t.name='test/t' and c.name='b'; +prtype +50179 +set @@system_versioning_alter_history=keep; +select c.prtype from information_schema.innodb_sys_columns as c +join information_schema.innodb_sys_tables as t on c.table_id=t.table_id +where t.name='test/t' and c.name='b'; +prtype +50179 +show create table t; +Table Create Table +t CREATE TABLE `t` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `row_start` SYS_DATATYPE GENERATED ALWAYS AS ROW START INVISIBLE, + `row_end` SYS_DATATYPE GENERATED ALWAYS AS ROW END INVISIBLE, + PERIOD FOR SYSTEM_TIME (`row_start`, `row_end`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING +select count(*) from t for system_time all; +count(*) +1 +update t set b=11; +select count(*) from t for system_time all; +count(*) +2 +drop database test; +create database test; diff --git a/mysql-test/suite/versioning/r/trx_id_versioning_attribute_persistence.result b/mysql-test/suite/versioning/r/trx_id_versioning_attribute_persistence.result deleted file mode 100644 index bf555fb2a7d..00000000000 --- a/mysql-test/suite/versioning/r/trx_id_versioning_attribute_persistence.result +++ /dev/null @@ -1,86 +0,0 @@ -CREATE OR REPLACE TABLE t1 ( -a INT, -b INT, -row_start BIGINT UNSIGNED AS ROW START INVISIBLE, -row_end BIGINT UNSIGNED AS ROW END INVISIBLE, -PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; -CREATE OR REPLACE TABLE t2 ( -a INT WITHOUT SYSTEM VERSIONING, -b INT, -row_start BIGINT UNSIGNED AS ROW START INVISIBLE, -row_end BIGINT UNSIGNED AS ROW END INVISIBLE, -PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; -INSERT INTO t1 VALUES (1,1); -INSERT INTO t2 VALUES (1,1); -SET @@SYSTEM_VERSIONING_ALTER_HISTORY=KEEP; -# without rebuild -ALTER TABLE t1 -CHANGE a a INT WITHOUT SYSTEM VERSIONING, -ALGORITHM=INSTANT; -affected rows: 0 -info: Records: 0 Duplicates: 0 Warnings: 0 -ALTER TABLE t2 -CHANGE a a INT WITH SYSTEM VERSIONING, -ADD PRIMARY KEY pk (a), -ALGORITHM=INSTANT; -ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE -# with rebuild -ALTER TABLE t2 -CHANGE a a INT WITH SYSTEM VERSIONING, -ADD PRIMARY KEY pk (a); -affected rows: 0 -info: Records: 0 Duplicates: 0 Warnings: 0 -UPDATE t1 SET a=2; -SELECT COUNT(*) FROM t1 FOR SYSTEM_TIME ALL; -COUNT(*) -1 -UPDATE t2 SET a=2; -SELECT COUNT(*) FROM t2 FOR SYSTEM_TIME ALL; -COUNT(*) -2 -DROP TABLE t1, t2; -# rollback ALTER TABLE: nothing should change -CREATE TABLE t ( -a INT, -b INT, -row_start BIGINT UNSIGNED AS ROW START INVISIBLE, -row_end BIGINT UNSIGNED AS ROW END INVISIBLE, -PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; -INSERT INTO t VALUES (1, 1); -SELECT C.PRTYPE FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS AS C -JOIN INFORMATION_SCHEMA.INNODB_SYS_TABLES AS t ON C.TABLE_ID=t.TABLE_ID -WHERE t.NAME='test/t' AND C.NAME='b'; -PRTYPE -50179 -SET @@SYSTEM_VERSIONING_ALTER_HISTORY=KEEP; -SET @SAVED_DEBUG_DBUG = @@SESSION.DEBUG_DBUG; -SET DEBUG_DBUG='+d,ib_commit_inplace_fail_1'; -ALTER TABLE t -CHANGE b b INT WITHOUT SYSTEM VERSIONING; -ERROR HY000: Internal error: Injected error! -SET DEBUG_DBUG = @SAVED_DEBUG_DBUG; -SELECT C.PRTYPE FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS AS C -JOIN INFORMATION_SCHEMA.INNODB_SYS_TABLES AS t ON C.TABLE_ID=t.TABLE_ID -WHERE t.NAME='test/t' AND C.NAME='b'; -PRTYPE -50179 -SHOW CREATE TABLE t; -Table Create Table -t CREATE TABLE `t` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `row_start` bigint(20) unsigned GENERATED ALWAYS AS ROW START INVISIBLE, - `row_end` bigint(20) unsigned GENERATED ALWAYS AS ROW END INVISIBLE, - PERIOD FOR SYSTEM_TIME (`row_start`, `row_end`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING -SELECT COUNT(*) FROM t FOR SYSTEM_TIME ALL; -COUNT(*) -1 -UPDATE t SET b=11; -SELECT COUNT(*) FROM t FOR SYSTEM_TIME ALL; -COUNT(*) -2 -DROP TABLE t; diff --git a/mysql-test/suite/versioning/t/online.test b/mysql-test/suite/versioning/t/online.test index 4fbd5d85100..37405bb99ce 100644 --- a/mysql-test/suite/versioning/t/online.test +++ b/mysql-test/suite/versioning/t/online.test @@ -1,42 +1,138 @@ ---source include/have_innodb.inc +--source suite/versioning/innodb.inc +--source suite/versioning/common.inc +--source include/maybe_debug.inc set system_versioning_alter_history=keep; -create or replace table t (a int, b int) engine=innodb; - +create or replace table t (a int); --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON alter table t add system versioning, lock=none; +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +alter table t add system versioning, algorithm=inplace; alter table t add system versioning, lock=shared; ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON -alter table t drop column b, lock=none; ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON -alter table t drop column b, algorithm=inplace; -alter table t add index idx(a), lock=none; + +alter table t add column b int, change column a a int without system versioning, lock=none; --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON alter table t drop system versioning, lock=none; ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON alter table t drop system versioning, algorithm=inplace; - -create or replace table t (a int, b int) engine=innodb; - ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON -alter table t - add s bigint unsigned as row start, - add e bigint unsigned as row end, - add period for system_time(s, e), - add system versioning, - lock=none; -alter table t - add s bigint unsigned as row start, - add e bigint unsigned as row end, - add period for system_time(s, e), - add system versioning; ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +--replace_result $sys_datatype_expl SYS_DATATYPE +eval create or replace table t ( + a int, b int, + row_start $sys_datatype_expl as row start invisible, + row_end $sys_datatype_expl as row end invisible, + period for system_time (row_start, row_end) +) with system versioning; +insert into t values (1, 0); +insert into t values (2, 0); +delete from t where a = 2; alter table t drop column b, lock=none; alter table t add index idx(a), lock=none; -alter table t drop column s, drop column e; ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON -alter table t drop system versioning, lock=none; -drop table t; +select a, check_row(row_start, row_end) from t for system_time all order by a; + +--echo # MDEV-17038 ALTER TABLE CHANGE COLUMN c1 c1 bigint NOT NULL - +--echo # generates error if table uses SYSTEM VERSIONING [tempesta-tech/mariadb#540] +create or replace table t1 (a int, key(a)) with system versioning; +create or replace table t2 like t; +alter table t2 add foreign key(a) references t1(a); +alter table t2 modify column a int not null, lock=none; + +drop table t2; +drop table t1; + +--echo # MDEV-16330 Allow instant change of WITH SYSTEM VERSIONING column attribute +--replace_result $sys_datatype_expl SYS_DATATYPE +eval create or replace table t1 ( + a int, + b int, + row_start $sys_datatype_expl as row start invisible, + row_end $sys_datatype_expl as row end invisible, + period for system_time(row_start, row_end) +) with system versioning; + +--replace_result $sys_datatype_expl SYS_DATATYPE +eval create or replace table t2 ( + a int without system versioning, + b int, + row_start $sys_datatype_expl as row start invisible, + row_end $sys_datatype_expl as row end invisible, + period for system_time(row_start, row_end) +) with system versioning; + +insert into t1 values (1,1); +insert into t2 values (1,1); + +set @@system_versioning_alter_history=keep; + +--enable_info +--echo # without rebuild +alter table t1 + change a a int without system versioning, + algorithm=instant; + +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t2 + change a a int with system versioning, + add primary key pk (a), + algorithm=instant; + +--echo # with rebuild +alter table t2 + change a a int with system versioning, + add primary key pk (a); +--disable_info + +--source include/restart_mysqld.inc + +update t1 set a=2; +select count(*) from t1 for system_time all; + +update t2 set a=2; +select count(*) from t2 for system_time all; + +drop table t1, t2; + +--echo # rollback ALTER TABLE: nothing should change +--replace_result $sys_datatype_expl SYS_DATATYPE +eval create or replace table t ( + a int, + b int, + row_start $sys_datatype_expl as row start invisible, + row_end $sys_datatype_expl as row end invisible, + period for system_time(row_start, row_end) +) with system versioning; + +insert into t values (1, 1); + +select c.prtype from information_schema.innodb_sys_columns as c + join information_schema.innodb_sys_tables as t on c.table_id=t.table_id + where t.name='test/t' and c.name='b'; + +set @@system_versioning_alter_history=keep; + +if ($have_debug) { +--disable_query_log +--disable_result_log +set debug_dbug='+d,ib_commit_inplace_fail_1'; +--error ER_INTERNAL_ERROR +alter table t + change b b int without system versioning; +set debug_dbug= default; +--enable_query_log +--enable_result_log +} + +select c.prtype from information_schema.innodb_sys_columns as c + join information_schema.innodb_sys_tables as t on c.table_id=t.table_id + where t.name='test/t' and c.name='b'; + +--replace_result $sys_datatype_expl SYS_DATATYPE +show create table t; + +select count(*) from t for system_time all; +update t set b=11; +select count(*) from t for system_time all; + +drop database test; +create database test; diff --git a/mysql-test/suite/versioning/t/trx_id_versioning_attribute_persistence.test b/mysql-test/suite/versioning/t/trx_id_versioning_attribute_persistence.test deleted file mode 100644 index 4e92ee247e4..00000000000 --- a/mysql-test/suite/versioning/t/trx_id_versioning_attribute_persistence.test +++ /dev/null @@ -1,87 +0,0 @@ --- source include/have_innodb.inc --- source include/have_debug.inc - -CREATE OR REPLACE TABLE t1 ( - a INT, - b INT, - row_start BIGINT UNSIGNED AS ROW START INVISIBLE, - row_end BIGINT UNSIGNED AS ROW END INVISIBLE, - PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; - -CREATE OR REPLACE TABLE t2 ( - a INT WITHOUT SYSTEM VERSIONING, - b INT, - row_start BIGINT UNSIGNED AS ROW START INVISIBLE, - row_end BIGINT UNSIGNED AS ROW END INVISIBLE, - PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; - -INSERT INTO t1 VALUES (1,1); -INSERT INTO t2 VALUES (1,1); - -SET @@SYSTEM_VERSIONING_ALTER_HISTORY=KEEP; - ---enable_info ---echo # without rebuild -ALTER TABLE t1 - CHANGE a a INT WITHOUT SYSTEM VERSIONING, - ALGORITHM=INSTANT; - ---error ER_ALTER_OPERATION_NOT_SUPPORTED -ALTER TABLE t2 - CHANGE a a INT WITH SYSTEM VERSIONING, - ADD PRIMARY KEY pk (a), - ALGORITHM=INSTANT; - ---echo # with rebuild -ALTER TABLE t2 - CHANGE a a INT WITH SYSTEM VERSIONING, - ADD PRIMARY KEY pk (a); ---disable_info - ---source include/restart_mysqld.inc - -UPDATE t1 SET a=2; -SELECT COUNT(*) FROM t1 FOR SYSTEM_TIME ALL; - -UPDATE t2 SET a=2; -SELECT COUNT(*) FROM t2 FOR SYSTEM_TIME ALL; - -DROP TABLE t1, t2; - ---echo # rollback ALTER TABLE: nothing should change -CREATE TABLE t ( - a INT, - b INT, - row_start BIGINT UNSIGNED AS ROW START INVISIBLE, - row_end BIGINT UNSIGNED AS ROW END INVISIBLE, - PERIOD FOR SYSTEM_TIME(row_start, row_end) -) WITH SYSTEM VERSIONING ENGINE=INNODB; - -INSERT INTO t VALUES (1, 1); - -SELECT C.PRTYPE FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS AS C - JOIN INFORMATION_SCHEMA.INNODB_SYS_TABLES AS t ON C.TABLE_ID=t.TABLE_ID - WHERE t.NAME='test/t' AND C.NAME='b'; - -SET @@SYSTEM_VERSIONING_ALTER_HISTORY=KEEP; - -SET @SAVED_DEBUG_DBUG = @@SESSION.DEBUG_DBUG; -SET DEBUG_DBUG='+d,ib_commit_inplace_fail_1'; ---error ER_INTERNAL_ERROR -ALTER TABLE t - CHANGE b b INT WITHOUT SYSTEM VERSIONING; -SET DEBUG_DBUG = @SAVED_DEBUG_DBUG; - -SELECT C.PRTYPE FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS AS C - JOIN INFORMATION_SCHEMA.INNODB_SYS_TABLES AS t ON C.TABLE_ID=t.TABLE_ID - WHERE t.NAME='test/t' AND C.NAME='b'; - -SHOW CREATE TABLE t; - -SELECT COUNT(*) FROM t FOR SYSTEM_TIME ALL; -UPDATE t SET b=11; -SELECT COUNT(*) FROM t FOR SYSTEM_TIME ALL; - -DROP TABLE t; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 13a963575a6..4cca714b5d0 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -73,6 +73,12 @@ static const alter_table_operations INNOBASE_DEFAULTS = ALTER_COLUMN_NOT_NULLABLE | ALTER_ADD_STORED_BASE_COLUMN; + +/** Operations that require knowledge about row_start, row_end values */ +static const alter_table_operations INNOBASE_ALTER_VERSIONED_REBUILD + = ALTER_ADD_SYSTEM_VERSIONING + | ALTER_DROP_SYSTEM_VERSIONING; + /** Operations for rebuilding a table in place */ static const alter_table_operations INNOBASE_ALTER_REBUILD = ALTER_ADD_PK_INDEX @@ -87,8 +93,7 @@ static const alter_table_operations INNOBASE_ALTER_REBUILD /* | ALTER_STORED_COLUMN_TYPE */ - | ALTER_ADD_SYSTEM_VERSIONING - | ALTER_DROP_SYSTEM_VERSIONING + | INNOBASE_ALTER_VERSIONED_REBUILD ; /** Operations that require changes to data */ @@ -867,10 +872,11 @@ ha_innobase::check_if_supported_inplace_alter( const bool need_rebuild = innobase_need_rebuild(ha_alter_info, table); if (need_rebuild - && (table->versioned(VERS_TIMESTAMP) - || altered_table->versioned(VERS_TIMESTAMP))) { + && altered_table->versioned(VERS_TIMESTAMP) + && (ha_alter_info->handler_flags + & INNOBASE_ALTER_VERSIONED_REBUILD)) { ha_alter_info->unsupported_reason = - "Not implemented for system-versioned tables"; + "Not implemented for system-versioned timestamp tables"; DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } @@ -1409,13 +1415,12 @@ cannot_create_many_fulltext_index: } } - // FIXME: implement Online DDL for system-versioned tables - if (need_rebuild && - (table->versioned(VERS_TRX_ID) - || altered_table->versioned(VERS_TRX_ID))) { + // FIXME: implement Online DDL for system-versioned operations + if (ha_alter_info->handler_flags & INNOBASE_ALTER_VERSIONED_REBUILD) { + if (ha_alter_info->online) { ha_alter_info->unsupported_reason = - "Not implemented for system-versioned tables"; + "Not implemented for system-versioned operations"; } online = false; @@ -5366,7 +5371,9 @@ new_clustered_failed: ut_d(const dict_index_t* index = user_table->indexes.start); DBUG_SLOW_ASSERT(col->mtype == old_col->mtype); - DBUG_SLOW_ASSERT(col->prtype == old_col->prtype); + ut_ad(col->prtype == old_col->prtype + || col->prtype + == (old_col->prtype & ~DATA_VERSIONED)); DBUG_SLOW_ASSERT(col->mbminlen == old_col->mbminlen); DBUG_SLOW_ASSERT(col->mbmaxlen @@ -8457,14 +8464,14 @@ innobase_update_foreign_cache( @retval false on success */ static bool -change_field_versioning_try( +vers_change_field_try( trx_t* trx, const char* table_name, const table_id_t tableid, const ulint pos, const ulint prtype) { - DBUG_ENTER("change_field_versioning_try"); + DBUG_ENTER("vers_change_field_try"); pars_info_t* info = pars_info_create(); @@ -8500,25 +8507,24 @@ change_field_versioning_try( @retval false on success */ static bool -change_fields_versioning_try( +vers_change_fields_try( const Alter_inplace_info* ha_alter_info, const ha_innobase_inplace_ctx* ctx, trx_t* trx, const TABLE* table) { - DBUG_ENTER("change_fields_versioning_try"); + DBUG_ENTER("vers_change_fields_try"); DBUG_ASSERT(ha_alter_info); DBUG_ASSERT(ctx); - if (!(ha_alter_info->handler_flags & ALTER_COLUMN_UNVERSIONED)){ - DBUG_RETURN(false); - } - List_iterator_fast it( ha_alter_info->alter_info->create_list); while (const Create_field* create_field = it++) { + if (!create_field->field) { + continue; + } if (create_field->versioning == Column_definition::VERSIONING_NOT_SET) { continue; @@ -8537,9 +8543,9 @@ change_fields_versioning_try( ? col->prtype & ~DATA_VERSIONED : col->prtype | DATA_VERSIONED; - if (change_field_versioning_try(trx, table->s->table_name.str, - new_table->id, pos, - new_prtype)) { + if (vers_change_field_try(trx, table->s->table_name.str, + new_table->id, pos, + new_prtype)) { DBUG_RETURN(true); } } @@ -8554,12 +8560,12 @@ in the data dictionary cache. @param table MySQL table as it is before the ALTER operation */ static void -change_fields_versioning_cache( +vers_change_fields_cache( Alter_inplace_info* ha_alter_info, const ha_innobase_inplace_ctx* ctx, const TABLE* table) { - DBUG_ENTER("change_fields_versioning"); + DBUG_ENTER("vers_change_fields_cache"); DBUG_ASSERT(ha_alter_info); DBUG_ASSERT(ctx); @@ -8569,6 +8575,9 @@ change_fields_versioning_cache( ha_alter_info->alter_info->create_list); while (const Create_field* create_field = it++) { + if (!create_field->field) { + continue; + } dict_col_t* col = dict_table_get_nth_col( ctx->new_table, innodb_col_no(create_field->field)); @@ -8959,7 +8968,8 @@ commit_try_norebuild( DBUG_RETURN(true); } - if (change_fields_versioning_try(ha_alter_info, ctx, trx, old_table)) { + if ((ha_alter_info->handler_flags & ALTER_COLUMN_UNVERSIONED) + && vers_change_fields_try(ha_alter_info, ctx, trx, old_table)) { DBUG_RETURN(true); } @@ -9220,7 +9230,7 @@ commit_cache_norebuild( } if (ha_alter_info->handler_flags & ALTER_COLUMN_UNVERSIONED) { - change_fields_versioning_cache(ha_alter_info, ctx, table); + vers_change_fields_cache(ha_alter_info, ctx, table); } #ifdef MYSQL_RENAME_INDEX From dd6e74c62a2aa44d9d5e1790bcd36d3ff1c7d98b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 31 Oct 2018 22:20:51 +0100 Subject: [PATCH 110/127] MDEV-16774 SET PASSWORD and ALTER USER with slightly different results set both `password` and `authentication_string` columns in `mysql`.`user` table for now. Suppress the "password was ignored" warning if the password is the same as the authentication string --- mysql-test/r/set_password.result | 18 +++++++++--------- .../r/show_grants_with_plugin-7985.result | 2 +- .../suite/rpl/r/rpl_innodb_mixed_dml.result | 8 ++++---- sql/sql_acl.cc | 5 +++-- .../rpl/r/rpl_tokudb_mixed_dml.result | 8 ++++---- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/mysql-test/r/set_password.result b/mysql-test/r/set_password.result index 315d0bef9fb..733d9c96187 100644 --- a/mysql-test/r/set_password.result +++ b/mysql-test/r/set_password.result @@ -11,10 +11,10 @@ select user, host, password, plugin, authentication_string from mysql.user where user host password plugin authentication_string natauth localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 newpass localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 -newpassnat localhost mysql_native_password *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 +newpassnat localhost *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 mysql_native_password *94BDCEBE19083CE2A1F959FD02F964C7AF4CFC29 oldauth localhost 378b243e220ca493 oldpass localhost 378b243e220ca493 -oldpassold localhost mysql_old_password 378b243e220ca493 +oldpassold localhost 378b243e220ca493 mysql_old_password 378b243e220ca493 connect con,localhost,natauth,test,; select current_user(); current_user() @@ -86,12 +86,12 @@ set password for oldpass@localhost = PASSWORD('test2'); set password for oldpassold@localhost = PASSWORD('test2'); select user, host, password, plugin, authentication_string from mysql.user where user != 'root'; user host password plugin authentication_string -natauth localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -newpass localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -newpassnat localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldauth localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldpass localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E -oldpassold localhost mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +natauth localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +newpass localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +newpassnat localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldauth localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldpass localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E +oldpassold localhost *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E mysql_native_password *7CEB3FDE5F7A9C4CE5FBE610D7D8EDA62EBE5F4E connect con,localhost,natauth,test2,; select current_user(); current_user() @@ -173,7 +173,7 @@ disconnect foo; connection default; select user,host,password,plugin,authentication_string from mysql.user where user='foo'; user host password plugin authentication_string -foo localhost mysql_native_password *E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB +foo localhost *E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB mysql_native_password *E8D46CE25265E545D225A8A6F1BAF642FEBEE5CB set password for 'foo'@'localhost' = ''; select user,host,password,plugin,authentication_string from mysql.user where user='foo'; user host password plugin authentication_string diff --git a/mysql-test/r/show_grants_with_plugin-7985.result b/mysql-test/r/show_grants_with_plugin-7985.result index 0f8e1e39969..bb475b7cba9 100644 --- a/mysql-test/r/show_grants_with_plugin-7985.result +++ b/mysql-test/r/show_grants_with_plugin-7985.result @@ -71,7 +71,7 @@ connection default; set password for u1 = PASSWORD('SOMETHINGELSE'); select user, host, password, plugin, authentication_string from mysql.user where user = 'u1'; user host password plugin authentication_string -u1 % mysql_native_password *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 +u1 % *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 mysql_native_password *054B7BBD2B9A553DA560520DCD3F76DA2D81B7C6 # # Here we should use the password field, as that primes over # the authentication_string field. diff --git a/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result b/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result index 24ce8899e2c..00b50df4a68 100644 --- a/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result +++ b/mysql-test/suite/rpl/r/rpl_innodb_mixed_dml.result @@ -518,24 +518,24 @@ connection master; SET PASSWORD FOR 'user_test_rpl'@'localhost' = '*0000000000000000000000000000000000000000'; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** RENAME USER ******************** RENAME USER 'user_test_rpl'@'localhost' TO 'user_test_rpl_2'@'localhost'; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl_2 *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl_2 *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** DROP USER ******************** diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 733d844e5e4..fd213c528e1 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -2056,7 +2056,9 @@ static bool acl_load(THD *thd, const Grant_tables& tables) user_table.authentication_string())); user.auth_string.length= strlen(user.auth_string.str); - if (user.auth_string.length && password_len) + if (user.auth_string.length && password_len && + (user.auth_string.length != password_len || + memcmp(user.auth_string.str, password, password_len))) { sql_print_warning("'user' entry '%s@%s' has both a password " "and an authentication plugin specified. The " @@ -3927,7 +3929,6 @@ static bool update_user_table(THD *thd, const User_table& user_table, { set_authentication_plugin_from_password(user_table, new_password, new_password_len); - new_password_len= 0; } if (user_table.password()) diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result index a5060d9e3bf..65057791b48 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_tokudb_mixed_dml.result @@ -521,24 +521,24 @@ connection master; SET PASSWORD FOR 'user_test_rpl'@'localhost' = '*0000000000000000000000000000000000000000'; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** RENAME USER ******************** RENAME USER 'user_test_rpl'@'localhost' TO 'user_test_rpl_2'@'localhost'; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl_2 *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection slave; USE test_rpl; SELECT host, user, password, plugin, authentication_string, select_priv FROM mysql.user WHERE user LIKE 'user_test_rpl%'; host user password plugin authentication_string select_priv -localhost user_test_rpl_2 mysql_native_password *0000000000000000000000000000000000000000 N +localhost user_test_rpl_2 *0000000000000000000000000000000000000000 mysql_native_password *0000000000000000000000000000000000000000 N connection master; ******************** DROP USER ******************** From 9eb8a46790c556a9f683a14aea71e775f4fbe802 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Thu, 1 Nov 2018 11:09:32 -0400 Subject: [PATCH 111/127] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2eee908ea01..163137684a2 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=37 +MYSQL_VERSION_PATCH=38 From a33c0e3f34afd024ded83d3e5ec122c50d8b38a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 1 Nov 2018 17:30:11 +0200 Subject: [PATCH 112/127] Minor clean-up for MDEV-17038 ha_innobase::check_if_supported_inplace_alter(): Remove a redundant condition and defer some computations. If INNOBASE_ALTER_VERSIONED_REBUILD is set, innobase_need_rebuild() will necessarily hold. Therefore, it is not necessary to assign need_rebuild at the start of the function. --- storage/innobase/handler/handler0alter.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 4cca714b5d0..482484d17d7 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -869,12 +869,9 @@ ha_innobase::check_if_supported_inplace_alter( { DBUG_ENTER("check_if_supported_inplace_alter"); - const bool need_rebuild = innobase_need_rebuild(ha_alter_info, table); - - if (need_rebuild - && altered_table->versioned(VERS_TIMESTAMP) - && (ha_alter_info->handler_flags - & INNOBASE_ALTER_VERSIONED_REBUILD)) { + if ((ha_alter_info->handler_flags + & INNOBASE_ALTER_VERSIONED_REBUILD) + && altered_table->versioned(VERS_TIMESTAMP)) { ha_alter_info->unsupported_reason = "Not implemented for system-versioned timestamp tables"; DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -1312,7 +1309,8 @@ next_column: DBUG_RETURN(HA_ALTER_INPLACE_INSTANT); } - bool fts_need_rebuild = false; + bool fts_need_rebuild = false; + const bool need_rebuild = innobase_need_rebuild(ha_alter_info, table); if (!online) { /* We already determined that only a non-locking From 1c6b982e02eeaa75bb6c2f2a3c2b64491dd6d3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Thu, 1 Nov 2018 08:55:16 +0200 Subject: [PATCH 113/127] MDEV-12779 Oracle/DB2 Compatibility Implicit Ordering for ROW_NUMBER OVER Users expect window functions to produce a certain ordering of rows in the final result set. Although the standard does not require this, we already have the filesort result done for when we computed the window function. If there is no ORDER BY attached to the query, just keep it till the SELECT is completely evaluated and use that to print the result. Update test cases as many did not take care to guarantee a stable result. --- mysql-test/r/win.result | 380 +++++++++++------------ mysql-test/r/win_first_last_value.result | 20 +- mysql-test/r/win_percent_cume.result | 72 ++--- mysql-test/r/win_rank.result | 83 ++--- mysql-test/t/win.test | 24 +- mysql-test/t/win_percent_cume.test | 6 +- mysql-test/t/win_rank.test | 17 +- mysql-test/t/win_std.test | 4 + mysys/mf_iocache.c | 9 + sql/sql_select.cc | 9 +- sql/sql_window.cc | 17 +- sql/sql_window.h | 4 +- 12 files changed, 339 insertions(+), 306 deletions(-) diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result index b73fe346303..790b264fc09 100644 --- a/mysql-test/r/win.result +++ b/mysql-test/r/win.result @@ -38,10 +38,10 @@ a row_number() over (partition by a order by b) select a, b, x, row_number() over (partition by a order by x) from t1; a b x row_number() over (partition by a order by x) 2 10 xx 1 -2 10 zz 3 2 20 yy 2 -3 10 xxx 2 +2 10 zz 3 3 20 vvv 1 +3 10 xxx 2 drop table t1; create table t1 (pk int primary key, a int, b int); insert into t1 values @@ -54,13 +54,14 @@ select pk, a, b, row_number() over (order by a), row_number() over (order by b) -from t1; +from t1 +order by b; pk a b row_number() over (order by a) row_number() over (order by b) -1 10 22 1 5 -2 11 21 2 4 -3 12 20 3 3 -4 13 19 4 2 5 14 18 5 1 +4 13 19 4 2 +3 12 20 3 3 +2 11 21 2 4 +1 10 22 1 5 drop table t1; # # Try RANK() function @@ -228,16 +229,16 @@ count(*) over (partition by c order by pk desc rows between 2 preceding and 2 following) as CNT from t1; pk c CNT -1 1 3 -2 1 4 -3 1 4 4 1 3 -5 2 3 -6 2 4 -7 2 5 -8 2 5 -9 2 4 +3 1 4 +2 1 4 +1 1 3 10 2 3 +9 2 4 +8 2 5 +7 2 5 +6 2 4 +5 2 3 drop table t0,t1; # # Resolution of window names @@ -535,42 +536,40 @@ insert into t3 values (5, 2), (6, 2); select -pk, val, count(val) over (order by val range between current row and current row) as CNT from t3; -pk val CNT -0 1 3 -1 1 3 -2 1 3 -3 2 4 -4 2 4 -5 2 4 -6 2 4 +val CNT +1 3 +1 3 +1 3 +2 4 +2 4 +2 4 +2 4 insert into t3 values (7, 3), (8, 3); select -pk, val, count(val) over (order by val range between current row and current row) as CNT from t3; -pk val CNT -0 1 3 -1 1 3 -2 1 3 -3 2 4 -4 2 4 -5 2 4 -6 2 4 -7 3 2 -8 3 2 +val CNT +1 3 +1 3 +1 3 +2 4 +2 4 +2 4 +2 4 +3 2 +3 2 drop table t3; # Now, check with PARTITION BY create table t4 ( @@ -599,7 +598,6 @@ insert into t4 values (5678, 208, 3); select part_id, -pk, val, count(val) over (partition by part_id order by val @@ -607,31 +605,30 @@ range between current row and current row) as CNT from t4; -part_id pk val CNT -1234 100 1 3 -1234 101 1 3 -1234 102 1 3 -1234 103 2 4 -1234 104 2 4 -1234 105 2 4 -1234 106 2 4 -1234 107 3 2 -1234 108 3 2 -5678 200 1 3 -5678 201 1 3 -5678 202 1 3 -5678 203 2 4 -5678 204 2 4 -5678 205 2 4 -5678 206 2 4 -5678 207 3 2 -5678 208 3 2 +part_id val CNT +1234 1 3 +1234 1 3 +1234 1 3 +1234 2 4 +1234 2 4 +1234 2 4 +1234 2 4 +1234 3 2 +1234 3 2 +5678 1 3 +5678 1 3 +5678 1 3 +5678 2 4 +5678 2 4 +5678 2 4 +5678 2 4 +5678 3 2 +5678 3 2 # # Try RANGE UNBOUNDED PRECEDING | FOLLOWING # select part_id, -pk, val, count(val) over (partition by part_id order by val @@ -639,28 +636,27 @@ range between unbounded preceding and current row) as CNT from t4; -part_id pk val CNT -1234 100 1 3 -1234 101 1 3 -1234 102 1 3 -1234 103 2 7 -1234 104 2 7 -1234 105 2 7 -1234 106 2 7 -1234 107 3 9 -1234 108 3 9 -5678 200 1 3 -5678 201 1 3 -5678 202 1 3 -5678 203 2 7 -5678 204 2 7 -5678 205 2 7 -5678 206 2 7 -5678 207 3 9 -5678 208 3 9 +part_id val CNT +1234 1 3 +1234 1 3 +1234 1 3 +1234 2 7 +1234 2 7 +1234 2 7 +1234 2 7 +1234 3 9 +1234 3 9 +5678 1 3 +5678 1 3 +5678 1 3 +5678 2 7 +5678 2 7 +5678 2 7 +5678 2 7 +5678 3 9 +5678 3 9 select part_id, -pk, val, count(val) over (partition by part_id order by val @@ -668,28 +664,27 @@ range between current row and unbounded following) as CNT from t4; -part_id pk val CNT -1234 100 1 9 -1234 101 1 9 -1234 102 1 9 -1234 103 2 6 -1234 104 2 6 -1234 105 2 6 -1234 106 2 6 -1234 107 3 2 -1234 108 3 2 -5678 200 1 9 -5678 201 1 9 -5678 202 1 9 -5678 203 2 6 -5678 204 2 6 -5678 205 2 6 -5678 206 2 6 -5678 207 3 2 -5678 208 3 2 +part_id val CNT +1234 1 9 +1234 1 9 +1234 1 9 +1234 2 6 +1234 2 6 +1234 2 6 +1234 2 6 +1234 3 2 +1234 3 2 +5678 1 9 +5678 1 9 +5678 1 9 +5678 2 6 +5678 2 6 +5678 2 6 +5678 2 6 +5678 3 2 +5678 3 2 select part_id, -pk, val, count(val) over (partition by part_id order by val @@ -697,25 +692,25 @@ range between unbounded preceding and unbounded following) as CNT from t4; -part_id pk val CNT -1234 100 1 9 -1234 101 1 9 -1234 102 1 9 -1234 103 2 9 -1234 104 2 9 -1234 105 2 9 -1234 106 2 9 -1234 107 3 9 -1234 108 3 9 -5678 200 1 9 -5678 201 1 9 -5678 202 1 9 -5678 203 2 9 -5678 204 2 9 -5678 205 2 9 -5678 206 2 9 -5678 207 3 9 -5678 208 3 9 +part_id val CNT +1234 1 9 +1234 1 9 +1234 1 9 +1234 2 9 +1234 2 9 +1234 2 9 +1234 2 9 +1234 3 9 +1234 3 9 +5678 1 9 +5678 1 9 +5678 1 9 +5678 2 9 +5678 2 9 +5678 2 9 +5678 2 9 +5678 3 9 +5678 3 9 drop table t4; # # MDEV-9695: Wrong window frame when using RANGE BETWEEN N FOLLOWING AND PRECEDING @@ -881,15 +876,15 @@ RANGE BETWEEN UNBOUNDED PRECEDING AND 10 FOLLOWING) as cnt from t1; pk a cnt -1 1 9 -2 2 9 -3 4 9 -4 8 9 -5 26 5 -6 27 5 -7 40 3 -8 71 2 9 72 2 +8 71 2 +7 40 3 +6 27 5 +5 26 5 +4 8 9 +3 4 9 +2 2 9 +1 1 9 select pk, a, count(a) over (ORDER BY a @@ -929,15 +924,15 @@ RANGE BETWEEN UNBOUNDED PRECEDING AND 10 PRECEDING) as cnt from t1; pk a cnt -1 1 5 -2 2 5 -3 4 5 -4 8 5 -5 26 3 -6 27 3 -7 40 2 -8 71 0 9 72 0 +8 71 0 +7 40 2 +6 27 3 +5 26 3 +4 8 5 +3 4 5 +2 2 5 +1 1 5 select pk, a, count(a) over (ORDER BY a @@ -977,15 +972,15 @@ RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) as cnt from t1; pk a cnt -1 1 2 -2 2 1 -3 4 1 -4 8 1 -5 26 2 -6 27 1 -7 40 1 -8 71 2 9 72 1 +8 71 2 +7 40 1 +6 27 1 +5 26 2 +4 8 1 +3 4 1 +2 2 1 +1 1 2 select pk, a, count(a) over (ORDER BY a @@ -1026,15 +1021,15 @@ range between current row and 1 following) as cnt from t1; pk a cnt -1 1 1 -2 2 2 -3 4 1 -4 8 1 -5 26 1 -6 27 2 -7 40 1 -8 71 1 9 72 2 +8 71 1 +7 40 1 +6 27 2 +5 26 1 +4 8 1 +3 4 1 +2 2 2 +1 1 1 insert into t1 select 22, pk, a from t1; select part_id, pk, a, @@ -1137,7 +1132,8 @@ pk, a, count(b) over (order by a range between 2 preceding and 2 following) as CNT -from t1; +from t1 +order by a, pk; pk a CNT 1 NULL 3 2 NULL 3 @@ -1969,12 +1965,12 @@ ROW_NUMBER() OVER (order by s1), CUME_DIST() OVER (order by -s1) from t1; s1 s2 ROW_NUMBER() OVER (order by s1) CUME_DIST() OVER (order by -s1) +-1 2 1.0000000000 1 a 3 0.8333333333 -NULL NULL 1 0.1666666667 +2 b 4 0.6666666667 3 NULL 5 0.5000000000 4 a 6 0.3333333333 -2 b 4 0.6666666667 --1 2 1.0000000000 +NULL NULL 1 0.1666666667 drop table t1; # # MDEV-9925: Wrong result with aggregate function as a window function @@ -2012,8 +2008,8 @@ o_custkey avg(o_custkey) OVER (PARTITION BY abs(o_custkey) ORDER BY o_custkey RANGE BETWEEN 15 FOLLOWING AND 15 FOLLOWING) -242 NULL 238 NULL +242 NULL DROP table orders; # # MDEV-10842: window functions with the same order column @@ -2042,20 +2038,20 @@ row_number() over (order by pk desc) as r_desc, row_number() over (order by pk asc) as r_asc from t1; pk r_desc r_asc -1 11 1 -2 10 2 -3 9 3 -4 8 4 -5 7 5 -6 6 6 -7 5 7 -8 4 8 -9 3 9 -10 2 10 11 1 11 +10 2 10 +9 3 9 +8 4 8 +7 5 7 +6 6 6 +5 7 5 +4 8 4 +3 9 3 +2 10 2 +1 11 1 drop table t1; # -# MDEV-10874: two window functions with ccompatible sorting +# MDEV-10874: two window functions with compatible sorting # create table t1 ( pk int primary key, @@ -2130,17 +2126,17 @@ sum(d) over (order by a ROWS BETWEEN 1 preceding and 2 following) as sum_2 from t1; pk a d sum_1 sum_2 -1 0 0.100 0.300 1.400 -2 0 0.200 0.500 1.200 3 0 0.300 0.300 0.600 -4 1 0.400 0.900 2.600 -5 1 0.500 1.100 2.400 +2 0 0.200 0.500 1.200 +1 0 0.100 0.300 1.400 6 1 0.600 0.600 1.600 -7 2 0.500 0.500 0.500 -8 2 NULL 0.700 1.200 -9 2 0.700 1.500 2.000 -10 2 0.800 1.700 2.400 +5 1 0.500 1.100 2.400 +4 1 0.400 0.900 2.600 11 2 0.900 0.900 2.800 +10 2 0.800 1.700 2.400 +9 2 0.700 1.500 2.000 +8 2 NULL 0.700 1.200 +7 2 0.500 0.500 0.500 drop table t1; # # MDEV-9941: two window functions with compatible partitions @@ -2232,9 +2228,9 @@ SELECT sum(t.a) over (partition by t.b order by a), sqrt(ifnull((sum(t.a) over (partition by t.b order by a)), 0)) from t; sum(t.a) over (partition by t.b order by a) sqrt(ifnull((sum(t.a) over (partition by t.b order by a)), 0)) +0.0000000000 0 1.0000000000 1 3.0000000000 1.7320508075688772 -0.0000000000 0 drop table t; # # MDEV-10868: view definitions with window functions @@ -2391,9 +2387,9 @@ CREATE TABLE t2 (c VARCHAR(8)); INSERT INTO t2 VALUES ('foo'),('bar'),('foo'); SELECT COUNT(*) OVER (PARTITION BY c) FROM t2; COUNT(*) OVER (PARTITION BY c) -2 1 2 +2 SELECT * FROM t1 WHERE i IN ( SELECT COUNT(*) OVER (PARTITION BY c) FROM t2 ); i 1 @@ -2511,14 +2507,14 @@ select id, rank() over (order by id) from t1; id rank() over (order by id) 1 1 2 2 -3 4 2 2 +3 4 select * from v1; id rnk 1 1 2 2 -3 4 2 2 +3 4 drop view v1; drop table t1; # @@ -2879,64 +2875,64 @@ CDEC min(TDEC.CDEC) over () NULL -1 -1 -1 0 -1 -1 -1 0 -1 +1 -1 10 -1 select VDEC.CDEC, min(VDEC.CDEC) over () from VDEC; CDEC min(VDEC.CDEC) over () NULL -1 -1 -1 0 -1 -1 -1 0 -1 +1 -1 10 -1 select TDEC.CDEC, max(TDEC.CDEC) over () from TDEC; CDEC max(TDEC.CDEC) over () NULL 10 -1 10 0 10 -1 10 0 10 +1 10 10 10 select VDEC.CDEC, max(VDEC.CDEC) over () from VDEC; CDEC max(VDEC.CDEC) over () NULL 10 -1 10 0 10 -1 10 0 10 +1 10 10 10 select TDEC.CDEC, min(distinct TDEC.CDEC) over () from TDEC; CDEC min(distinct TDEC.CDEC) over () NULL -1 -1 -1 0 -1 -1 -1 0 -1 +1 -1 10 -1 select VDEC.CDEC, min(distinct VDEC.CDEC) over () from VDEC; CDEC min(distinct VDEC.CDEC) over () NULL -1 -1 -1 0 -1 -1 -1 0 -1 +1 -1 10 -1 select TDEC.CDEC, max(distinct TDEC.CDEC) over () from TDEC; CDEC max(distinct TDEC.CDEC) over () NULL 10 -1 10 0 10 -1 10 0 10 +1 10 10 10 select VDEC.CDEC, max(distinct VDEC.CDEC) over () from VDEC; CDEC max(distinct VDEC.CDEC) over () NULL 10 -1 10 0 10 -1 10 0 10 +1 10 10 10 # # These should be removed once support for them is added. @@ -3339,27 +3335,29 @@ SET @save_sql_mode= @@sql_mode; SET sql_mode = 'ONLY_FULL_GROUP_BY'; SELECT name, test, score, AVG(score) OVER (PARTITION BY test) AS average_by_test -FROM t1; +FROM t1 +ORDER BY test, name; name test score average_by_test Chun SQL 75 65.2500 -Chun Tuning 73 68.7500 Esben SQL 43 65.2500 -Esben Tuning 31 68.7500 Kaolin SQL 56 65.2500 -Kaolin Tuning 88 68.7500 Tatiana SQL 87 65.2500 +Chun Tuning 73 68.7500 +Esben Tuning 31 68.7500 +Kaolin Tuning 88 68.7500 Tatiana Tuning 83 68.7500 set @@sql_mode= @save_sql_mode; SELECT name, test, score, AVG(score) OVER (PARTITION BY test) AS average_by_test -FROM t1; +FROM t1 +ORDER BY test, name; name test score average_by_test Chun SQL 75 65.2500 -Chun Tuning 73 68.7500 Esben SQL 43 65.2500 -Esben Tuning 31 68.7500 Kaolin SQL 56 65.2500 -Kaolin Tuning 88 68.7500 Tatiana SQL 87 65.2500 +Chun Tuning 73 68.7500 +Esben Tuning 31 68.7500 +Kaolin Tuning 88 68.7500 Tatiana Tuning 83 68.7500 drop table t1; diff --git a/mysql-test/r/win_first_last_value.result b/mysql-test/r/win_first_last_value.result index b8fab458249..1717bf19eaa 100644 --- a/mysql-test/r/win_first_last_value.result +++ b/mysql-test/r/win_first_last_value.result @@ -68,17 +68,17 @@ first_value(pk) over (order by pk desc), last_value(pk) over (order by pk desc) from t1; pk first_value(pk) over (order by pk desc) last_value(pk) over (order by pk desc) -1 11 1 -2 11 2 -3 11 3 -4 11 4 -5 11 5 -6 11 6 -7 11 7 -8 11 8 -9 11 9 -10 11 10 11 11 11 +10 11 10 +9 11 9 +8 11 8 +7 11 7 +6 11 6 +5 11 5 +4 11 4 +3 11 3 +2 11 2 +1 11 1 select pk, a, b, c, d, e, first_value(b) over (partition by a order by pk) as fst_b, last_value(b) over (partition by a order by pk) as lst_b, diff --git a/mysql-test/r/win_percent_cume.result b/mysql-test/r/win_percent_cume.result index d38c95c9ea7..bc2a4182e30 100644 --- a/mysql-test/r/win_percent_cume.result +++ b/mysql-test/r/win_percent_cume.result @@ -14,49 +14,49 @@ insert into t1 values ( 7 , 2, 20), ( 9 , 4, 20), (10 , 4, 20); -select pk, a, b, +select a, percent_rank() over (order by a), cume_dist() over (order by a) from t1; -pk a b percent_rank() over (order by a) cume_dist() over (order by a) -1 0 10 0.0000000000 0.2000000000 -2 0 10 0.0000000000 0.2000000000 -3 1 10 0.2222222222 0.4000000000 -4 1 10 0.2222222222 0.4000000000 -8 2 10 0.4444444444 0.8000000000 -5 2 20 0.4444444444 0.8000000000 -6 2 20 0.4444444444 0.8000000000 -7 2 20 0.4444444444 0.8000000000 -9 4 20 0.8888888889 1.0000000000 -10 4 20 0.8888888889 1.0000000000 -select pk, a, b, +a percent_rank() over (order by a) cume_dist() over (order by a) +0 0.0000000000 0.2000000000 +0 0.0000000000 0.2000000000 +1 0.2222222222 0.4000000000 +1 0.2222222222 0.4000000000 +2 0.4444444444 0.8000000000 +2 0.4444444444 0.8000000000 +2 0.4444444444 0.8000000000 +2 0.4444444444 0.8000000000 +4 0.8888888889 1.0000000000 +4 0.8888888889 1.0000000000 +select pk, percent_rank() over (order by pk), cume_dist() over (order by pk) from t1 order by pk; -pk a b percent_rank() over (order by pk) cume_dist() over (order by pk) -1 0 10 0.0000000000 0.1000000000 -2 0 10 0.1111111111 0.2000000000 -3 1 10 0.2222222222 0.3000000000 -4 1 10 0.3333333333 0.4000000000 -5 2 20 0.4444444444 0.5000000000 -6 2 20 0.5555555556 0.6000000000 -7 2 20 0.6666666667 0.7000000000 -8 2 10 0.7777777778 0.8000000000 -9 4 20 0.8888888889 0.9000000000 -10 4 20 1.0000000000 1.0000000000 -select pk, a, b, +pk percent_rank() over (order by pk) cume_dist() over (order by pk) +1 0.0000000000 0.1000000000 +2 0.1111111111 0.2000000000 +3 0.2222222222 0.3000000000 +4 0.3333333333 0.4000000000 +5 0.4444444444 0.5000000000 +6 0.5555555556 0.6000000000 +7 0.6666666667 0.7000000000 +8 0.7777777778 0.8000000000 +9 0.8888888889 0.9000000000 +10 1.0000000000 1.0000000000 +select a, percent_rank() over (partition by a order by a), cume_dist() over (partition by a order by a) from t1; -pk a b percent_rank() over (partition by a order by a) cume_dist() over (partition by a order by a) -1 0 10 0.0000000000 1.0000000000 -2 0 10 0.0000000000 1.0000000000 -3 1 10 0.0000000000 1.0000000000 -4 1 10 0.0000000000 1.0000000000 -8 2 10 0.0000000000 1.0000000000 -5 2 20 0.0000000000 1.0000000000 -6 2 20 0.0000000000 1.0000000000 -7 2 20 0.0000000000 1.0000000000 -9 4 20 0.0000000000 1.0000000000 -10 4 20 0.0000000000 1.0000000000 +a percent_rank() over (partition by a order by a) cume_dist() over (partition by a order by a) +0 0.0000000000 1.0000000000 +0 0.0000000000 1.0000000000 +1 0.0000000000 1.0000000000 +1 0.0000000000 1.0000000000 +2 0.0000000000 1.0000000000 +2 0.0000000000 1.0000000000 +2 0.0000000000 1.0000000000 +2 0.0000000000 1.0000000000 +4 0.0000000000 1.0000000000 +4 0.0000000000 1.0000000000 drop table t1; diff --git a/mysql-test/r/win_rank.result b/mysql-test/r/win_rank.result index 725683d3869..e74c2edd662 100644 --- a/mysql-test/r/win_rank.result +++ b/mysql-test/r/win_rank.result @@ -17,34 +17,34 @@ insert into t1 values ( 7 , 2, 20), ( 9 , 4, 20), (10 , 4, 20); -select pk, a, b, rank() over (order by a) as rank, +select a, rank() over (order by a) as rank, dense_rank() over (order by a) as dense_rank from t1; -pk a b rank dense_rank -1 0 10 1 1 -2 0 10 1 1 -3 1 10 3 2 -4 1 10 3 2 -8 2 10 5 3 -5 2 20 5 3 -6 2 20 5 3 -7 2 20 5 3 -9 4 20 9 4 -10 4 20 9 4 -select pk, a, b, rank() over (partition by b order by a) as rank, +a rank dense_rank +0 1 1 +0 1 1 +1 3 2 +1 3 2 +2 5 3 +2 5 3 +2 5 3 +2 5 3 +4 9 4 +4 9 4 +select a, b, rank() over (partition by b order by a) as rank, dense_rank() over (partition by b order by a) as dense_rank from t1; -pk a b rank dense_rank -1 0 10 1 1 -2 0 10 1 1 -3 1 10 3 2 -4 1 10 3 2 -8 2 10 5 3 -5 2 20 1 1 -6 2 20 1 1 -7 2 20 1 1 -9 4 20 4 2 -10 4 20 4 2 +a b rank dense_rank +0 10 1 1 +0 10 1 1 +1 10 3 2 +1 10 3 2 +2 10 5 3 +2 20 1 1 +2 20 1 1 +2 20 1 1 +4 20 4 2 +4 20 4 2 drop table t1; # # Test with null values in the table. @@ -59,46 +59,49 @@ insert into t2 values (2,'b'); insert into t2 values (-1,''); select *, rank() over (order by s1) as rank, dense_rank() over (order by s1) as dense_rank -from t2; +from t2 +order by s1, s2; s1 s2 rank dense_rank -1 a 5 3 NULL NULL 1 1 -1 NULL 5 3 NULL a 1 1 NULL c 1 1 -2 b 7 4 -1 4 2 +1 NULL 5 3 +1 a 5 3 +2 b 7 4 select *, rank() over (partition by s2 order by s1) as rank, dense_rank() over (partition by s2 order by s1) as dense_rank -from t2; +from t2 +order by s1, s2; s1 s2 rank dense_rank -1 a 2 2 NULL NULL 1 1 -1 NULL 2 2 NULL a 1 1 NULL c 1 1 -2 b 1 1 -1 1 1 +1 NULL 2 2 +1 a 2 2 +2 b 1 1 select *, rank() over (order by s2) as rank, dense_rank() over (order by s2) as dense_rank -from t2; +from t2 +order by s2, s1; s1 s2 rank dense_rank -1 a 4 3 NULL NULL 1 1 1 NULL 1 1 -NULL a 4 3 -NULL c 7 5 -2 b 6 4 -1 3 2 +NULL a 4 3 +1 a 4 3 +2 b 6 4 +NULL c 7 5 select *, rank() over (partition by s1 order by s2) as rank, dense_rank() over (partition by s1 order by s2) as dense_rank from t2; s1 s2 rank dense_rank -1 a 2 2 NULL NULL 1 1 -1 NULL 1 1 NULL a 2 2 NULL c 3 3 -2 b 1 1 -1 1 1 +1 NULL 1 1 +1 a 2 2 +2 b 1 1 drop table t2; diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test index e24c6533dcd..e9c8ee05773 100644 --- a/mysql-test/t/win.test +++ b/mysql-test/t/win.test @@ -58,7 +58,8 @@ select pk, a, b, row_number() over (order by a), row_number() over (order by b) -from t1; +from t1 +order by b; drop table t1; @@ -391,7 +392,6 @@ insert into t3 values (6, 2); select - pk, val, count(val) over (order by val range between current row and @@ -404,7 +404,6 @@ insert into t3 values (8, 3); select - pk, val, count(val) over (order by val range between current row and @@ -444,7 +443,6 @@ insert into t4 values select part_id, - pk, val, count(val) over (partition by part_id order by val @@ -458,7 +456,6 @@ from t4; --echo # select part_id, - pk, val, count(val) over (partition by part_id order by val @@ -469,7 +466,6 @@ from t4; select part_id, - pk, val, count(val) over (partition by part_id order by val @@ -480,7 +476,6 @@ from t4; select part_id, - pk, val, count(val) over (partition by part_id order by val @@ -723,7 +718,8 @@ select count(b) over (order by a range between 2 preceding and 2 following) as CNT -from t1; +from t1 +order by a, pk; drop table t1; --echo # @@ -1023,6 +1019,7 @@ insert into t1 values --echo # Check using counters flush status; +--sorted_result select rank() over (partition by c order by a), rank() over (partition by c order by b) @@ -1030,6 +1027,7 @@ from t1; show status like '%sort%'; flush status; +--sorted_result select rank() over (partition by c order by a), rank() over (partition by c order by a) @@ -1101,6 +1099,7 @@ insert into t1 values (2,2), (3,1); +--sorted_result select a, b, rank() over (order by a), rank() over (order by b), @@ -1192,6 +1191,7 @@ insert into t1 values (4,'a'); insert into t1 values (2,'b'); insert into t1 values (-1,''); +--sorted_result select *, ROW_NUMBER() OVER (order by s1), @@ -1267,7 +1267,7 @@ from t1; drop table t1; --echo # ---echo # MDEV-10874: two window functions with ccompatible sorting +--echo # MDEV-10874: two window functions with compatible sorting --echo # create table t1 ( @@ -2110,10 +2110,12 @@ SET sql_mode = 'ONLY_FULL_GROUP_BY'; SELECT name, test, score, AVG(score) OVER (PARTITION BY test) AS average_by_test -FROM t1; +FROM t1 +ORDER BY test, name; set @@sql_mode= @save_sql_mode; SELECT name, test, score, AVG(score) OVER (PARTITION BY test) AS average_by_test -FROM t1; +FROM t1 +ORDER BY test, name; drop table t1; diff --git a/mysql-test/t/win_percent_cume.test b/mysql-test/t/win_percent_cume.test index b851185cb32..05ace95d571 100644 --- a/mysql-test/t/win_percent_cume.test +++ b/mysql-test/t/win_percent_cume.test @@ -17,17 +17,17 @@ insert into t1 values ( 9 , 4, 20), (10 , 4, 20); -select pk, a, b, +select a, percent_rank() over (order by a), cume_dist() over (order by a) from t1; -select pk, a, b, +select pk, percent_rank() over (order by pk), cume_dist() over (order by pk) from t1 order by pk; -select pk, a, b, +select a, percent_rank() over (partition by a order by a), cume_dist() over (partition by a order by a) from t1; diff --git a/mysql-test/t/win_rank.test b/mysql-test/t/win_rank.test index eda1f458205..c58ff054117 100644 --- a/mysql-test/t/win_rank.test +++ b/mysql-test/t/win_rank.test @@ -20,11 +20,11 @@ insert into t1 values ( 9 , 4, 20), (10 , 4, 20); -select pk, a, b, rank() over (order by a) as rank, - dense_rank() over (order by a) as dense_rank +select a, rank() over (order by a) as rank, + dense_rank() over (order by a) as dense_rank from t1; -select pk, a, b, rank() over (partition by b order by a) as rank, - dense_rank() over (partition by b order by a) as dense_rank +select a, b, rank() over (partition by b order by a) as rank, + dense_rank() over (partition by b order by a) as dense_rank from t1; drop table t1; @@ -44,13 +44,16 @@ insert into t2 values (-1,''); select *, rank() over (order by s1) as rank, dense_rank() over (order by s1) as dense_rank -from t2; +from t2 +order by s1, s2; select *, rank() over (partition by s2 order by s1) as rank, dense_rank() over (partition by s2 order by s1) as dense_rank -from t2; +from t2 +order by s1, s2; select *, rank() over (order by s2) as rank, dense_rank() over (order by s2) as dense_rank -from t2; +from t2 +order by s2, s1; select *, rank() over (partition by s1 order by s2) as rank, dense_rank() over (partition by s1 order by s2) as dense_rank from t2; diff --git a/mysql-test/t/win_std.test b/mysql-test/t/win_std.test index 5ed999431c0..c1964ecac98 100644 --- a/mysql-test/t/win_std.test +++ b/mysql-test/t/win_std.test @@ -80,9 +80,11 @@ select std(b) over (order by a rows between current row and 0 following) from t2; --echo # Only peers frame. +--sorted_result select a, b, std(b) over (order by a range between 0 preceding and 0 preceding) from t2; +--sorted_result select a, b, std(b) over (order by a range between 0 preceding and current row) from t2; @@ -90,9 +92,11 @@ from t2; select a, b, std(b) over (order by a range between current row and 0 preceding) from t2; +--sorted_result select a, b, std(b) over (order by a range between current row and 0 following) from t2; +--sorted_result select a, b, std(b) over (order by a range between 0 following and 0 following) from t2; diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index aea41886f05..673f9762a86 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -353,6 +353,15 @@ int init_slave_io_cache(IO_CACHE *master, IO_CACHE *slave) void end_slave_io_cache(IO_CACHE *cache) { + /* Remove the cache from the next_file_user circular linked list. */ + if (cache->next_file_user != cache) + { + IO_CACHE *p= cache->next_file_user; + while (p->next_file_user != cache) + p= p->next_file_user; + p->next_file_user= cache->next_file_user; + + } my_free(cache->buffer); } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1fa80da85a6..0cdecf1bf2e 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -26649,9 +26649,10 @@ AGGR_OP::end_send() // Update ref array join_tab->join->set_items_ref_array(*join_tab->ref_array); + bool keep_last_filesort_result = join_tab->filesort ? false : true; if (join_tab->window_funcs_step) { - if (join_tab->window_funcs_step->exec(join)) + if (join_tab->window_funcs_step->exec(join, keep_last_filesort_result)) return NESTED_LOOP_ERROR; } @@ -26705,6 +26706,12 @@ AGGR_OP::end_send() } } + if (keep_last_filesort_result) + { + delete join_tab->filesort_result; + join_tab->filesort_result= NULL; + } + // Finish rnd scn after sending records if (join_tab->table->file->inited) join_tab->table->file->ha_rnd_end(); diff --git a/sql/sql_window.cc b/sql/sql_window.cc index cd09e174808..465c6ae032c 100644 --- a/sql/sql_window.cc +++ b/sql/sql_window.cc @@ -2751,7 +2751,7 @@ bool Window_func_runner::exec(THD *thd, TABLE *tbl, SORT_INFO *filesort_result) } -bool Window_funcs_sort::exec(JOIN *join) +bool Window_funcs_sort::exec(JOIN *join, bool keep_filesort_result) { THD *thd= join->thd; JOIN_TAB *join_tab= join->join_tab + join->total_join_tab_cnt(); @@ -2766,8 +2766,11 @@ bool Window_funcs_sort::exec(JOIN *join) bool is_error= runner.exec(thd, tbl, filesort_result); - delete join_tab->filesort_result; - join_tab->filesort_result= NULL; + if (!keep_filesort_result) + { + delete join_tab->filesort_result; + join_tab->filesort_result= NULL; + } return is_error; } @@ -2876,14 +2879,18 @@ bool Window_funcs_computation::setup(THD *thd, } -bool Window_funcs_computation::exec(JOIN *join) +bool Window_funcs_computation::exec(JOIN *join, bool keep_last_filesort_result) { List_iterator it(win_func_sorts); Window_funcs_sort *srt; + uint counter= 0; /* Count how many sorts we've executed. */ /* Execute each sort */ while ((srt = it++)) { - if (srt->exec(join)) + counter++; + bool keep_filesort_result= keep_last_filesort_result && + counter == win_func_sorts.elements; + if (srt->exec(join, keep_filesort_result)) return true; } return false; diff --git a/sql/sql_window.h b/sql/sql_window.h index 6a56fc84392..e0c1563e5bb 100644 --- a/sql/sql_window.h +++ b/sql/sql_window.h @@ -195,7 +195,7 @@ class Window_funcs_sort : public Sql_alloc public: bool setup(THD *thd, SQL_SELECT *sel, List_iterator &it, st_join_table *join_tab); - bool exec(JOIN *join); + bool exec(JOIN *join, bool keep_filesort_result); void cleanup() { delete filesort; } friend class Window_funcs_computation; @@ -225,7 +225,7 @@ class Window_funcs_computation : public Sql_alloc List win_func_sorts; public: bool setup(THD *thd, List *window_funcs, st_join_table *tab); - bool exec(JOIN *join); + bool exec(JOIN *join, bool keep_last_filesort_result); Explain_aggr_window_funcs *save_explain_plan(MEM_ROOT *mem_root, bool is_analyze); void cleanup(); From f0cb21ea2e00b9f985ed0a76f1e6edbdf9d846cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Nov 2018 12:42:01 +0200 Subject: [PATCH 114/127] Remove dead code is_thd_killed() --- include/mysql/service_kill_statement.h | 4 ++-- storage/xtradb/handler/ha_innodb.cc | 11 +---------- storage/xtradb/handler/ha_innodb.h | 6 +----- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/include/mysql/service_kill_statement.h b/include/mysql/service_kill_statement.h index 995b21f0a9f..bfb222301eb 100644 --- a/include/mysql/service_kill_statement.h +++ b/include/mysql/service_kill_statement.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, Monty Program Ab. +/* Copyright (c) 2013, 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ time-consuming loops, and gracefully abort the operation if it is non-zero. - thd_is_killed(thd) + thd_killed(thd) @return 0 - no KILL statement was issued, continue normally @return 1 - there was a KILL statement, abort the execution. diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 5179b49dc04..e7992c3f161 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2018, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -19611,15 +19611,6 @@ int ha_innobase::multi_range_read_explain_info(uint mrr_mode, char *str, size_t return ds_mrr.dsmrr_explain_info(mrr_mode, str, size); } -/* - A helper function used only in index_cond_func_innodb -*/ - -bool ha_innobase::is_thd_killed() -{ - return thd_kill_level(user_thd); -} - /********************************************************************** Issue a warning that the row is too big. */ UNIV_INTERN diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index cf44e14aff0..e65e0964f9d 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -364,10 +364,6 @@ public: * @return idx_cond if pushed; NULL if not pushed */ class Item* idx_cond_push(uint keyno, class Item* idx_cond); - - /* An helper function for index_cond_func_innodb: */ - bool is_thd_killed(); - private: /** The multi range read session object */ DsMrr_impl ds_mrr; From cfa047069e7466a2ff9c99a266b67360bbdb22ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Nov 2018 14:17:11 +0200 Subject: [PATCH 115/127] Remove an unused declaration --- storage/innobase/handler/ha_innodb.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 48c244eff51..da2cc403796 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -416,9 +416,6 @@ public: Item* idx_cond_push(uint keyno, Item* idx_cond); /* @} */ - /* An helper function for index_cond_func_innodb: */ - bool is_thd_killed(); - protected: /** From 1a89356382cc1b19d878e6a29163eecc8605f414 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Fri, 2 Nov 2018 12:27:38 -0400 Subject: [PATCH 116/127] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 15d82f0dea5..f95ab5017dc 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=37 +MYSQL_VERSION_PATCH=38 From 8a346f31b913daa011085afec2b2d38450c73e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Nov 2018 14:17:19 +0200 Subject: [PATCH 117/127] =?UTF-8?q?MDEV-17073=20INSERT=E2=80=A6ON=20DUPLIC?= =?UTF-8?q?ATE=20KEY=20UPDATE=20became=20more=20deadlock-prone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit thd_rpl_stmt_based(): A new predicate to check if statement-based replication is active. (This can also hold when replication is not in use, but binlog is.) que_thr_stop(), row_ins_duplicate_error_in_clust(), row_ins_sec_index_entry_low(), row_ins(): On a duplicate key error, only lock all index records when statement-based replication is in use. --- .../innodb/include/innodb_binlog.combinations | 3 + .../suite/innodb/include/innodb_binlog.inc | 3 + .../r/auto_increment_dup,skip-log-bin.rdiff | 51 +++++++++++++++++ .../suite/innodb/r/auto_increment_dup.result | 22 +------- .../suite/innodb/t/auto_increment_dup.test | 40 +++++-------- sql/sql_class.cc | 5 ++ storage/innobase/include/ha_prototypes.h | 3 + storage/innobase/que/que0que.cc | 3 +- storage/innobase/row/row0ins.cc | 56 +++++++++---------- 9 files changed, 109 insertions(+), 77 deletions(-) create mode 100644 mysql-test/suite/innodb/include/innodb_binlog.combinations create mode 100644 mysql-test/suite/innodb/include/innodb_binlog.inc create mode 100644 mysql-test/suite/innodb/r/auto_increment_dup,skip-log-bin.rdiff diff --git a/mysql-test/suite/innodb/include/innodb_binlog.combinations b/mysql-test/suite/innodb/include/innodb_binlog.combinations new file mode 100644 index 00000000000..46d31e733b1 --- /dev/null +++ b/mysql-test/suite/innodb/include/innodb_binlog.combinations @@ -0,0 +1,3 @@ +[log-bin] +log-bin +[skip-log-bin] diff --git a/mysql-test/suite/innodb/include/innodb_binlog.inc b/mysql-test/suite/innodb/include/innodb_binlog.inc new file mode 100644 index 00000000000..3f6ece2422e --- /dev/null +++ b/mysql-test/suite/innodb/include/innodb_binlog.inc @@ -0,0 +1,3 @@ +# See innodb_binlog.combinations +# --log-bin is ignored in the embedded server +--source include/not_embedded.inc diff --git a/mysql-test/suite/innodb/r/auto_increment_dup,skip-log-bin.rdiff b/mysql-test/suite/innodb/r/auto_increment_dup,skip-log-bin.rdiff new file mode 100644 index 00000000000..7b4ec54eed8 --- /dev/null +++ b/mysql-test/suite/innodb/r/auto_increment_dup,skip-log-bin.rdiff @@ -0,0 +1,51 @@ +--- auto_increment_dup.result ++++ auto_increment_dup,skip-log-bin.reject +@@ -89,13 +89,14 @@ + SET DEBUG_SYNC='execute_command_after_close_tables SIGNAL continue'; + affected rows: 0 + INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; +-ERROR HY000: Lock wait timeout exceeded; try restarting transaction ++affected rows: 3 ++info: Records: 3 Duplicates: 0 Warnings: 0 + connection con1; + # + # 2 duplicates + # +-affected rows: 3 +-info: Records: 3 Duplicates: 0 Warnings: 0 ++affected rows: 4 ++info: Records: 3 Duplicates: 1 Warnings: 0 + connection default; + # + # 3 rows +@@ -103,19 +104,21 @@ + SELECT * FROM t1 order by k; + id k c + 1 1 NULL +-2 2 NULL +-3 3 NULL +-affected rows: 3 ++4 2 1 ++2 3 NULL ++5 4 NULL ++6 5 NULL ++affected rows: 5 + INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; +-affected rows: 4 +-info: Records: 3 Duplicates: 1 Warnings: 0 ++affected rows: 6 ++info: Records: 3 Duplicates: 3 Warnings: 0 + SELECT * FROM t1 order by k; + id k c + 1 1 NULL +-2 2 2 +-3 3 NULL +-7 4 NULL +-8 5 NULL ++4 2 2 ++2 3 NULL ++5 4 2 ++6 5 2 + affected rows: 5 + disconnect con1; + disconnect con2; diff --git a/mysql-test/suite/innodb/r/auto_increment_dup.result b/mysql-test/suite/innodb/r/auto_increment_dup.result index fa0921b57a5..1467a459fc1 100644 --- a/mysql-test/suite/innodb/r/auto_increment_dup.result +++ b/mysql-test/suite/innodb/r/auto_increment_dup.result @@ -1,4 +1,3 @@ -drop table if exists t1; set global transaction isolation level repeatable read; CREATE TABLE t1( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -79,20 +78,13 @@ affected rows: 0 # # Parallel execution # -connect con1, localhost, root; connect con2, localhost, root; SET DEBUG_SYNC='now WAIT_FOR write_row_done'; -connection con1; -# -# Connection 1 -# +connect con1, localhost, root; SET DEBUG_SYNC='ha_write_row_end SIGNAL write_row_done WAIT_FOR continue'; affected rows: 0 INSERT INTO t1(k) VALUES (1), (2), (3) ON DUPLICATE KEY UPDATE c='1'; connection con2; -# -# Connection 2 -# affected rows: 0 SET DEBUG_SYNC='execute_command_after_close_tables SIGNAL continue'; affected rows: 0 @@ -140,18 +132,10 @@ k INT, c CHAR(1), UNIQUE KEY(k)) ENGINE=InnoDB; connect con1, localhost, root; -connect con2, localhost, root; -connection con1; -# -# Connection 1 -# SET DEBUG_SYNC='ha_write_row_end SIGNAL continue2 WAIT_FOR continue1'; affected rows: 0 INSERT INTO t1(k) VALUES (1), (2), (3) ON DUPLICATE KEY UPDATE c='1'; -connection con2; -# -# Connection 2 -# +connect con2, localhost, root; SET DEBUG_SYNC='ha_write_row_start WAIT_FOR continue2'; affected rows: 0 SET DEBUG_SYNC='after_mysql_insert SIGNAL continue1'; @@ -159,6 +143,7 @@ affected rows: 0 INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; affected rows: 3 info: Records: 3 Duplicates: 0 Warnings: 0 +disconnect con2; connection con1; affected rows: 4 info: Records: 3 Duplicates: 1 Warnings: 0 @@ -174,7 +159,6 @@ id k c 5 4 NULL 6 5 NULL disconnect con1; -disconnect con2; connection default; DROP TABLE t1; set global transaction isolation level repeatable read; diff --git a/mysql-test/suite/innodb/t/auto_increment_dup.test b/mysql-test/suite/innodb/t/auto_increment_dup.test index 45e4559a038..aa399e5966d 100644 --- a/mysql-test/suite/innodb/t/auto_increment_dup.test +++ b/mysql-test/suite/innodb/t/auto_increment_dup.test @@ -4,11 +4,11 @@ ########################################################################## --source include/have_innodb.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +--source include/innodb_binlog.inc ---disable_warnings -drop table if exists t1; ---enable_warnings +let $stmt= `SELECT @@GLOBAL.log_bin`; set global transaction isolation level repeatable read; @@ -69,29 +69,28 @@ CREATE TABLE t1( k INT, c CHAR(1), UNIQUE KEY(k)) ENGINE=InnoDB; - + --echo # --echo # Parallel execution --echo # ---connect(con1, localhost, root) --connect(con2, localhost, root) - --send SET DEBUG_SYNC='now WAIT_FOR write_row_done' ---connection con1 ---echo # ---echo # Connection 1 ---echo # + +--connect(con1, localhost, root) SET DEBUG_SYNC='ha_write_row_end SIGNAL write_row_done WAIT_FOR continue'; --send INSERT INTO t1(k) VALUES (1), (2), (3) ON DUPLICATE KEY UPDATE c='1' --connection con2 ---echo # ---echo # Connection 2 ---echo # --reap + SET DEBUG_SYNC='execute_command_after_close_tables SIGNAL continue'; +if ($stmt) { --error ER_LOCK_WAIT_TIMEOUT INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; +} +if (!$stmt) { +INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; +} --connection con1 --echo # @@ -138,23 +137,14 @@ CREATE TABLE t1( --enable_info --connect(con1, localhost, root) ---connect(con2, localhost, root) - ---connection con1 - ---echo # ---echo # Connection 1 ---echo # SET DEBUG_SYNC='ha_write_row_end SIGNAL continue2 WAIT_FOR continue1'; --send INSERT INTO t1(k) VALUES (1), (2), (3) ON DUPLICATE KEY UPDATE c='1' ---connection con2 ---echo # ---echo # Connection 2 ---echo # +--connect(con2, localhost, root) SET DEBUG_SYNC='ha_write_row_start WAIT_FOR continue2'; SET DEBUG_SYNC='after_mysql_insert SIGNAL continue1'; INSERT INTO t1(k) VALUES (2), (4), (5) ON DUPLICATE KEY UPDATE c='2'; +--disconnect con2 --connection con1 --reap @@ -167,11 +157,9 @@ SET DEBUG_SYNC='RESET'; SELECT * FROM t1 ORDER BY k; --disconnect con1 ---disconnect con2 --connection default DROP TABLE t1; set global transaction isolation level repeatable read; - diff --git a/sql/sql_class.cc b/sql/sql_class.cc index e581ed0af25..71d5b80eaa3 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4523,6 +4523,11 @@ extern "C" int thd_rpl_is_parallel(const MYSQL_THD thd) return thd->rgi_slave && thd->rgi_slave->is_parallel_exec; } +extern "C" int thd_rpl_stmt_based(const MYSQL_THD thd) +{ + return !thd->is_current_stmt_binlog_format_row() && + !thd->is_current_stmt_binlog_disabled(); +} /* Returns high resolution timestamp for the start of the current query. */ diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 86defe9b166..8af4d320997 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -122,6 +122,9 @@ thd_is_replication_slave_thread( /*============================*/ THD* thd); /*!< in: thread handle */ +/** @return whether statement-based replication is active */ +extern "C" int thd_rpl_stmt_based(const THD* thd); + /******************************************************************//** Returns true if the transaction this thread is processing has edited non-transactional tables. Used by the deadlock detector when deciding diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 87d37e347f1..99d8c70a2c0 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -689,7 +689,8 @@ que_thr_stop( trx->lock.wait_thr = thr; thr->state = QUE_THR_LOCK_WAIT; - } else if (trx->duplicates && trx->error_state == DB_DUPLICATE_KEY) { + } else if (trx->duplicates && trx->error_state == DB_DUPLICATE_KEY + && thd_rpl_stmt_based(trx->mysql_thd)) { return(FALSE); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index f72ade422b3..3362e5302b1 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2299,10 +2299,10 @@ row_ins_duplicate_error_in_clust( true, ULINT_UNDEFINED, &heap); - ulint lock_type; - - lock_type = + ulint lock_type = trx->isolation_level <= TRX_ISO_READ_COMMITTED + || (trx->mysql_thd + && !thd_rpl_stmt_based(trx->mysql_thd)) ? LOCK_REC_NOT_GAP : LOCK_ORDINARY; /* We set a lock on the possible duplicate: this @@ -2342,10 +2342,7 @@ row_ins_duplicate_error_in_clust( if (row_ins_dupl_error_with_rec( rec, entry, cursor->index, offsets)) { -duplicate: - trx->error_info = cursor->index; - err = DB_DUPLICATE_KEY; - goto func_exit; + goto duplicate; } } } @@ -2388,7 +2385,10 @@ duplicate: if (row_ins_dupl_error_with_rec( rec, entry, cursor->index, offsets)) { - goto duplicate; +duplicate: + trx->error_info = cursor->index; + err = DB_DUPLICATE_KEY; + goto func_exit; } } @@ -3006,9 +3006,11 @@ row_ins_sec_index_entry_low( if (!(flags & BTR_NO_LOCKING_FLAG) && dict_index_is_unique(index) && thr_get_trx(thr)->duplicates - && thr_get_trx(thr)->isolation_level >= TRX_ISO_REPEATABLE_READ) { + && thr_get_trx(thr)->isolation_level >= TRX_ISO_REPEATABLE_READ + && thd_rpl_stmt_based(thr_get_trx(thr)->mysql_thd)) { - /* When using the REPLACE statement or ON DUPLICATE clause, a + /* In statement-based replication, when replicating a + REPLACE statement or ON DUPLICATE KEY UPDATE clause, a gap lock is taken on the position of the to-be-inserted record, to avoid other concurrent transactions from inserting the same record. */ @@ -3552,14 +3554,15 @@ row_ins( ins_node_t* node, /*!< in: row insert node */ que_thr_t* thr) /*!< in: query thread */ { - dberr_t err; - DBUG_ENTER("row_ins"); DBUG_PRINT("row_ins", ("table: %s", node->table->name.m_name)); + trx_t* trx = thr_get_trx(thr); + if (node->duplicate) { - thr_get_trx(thr)->error_state = DB_DUPLICATE_KEY; + ut_ad(thd_rpl_stmt_based(trx->mysql_thd)); + trx->error_state = DB_DUPLICATE_KEY; } if (node->state == INS_NODE_ALLOC_ROW_ID) { @@ -3585,7 +3588,7 @@ row_ins( while (node->index != NULL) { if (node->index->type != DICT_FTS) { - err = row_ins_index_entry_step(node, thr); + dberr_t err = row_ins_index_entry_step(node, thr); switch (err) { case DB_SUCCESS: @@ -3598,9 +3601,11 @@ row_ins( case DB_DUPLICATE_KEY: ut_ad(dict_index_is_unique(node->index)); - if (thr_get_trx(thr)->isolation_level + if (trx->isolation_level >= TRX_ISO_REPEATABLE_READ - && thr_get_trx(thr)->duplicates) { + && trx->duplicates + && !node->table->is_temporary() + && thd_rpl_stmt_based(trx->mysql_thd)) { /* When we are in REPLACE statement or INSERT .. ON DUPLICATE UPDATE @@ -3663,7 +3668,7 @@ row_ins( /* Save 1st dup error. Ignore subsequent dup errors. */ node->duplicate = node->index; - thr_get_trx(thr)->error_state + trx->error_state = DB_DUPLICATE_KEY; } break; @@ -3674,18 +3679,6 @@ row_ins( } } - if (node->duplicate && dict_table_is_temporary(node->table)) { - ut_ad(thr_get_trx(thr)->error_state - == DB_DUPLICATE_KEY); - /* For TEMPORARY TABLE, we won't lock anything, - so we can simply break here instead of requiring - GAP locks for other unique secondary indexes, - pretending we have consumed all indexes. */ - node->index = NULL; - node->entry = NULL; - break; - } - node->index = dict_table_get_next_index(node->index); node->entry = UT_LIST_GET_NEXT(tuple_list, node->entry); @@ -3704,6 +3697,7 @@ row_ins( insertion will take place. These gap locks are needed only for unique indexes. So skipping non-unique indexes. */ if (node->duplicate) { + ut_ad(thd_rpl_stmt_based(trx->mysql_thd)); while (node->index && !dict_index_is_unique(node->index)) { @@ -3712,13 +3706,13 @@ row_ins( node->entry = UT_LIST_GET_NEXT(tuple_list, node->entry); } - thr_get_trx(thr)->error_state = DB_DUPLICATE_KEY; + trx->error_state = DB_DUPLICATE_KEY; } } ut_ad(node->entry == NULL); - thr_get_trx(thr)->error_info = node->duplicate; + trx->error_info = node->duplicate; node->state = INS_NODE_ALLOC_ROW_ID; DBUG_RETURN(node->duplicate ? DB_DUPLICATE_KEY : DB_SUCCESS); From 6472c5c015a174ffd680fddc8f20c7a94684a552 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sat, 3 Nov 2018 14:24:15 +0400 Subject: [PATCH 118/127] MDEV-15890 Strange error message if you try to FLUSH TABLES after LOCK TABLES . LOCK view WRITE shouldn't block FLUSH view. So we set the view's mdl_request type to it's tables. --- mysql-test/r/flush.result | 1 - mysql-test/r/lock.result | 4 ++-- mysql-test/r/lock_multi.result | 9 ++++----- mysql-test/r/lock_sync.result | 6 +----- .../r/delayed_insert_limit_func.result | 4 ++-- .../r/sql_low_priority_updates_func.result | 4 ++-- .../sys_vars/t/delayed_insert_limit_func.test | 20 ++++++------------- .../t/sql_low_priority_updates_func.test | 4 ++-- mysql-test/t/flush.test | 1 - mysql-test/t/lock.test | 4 ++-- mysql-test/t/lock_multi.test | 14 ++++++++----- mysql-test/t/lock_sync.test | 8 ++------ sql/sql_view.cc | 3 +-- 13 files changed, 33 insertions(+), 49 deletions(-) diff --git a/mysql-test/r/flush.result b/mysql-test/r/flush.result index 2d7b81b1907..4e4aec4b93b 100644 --- a/mysql-test/r/flush.result +++ b/mysql-test/r/flush.result @@ -508,7 +508,6 @@ ERROR HY000: Table 't1' was locked with a READ lock and can't be updated UNLOCK TABLES; LOCK TABLES v1 WRITE; FLUSH TABLES v1; -ERROR HY000: Table 't1' was locked with a READ lock and can't be updated UNLOCK TABLES; LOCK TABLES v1 READ; FLUSH TABLES t1; diff --git a/mysql-test/r/lock.result b/mysql-test/r/lock.result index 0dcc0de828f..b6f19d11cd3 100644 --- a/mysql-test/r/lock.result +++ b/mysql-test/r/lock.result @@ -136,7 +136,7 @@ select * from t1; ERROR HY000: Table 't1' was not locked with LOCK TABLES unlock tables; create or replace view v_bug5719 as select * from t1; -lock tables v_bug5719 write; +lock tables v_bug5719 read; select * from v_bug5719; a @@ -299,7 +299,7 @@ create table t2 (j int); # # Try to perform DDL on table which is locked through view. create view v1 as select * from t2; -lock tables t1 write, v1 write; +lock tables t1 write, v1 read; flush table t2; ERROR HY000: Table 't2' was locked with a READ lock and can't be updated drop table t2; diff --git a/mysql-test/r/lock_multi.result b/mysql-test/r/lock_multi.result index 12960a4f5f6..4df415b06ad 100644 --- a/mysql-test/r/lock_multi.result +++ b/mysql-test/r/lock_multi.result @@ -276,15 +276,14 @@ DROP VIEW IF EXISTS v1; # # Test 1: LOCK TABLES v1 WRITE, t1 READ; # -# Thanks to the fact that we no longer allow DDL on tables -# which are locked for write implicitly, the exact scenario -# in which assert was failing is no longer repeatable. CREATE TABLE t1 ( f1 integer ); CREATE VIEW v1 AS SELECT f1 FROM t1 ; +# Connection 2 LOCK TABLES v1 WRITE, t1 READ; FLUSH TABLE t1; -ERROR HY000: Table 't1' was locked with a READ lock and can't be updated -UNLOCK TABLES; +# Connection 1 +LOCK TABLES t1 WRITE; +FLUSH TABLE t1; DROP TABLE t1; DROP VIEW v1; # diff --git a/mysql-test/r/lock_sync.result b/mysql-test/r/lock_sync.result index 219cc08342e..8143f3fde38 100644 --- a/mysql-test/r/lock_sync.result +++ b/mysql-test/r/lock_sync.result @@ -648,9 +648,6 @@ set debug_sync= 'RESET'; set @old_general_log = @@global.general_log; set @@global.general_log= OFF; create table t1 (i int) engine=InnoDB; -# We have to use view in order to make LOCK TABLES avoid -# acquiring SNRW metadata lock on table. -create view v1 as select * from t1; insert into t1 values (1); # Prepare user lock which will be used for resuming execution of # the first statement after it acquires TL_WRITE_ALLOW_WRITE lock. @@ -673,7 +670,7 @@ select count(*) > 0 from t1 as a, t1 as b for update;; # acquiring lock for the the first instance of 't1'. set debug_sync= 'now WAIT_FOR parked'; # Send LOCK TABLE statement which will try to get TL_WRITE lock on 't1': -lock table v1 write;; +lock table t1 write concurrent;; # Switch to connection 'default'. # Wait until this LOCK TABLES statement starts waiting for table lock. # Allow SELECT ... FOR UPDATE to resume. @@ -703,7 +700,6 @@ unlock tables; # Do clean-up. set debug_sync= 'RESET'; set @@global.general_log= @old_general_log; -drop view v1; drop table t1; # # Bug#50821 Deadlock between LOCK TABLES and ALTER TABLE diff --git a/mysql-test/suite/sys_vars/r/delayed_insert_limit_func.result b/mysql-test/suite/sys_vars/r/delayed_insert_limit_func.result index eeb7a28c816..6076d024d44 100644 --- a/mysql-test/suite/sys_vars/r/delayed_insert_limit_func.result +++ b/mysql-test/suite/sys_vars/r/delayed_insert_limit_func.result @@ -13,7 +13,7 @@ INSERT INTO t1 VALUES('3','1','1'); INSERT INTO t1 VALUES('4','1','1'); INSERT INTO t1 VALUES('5','1','1'); INSERT INTO t1 VALUES('6','1','1'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; ** Connection con1 ** INSERT DELAYED INTO t1 VALUES('7','1','1'); INSERT DELAYED INTO t1 VALUES('8','1','1'); @@ -82,7 +82,7 @@ INSERT INTO t1 VALUES('3'); INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; ** Connection con1 ** Asynchronous execute INSERT DELAYED INTO t1 VALUES('7'); diff --git a/mysql-test/suite/sys_vars/r/sql_low_priority_updates_func.result b/mysql-test/suite/sys_vars/r/sql_low_priority_updates_func.result index fe76c2c4b96..e9758e238af 100644 --- a/mysql-test/suite/sys_vars/r/sql_low_priority_updates_func.result +++ b/mysql-test/suite/sys_vars/r/sql_low_priority_updates_func.result @@ -20,7 +20,7 @@ INSERT INTO t1 VALUES('3'); INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 WRITE CONCURRENT; ** Connection con1 ** ** Asynchronous Execution ** UPDATE t1 SET a = CONCAT(a,"-updated");| @@ -56,7 +56,7 @@ INSERT INTO t1 VALUES('3'); INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; ** Connection con1 ** ** Asynchronous Execution ** UPDATE t1 SET a = CONCAT(a,"-updated");| diff --git a/mysql-test/suite/sys_vars/t/delayed_insert_limit_func.test b/mysql-test/suite/sys_vars/t/delayed_insert_limit_func.test index 427f2730e47..e83c48b67f1 100644 --- a/mysql-test/suite/sys_vars/t/delayed_insert_limit_func.test +++ b/mysql-test/suite/sys_vars/t/delayed_insert_limit_func.test @@ -61,7 +61,7 @@ INSERT INTO t1 VALUES('4','1','1'); INSERT INTO t1 VALUES('5','1','1'); INSERT INTO t1 VALUES('6','1','1'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; --echo ** Connection con1 ** connection con1; @@ -110,9 +110,8 @@ delimiter ;| --echo ** Connection con0 ** connection con0; let $wait_condition= - SELECT variable_value > @@global.delayed_insert_limit - FROM information_schema.global_status - WHERE variable_name like 'Not_flushed_delayed_rows'; + SELECT COUNT(*) = 1 FROM information_schema.processlist + WHERE state = 'Waiting for table level lock' AND user='delayed'; --source include/wait_condition.inc let $my_select= SELECT COUNT(*) FROM t1; send; @@ -137,13 +136,6 @@ connection con0; --echo Asynchronous "reap" result --echo The next result suffers from --echo '# Bug#35386 insert delayed inserts 1 + limit rows instead of just limit rows' -# -# on UNLOCK TABLES both SELECT in the con0 and delayed insert thread in the -# con1 were awaken. There's no FIFO for TL_WRITE_DELAYED and TL_READ, -# so either the first delayed_insert_limit rows will be inserted -# before select (which will see 21 row) or select will go first (and see 6 rows) -# ---replace_result 6 21 reap; --echo ** Connection default ** @@ -173,7 +165,7 @@ INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; --echo ** Connection con1 ** connection con1; @@ -204,8 +196,8 @@ delimiter ;| --echo ** Connection con0 ** connection con0; let $wait_condition= - SELECT variable_value > 0 FROM information_schema.global_status - WHERE variable_name like 'Not_flushed_delayed_rows'; + SELECT COUNT(*) = 1 FROM information_schema.processlist + WHERE state = 'Waiting for table level lock' AND user='delayed'; --source include/wait_condition.inc --echo Asynchronous execute # Due to performance and server behaveiour the test observes values between 6 and 22. diff --git a/mysql-test/suite/sys_vars/t/sql_low_priority_updates_func.test b/mysql-test/suite/sys_vars/t/sql_low_priority_updates_func.test index ba13558a135..5d7d6cbe734 100644 --- a/mysql-test/suite/sys_vars/t/sql_low_priority_updates_func.test +++ b/mysql-test/suite/sys_vars/t/sql_low_priority_updates_func.test @@ -70,7 +70,7 @@ INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 WRITE CONCURRENT; --echo ** Connection con1 ** connection con1; @@ -144,7 +144,7 @@ INSERT INTO t1 VALUES('4'); INSERT INTO t1 VALUES('5'); INSERT INTO t1 VALUES('6'); -LOCK TABLE v1 WRITE; +LOCK TABLE v1 READ; --echo ** Connection con1 ** connection con1; diff --git a/mysql-test/t/flush.test b/mysql-test/t/flush.test index 7736574b7df..11a1ac2a8c2 100644 --- a/mysql-test/t/flush.test +++ b/mysql-test/t/flush.test @@ -724,7 +724,6 @@ FLUSH TABLES v1; UNLOCK TABLES; LOCK TABLES v1 WRITE; ---error ER_TABLE_NOT_LOCKED_FOR_WRITE FLUSH TABLES v1; UNLOCK TABLES; diff --git a/mysql-test/t/lock.test b/mysql-test/t/lock.test index 6cfaf9fc320..734fcd8e626 100644 --- a/mysql-test/t/lock.test +++ b/mysql-test/t/lock.test @@ -192,7 +192,7 @@ drop view v_bug5719; select * from t1; unlock tables; create or replace view v_bug5719 as select * from t1; -lock tables v_bug5719 write; +lock tables v_bug5719 read; select * from v_bug5719; --echo --echo Allowed to use an underlying table under LOCK TABLES @@ -370,7 +370,7 @@ create table t2 (j int); --echo # --echo # Try to perform DDL on table which is locked through view. create view v1 as select * from t2; -lock tables t1 write, v1 write; +lock tables t1 write, v1 read; --error ER_TABLE_NOT_LOCKED_FOR_WRITE flush table t2; --error ER_TABLE_NOT_LOCKED_FOR_WRITE diff --git a/mysql-test/t/lock_multi.test b/mysql-test/t/lock_multi.test index efd0fe5d630..ee73c2e3585 100644 --- a/mysql-test/t/lock_multi.test +++ b/mysql-test/t/lock_multi.test @@ -771,17 +771,21 @@ DROP VIEW IF EXISTS v1; --echo # --echo # Test 1: LOCK TABLES v1 WRITE, t1 READ; --echo # ---echo # Thanks to the fact that we no longer allow DDL on tables ---echo # which are locked for write implicitly, the exact scenario ---echo # in which assert was failing is no longer repeatable. CREATE TABLE t1 ( f1 integer ); CREATE VIEW v1 AS SELECT f1 FROM t1 ; +--echo # Connection 2 +connect (con2,localhost,root); LOCK TABLES v1 WRITE, t1 READ; ---error ER_TABLE_NOT_LOCKED_FOR_WRITE FLUSH TABLE t1; -UNLOCK TABLES; +disconnect con2; +--source include/wait_until_disconnected.inc + +--echo # Connection 1 +connection default; +LOCK TABLES t1 WRITE; +FLUSH TABLE t1; # Assertion happened here # Cleanup DROP TABLE t1; diff --git a/mysql-test/t/lock_sync.test b/mysql-test/t/lock_sync.test index ef79cc2c0f4..0e0aa8f64c7 100644 --- a/mysql-test/t/lock_sync.test +++ b/mysql-test/t/lock_sync.test @@ -909,9 +909,6 @@ set @old_general_log = @@global.general_log; set @@global.general_log= OFF; create table t1 (i int) engine=InnoDB; ---echo # We have to use view in order to make LOCK TABLES avoid ---echo # acquiring SNRW metadata lock on table. -create view v1 as select * from t1; insert into t1 values (1); --echo # Prepare user lock which will be used for resuming execution of --echo # the first statement after it acquires TL_WRITE_ALLOW_WRITE lock. @@ -942,14 +939,14 @@ connection con_bug45143_3; --echo # acquiring lock for the the first instance of 't1'. set debug_sync= 'now WAIT_FOR parked'; --echo # Send LOCK TABLE statement which will try to get TL_WRITE lock on 't1': ---send lock table v1 write; +--send lock table t1 write concurrent; --echo # Switch to connection 'default'. connection default; --echo # Wait until this LOCK TABLES statement starts waiting for table lock. let $wait_condition= select count(*)= 1 from information_schema.processlist where state= 'Waiting for table level lock' and - info='lock table v1 write'; + info='lock table t1 write concurrent'; --source include/wait_condition.inc --echo # Allow SELECT ... FOR UPDATE to resume. --echo # Since it already has TL_WRITE_ALLOW_WRITE lock on the first instance @@ -993,7 +990,6 @@ disconnect con_bug45143_2; disconnect con_bug45143_3; set debug_sync= 'RESET'; set @@global.general_log= @old_general_log; -drop view v1; drop table t1; diff --git a/sql/sql_view.cc b/sql/sql_view.cc index b7ff4513a32..9a9309a133b 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -1534,8 +1534,7 @@ bool mysql_make_view(THD *thd, TABLE_SHARE *share, TABLE_LIST *table, for (tbl= view_main_select_tables; tbl; tbl= tbl->next_local) { tbl->lock_type= table->lock_type; - tbl->mdl_request.set_type((tbl->lock_type >= TL_WRITE_ALLOW_WRITE) ? - MDL_SHARED_WRITE : MDL_SHARED_READ); + tbl->mdl_request.set_type(table->mdl_request.type); } /* If the view is mergeable, we might want to From af9649c722810eb1754953eb406a84ec876ce693 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 3 Nov 2018 21:17:17 +0100 Subject: [PATCH 119/127] MDEV-17349 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed on concurrent SELECT and DELETE after RENAME from table with index on virtual column Race condition. field->flags were copied from s->field->flags during field->clone(), early in open_table_from_share(). But s->field->flags were getting their PART_INDIRECT_KEY_FLAG bit much later in TABLE::mark_columns_used_by_virtual_fields() and only once per share. If two threads were executing the code between field->clone() and mark_columns_used_by_virtual_fields() at the same time, only one would get PART_INDIRECT_KEY_FLAG bits in field[]. --- mysql-test/suite/vcol/r/races.result | 16 ++++++++++++++++ mysql-test/suite/vcol/t/races.test | 22 ++++++++++++++++++++++ sql/table.cc | 18 ++++++++++++++++-- sql/table.h | 4 +++- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/vcol/r/races.result create mode 100644 mysql-test/suite/vcol/t/races.test diff --git a/mysql-test/suite/vcol/r/races.result b/mysql-test/suite/vcol/r/races.result new file mode 100644 index 00000000000..c46ed5ba2ef --- /dev/null +++ b/mysql-test/suite/vcol/r/races.result @@ -0,0 +1,16 @@ +create table t1 (f text, vf tinytext as (f), key (vf(64))) engine=innodb; +insert t1 (f) values ('foo'); +flush tables; +connect con1,localhost,root,,test; +set debug_sync='TABLE_after_field_clone WAIT_FOR go'; +delete from t1; +connection default; +select * from t1; +f vf +foo foo +set debug_sync='now SIGNAL go'; +connection con1; +disconnect con1; +connection default; +drop table t1; +set debug_sync='reset'; diff --git a/mysql-test/suite/vcol/t/races.test b/mysql-test/suite/vcol/t/races.test new file mode 100644 index 00000000000..1bf4e43dec9 --- /dev/null +++ b/mysql-test/suite/vcol/t/races.test @@ -0,0 +1,22 @@ +# +# MDEV-17349 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed on concurrent SELECT and DELETE after RENAME from table with index on virtual column +# +source include/have_innodb.inc; +source include/have_debug_sync.inc; +create table t1 (f text, vf tinytext as (f), key (vf(64))) engine=innodb; +insert t1 (f) values ('foo'); +flush tables; +connect con1,localhost,root,,test; +set debug_sync='TABLE_after_field_clone WAIT_FOR go'; +send delete from t1; +connection default; +let $wait_condition= select state like 'debug sync point%' from information_schema.processlist; +source include/wait_condition.inc; +select * from t1; +set debug_sync='now SIGNAL go'; +connection con1; +reap; +disconnect con1; +connection default; +drop table t1; +set debug_sync='reset'; diff --git a/sql/table.cc b/sql/table.cc index f557799e7dd..2b278288784 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3013,6 +3013,7 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, uchar *record, *bitmaps; Field **field_ptr; uint8 save_context_analysis_only= thd->lex->context_analysis_only; + TABLE_SHARE::enum_v_keys check_set_initialized= share->check_set_initialized; DBUG_ENTER("open_table_from_share"); DBUG_PRINT("enter",("name: '%s.%s' form: %p", share->db.str, share->table_name.str, outparam)); @@ -3116,6 +3117,8 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, } (*field_ptr)= 0; // End marker + DEBUG_SYNC(thd, "TABLE_after_field_clone"); + if (share->found_next_number_field) outparam->found_next_number_field= outparam->field[(uint) (share->found_next_number_field - share->field)]; @@ -3371,6 +3374,16 @@ partititon_err: } outparam->mark_columns_used_by_virtual_fields(); + if (!check_set_initialized && + share->check_set_initialized == TABLE_SHARE::V_KEYS) + { + // copy PART_INDIRECT_KEY_FLAG that was set meanwhile by *some* thread + for (uint i= 0 ; i < share->fields ; i++) + { + if (share->field[i]->flags & PART_INDIRECT_KEY_FLAG) + outparam->field[i]->flags|= PART_INDIRECT_KEY_FLAG; + } + } if (share->table_category == TABLE_CATEGORY_LOG) { @@ -6708,6 +6721,7 @@ void TABLE::mark_columns_used_by_virtual_fields(void) { MY_BITMAP *save_read_set; Field **vfield_ptr; + TABLE_SHARE::enum_v_keys v_keys= TABLE_SHARE::NO_V_KEYS; /* If there is virtual fields are already initialized */ if (s->check_set_initialized) @@ -6748,12 +6762,12 @@ void TABLE::mark_columns_used_by_virtual_fields(void) if (bitmap_is_set(&tmp_set, i)) { s->field[i]->flags|= PART_INDIRECT_KEY_FLAG; - field[i]->flags|= PART_INDIRECT_KEY_FLAG; + v_keys= TABLE_SHARE::V_KEYS; } } bitmap_clear_all(&tmp_set); } - s->check_set_initialized= 1; + s->check_set_initialized= v_keys; if (s->tmp_table == NO_TMP_TABLE) mysql_mutex_unlock(&s->LOCK_share); } diff --git a/sql/table.h b/sql/table.h index 33b921cd946..4cd5c3ba5f3 100644 --- a/sql/table.h +++ b/sql/table.h @@ -695,6 +695,9 @@ struct TABLE_SHARE uint column_bitmap_size; uchar frm_version; + enum enum_v_keys { NOT_INITIALIZED=0, NO_V_KEYS, V_KEYS }; + enum_v_keys check_set_initialized; + bool use_ext_keys; /* Extended keys can be used */ bool null_field_first; bool system; /* Set if system table (one record) */ @@ -705,7 +708,6 @@ struct TABLE_SHARE bool table_creation_was_logged; bool non_determinstic_insert; bool vcols_need_refixing; - bool check_set_initialized; bool has_update_default_function; ulong table_map_id; /* for row-based replication */ From 03977e8273cbd33c3cbfec191ceee856f973ce1a Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Thu, 25 Oct 2018 21:36:24 +0300 Subject: [PATCH 120/127] MDEV-13671 InnoDB should use case-insensitive column name comparisons like the rest of the server Problem affects INPLACE ALTER rename columns. innobase_rename_column_try(): some strcmp() was replaced with my_strcasecmp(), queries to update data dictionary was updated to not match column name case. --- mysql-test/suite/innodb/r/innodb-alter.result | 138 +++++++++++++++++- mysql-test/suite/innodb/t/innodb-alter.test | 101 ++++++++++++- storage/innobase/dict/dict0mem.cc | 4 +- storage/innobase/handler/handler0alter.cc | 30 ++-- storage/xtradb/dict/dict0mem.cc | 4 +- storage/xtradb/handler/handler0alter.cc | 30 ++-- 6 files changed, 261 insertions(+), 46 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-alter.result b/mysql-test/suite/innodb/r/innodb-alter.result index bd82cc8a764..2f9b832328b 100644 --- a/mysql-test/suite/innodb/r/innodb-alter.result +++ b/mysql-test/suite/innodb/r/innodb-alter.result @@ -714,6 +714,7 @@ t2 CREATE TABLE `t2` ( CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`c2`) REFERENCES `t1` (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ALTER TABLE t1 CHANGE COLUMN c1 C1 INT; +ALTER TABLE t2 CHANGE COLUMN c2 C2 INT; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -723,24 +724,147 @@ t1 CREATE TABLE `t1` ( SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( - `c2` int(11) NOT NULL, - KEY `c2` (`c2`), + `C2` int(11) DEFAULT NULL, + KEY `c2` (`C2`), CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`c2`) REFERENCES `t1` (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 +ALTER TABLE t1 CHANGE COLUMN C1 c5 INT; +ALTER TABLE t2 CHANGE COLUMN C2 c6 INT; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `C1` int(11) NOT NULL DEFAULT '0', - PRIMARY KEY (`C1`) + `c5` int(11) NOT NULL DEFAULT '0', + PRIMARY KEY (`c5`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( - `c2` int(11) NOT NULL, - KEY `c2` (`c2`), - CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`c2`) REFERENCES `t1` (`c1`) + `c6` int(11) DEFAULT NULL, + KEY `c2` (`c6`), + CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`c6`) REFERENCES `t1` (`c5`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID +WHERE T.NAME='test/t1'; +NAME +c5 +SELECT F.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_FIELDS F INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_INDEXES I ON F.INDEX_ID=I.INDEX_ID INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON I.TABLE_ID=T.TABLE_ID +WHERE T.NAME='test/t1' AND I.NAME='PRIMARY'; +NAME +c5 +SELECT C.REF_COL_NAME, C.FOR_COL_NAME FROM INFORMATION_SCHEMA.INNODB_SYS_FOREIGN_COLS C INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_FOREIGN F ON C.ID=F.ID +WHERE F.FOR_NAME='test/t2'; +REF_COL_NAME FOR_COL_NAME +c5 c6 DROP TABLE t2, t1; +# virtual columns case too +CREATE TABLE t1 (a INT, b INT GENERATED ALWAYS AS (a) VIRTUAL) ENGINE = InnoDB; +ALTER TABLE t1 CHANGE COLUMN a A INT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `A` int(11) DEFAULT NULL, + `b` int(11) AS (a) VIRTUAL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID +WHERE T.NAME='test/t1'; +NAME +a +DROP TABLE t1; +# different FOREIGN KEY cases +CREATE TABLE t1 ( +a INT UNIQUE KEY, +b INT UNIQUE KEY, +c INT UNIQUE KEY, +d INT UNIQUE KEY +) ENGINE=INNODB; +CREATE TABLE t2 ( +aa INT, +bb INT, +cc INT, +dd INT +) ENGINE=INNODB; +INSERT INTO t1 VALUES (1, 1, 1, 1); +INSERT INTO t2 VALUES (1, 1, 1, 1); +ALTER TABLE t1 CHANGE a A INT, ALGORITHM=INPLACE; +ALTER TABLE t1 CHANGE c C INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE cc CC INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE dd DD INT, ALGORITHM=INPLACE; +SET foreign_key_checks=0; +ALTER TABLE t2 +ADD FOREIGN KEY(aa) REFERENCES t1(a), +ADD FOREIGN KEY(bb) REFERENCES t1(b), +ADD FOREIGN KEY(cc) REFERENCES t1(c), +ADD FOREIGN KEY(dd) REFERENCES t1(d), +ALGORITHM=INPLACE; +ALTER TABLE t1 CHANGE b B INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE aa AA INT, ALGORITHM=INPLACE; +ALTER TABLE t1 CHANGE d D INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE bb BB INT, ALGORITHM=INPLACE; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `A` int(11) DEFAULT NULL, + `B` int(11) DEFAULT NULL, + `C` int(11) DEFAULT NULL, + `D` int(11) DEFAULT NULL, + UNIQUE KEY `a` (`A`), + UNIQUE KEY `b` (`B`), + UNIQUE KEY `c` (`C`), + UNIQUE KEY `d` (`D`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `AA` int(11) DEFAULT NULL, + `BB` int(11) DEFAULT NULL, + `CC` int(11) DEFAULT NULL, + `DD` int(11) DEFAULT NULL, + KEY `aa` (`AA`), + KEY `bb` (`BB`), + KEY `CC` (`CC`), + KEY `DD` (`DD`), + CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`), + CONSTRAINT `t2_ibfk_2` FOREIGN KEY (`bb`) REFERENCES `t1` (`b`), + CONSTRAINT `t2_ibfk_3` FOREIGN KEY (`cc`) REFERENCES `t1` (`c`), + CONSTRAINT `t2_ibfk_4` FOREIGN KEY (`dd`) REFERENCES `t1` (`d`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +DELETE FROM t1 WHERE a=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE A=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE b=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE B=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE c=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE C=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE d=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DELETE FROM t1 WHERE D=1; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (`test`.`t2`, CONSTRAINT `t2_ibfk_1` FOREIGN KEY (`aa`) REFERENCES `t1` (`a`)) +DROP TABLE t2, t1; +# virtual columns case too +CREATE TABLE t1 (a INT, b INT GENERATED ALWAYS AS (a) VIRTUAL) ENGINE = InnoDB; +ALTER TABLE t1 CHANGE COLUMN a A INT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `A` int(11) DEFAULT NULL, + `b` int(11) AS (a) VIRTUAL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN +INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID +WHERE T.NAME='test/t1'; +NAME +a +DROP TABLE t1; # # BUG 20029625 - HANDLE_FATAL_SIGNAL (SIG=11) IN # DICT_MEM_TABLE_COL_RENAME_LOW diff --git a/mysql-test/suite/innodb/t/innodb-alter.test b/mysql-test/suite/innodb/t/innodb-alter.test index 0d3cc6f1733..30e3292ec10 100644 --- a/mysql-test/suite/innodb/t/innodb-alter.test +++ b/mysql-test/suite/innodb/t/innodb-alter.test @@ -419,14 +419,109 @@ CREATE TABLE t2(c2 INT NOT NULL, FOREIGN KEY(c2) REFERENCES t1(c1))ENGINE=INNODB SHOW CREATE TABLE t1; SHOW CREATE TABLE t2; ALTER TABLE t1 CHANGE COLUMN c1 C1 INT; +ALTER TABLE t2 CHANGE COLUMN c2 C2 INT; SHOW CREATE TABLE t1; SHOW CREATE TABLE t2; -# FIXME: MDEV-13671 InnoDB should use case-insensitive column name comparisons -# like the rest of the server -#ALTER TABLE t1 CHANGE COLUMN C1 c5 INT; +ALTER TABLE t1 CHANGE COLUMN C1 c5 INT; +ALTER TABLE t2 CHANGE COLUMN C2 c6 INT; SHOW CREATE TABLE t1; SHOW CREATE TABLE t2; + +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID + WHERE T.NAME='test/t1'; + +SELECT F.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_FIELDS F INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_INDEXES I ON F.INDEX_ID=I.INDEX_ID INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON I.TABLE_ID=T.TABLE_ID + WHERE T.NAME='test/t1' AND I.NAME='PRIMARY'; + +SELECT C.REF_COL_NAME, C.FOR_COL_NAME FROM INFORMATION_SCHEMA.INNODB_SYS_FOREIGN_COLS C INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_FOREIGN F ON C.ID=F.ID + WHERE F.FOR_NAME='test/t2'; + DROP TABLE t2, t1; +--echo # virtual columns case too +CREATE TABLE t1 (a INT, b INT GENERATED ALWAYS AS (a) VIRTUAL) ENGINE = InnoDB; +ALTER TABLE t1 CHANGE COLUMN a A INT; +SHOW CREATE TABLE t1; +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID + WHERE T.NAME='test/t1'; +DROP TABLE t1; + + +--echo # different FOREIGN KEY cases +CREATE TABLE t1 ( + a INT UNIQUE KEY, + b INT UNIQUE KEY, + c INT UNIQUE KEY, + d INT UNIQUE KEY +) ENGINE=INNODB; +CREATE TABLE t2 ( + aa INT, + bb INT, + cc INT, + dd INT +) ENGINE=INNODB; + +INSERT INTO t1 VALUES (1, 1, 1, 1); +INSERT INTO t2 VALUES (1, 1, 1, 1); + +ALTER TABLE t1 CHANGE a A INT, ALGORITHM=INPLACE; +ALTER TABLE t1 CHANGE c C INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE cc CC INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE dd DD INT, ALGORITHM=INPLACE; + +SET foreign_key_checks=0; +ALTER TABLE t2 + ADD FOREIGN KEY(aa) REFERENCES t1(a), + ADD FOREIGN KEY(bb) REFERENCES t1(b), + ADD FOREIGN KEY(cc) REFERENCES t1(c), + ADD FOREIGN KEY(dd) REFERENCES t1(d), + ALGORITHM=INPLACE; + +ALTER TABLE t1 CHANGE b B INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE aa AA INT, ALGORITHM=INPLACE; + +--source include/restart_mysqld.inc + +ALTER TABLE t1 CHANGE d D INT, ALGORITHM=INPLACE; +ALTER TABLE t2 CHANGE bb BB INT, ALGORITHM=INPLACE; +SHOW CREATE TABLE t1; +SHOW CREATE TABLE t2; + +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE a=1; +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE A=1; + +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE b=1; +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE B=1; + +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE c=1; +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE C=1; + +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE d=1; +--error ER_ROW_IS_REFERENCED_2 +DELETE FROM t1 WHERE D=1; + +DROP TABLE t2, t1; + +--echo # virtual columns case too +CREATE TABLE t1 (a INT, b INT GENERATED ALWAYS AS (a) VIRTUAL) ENGINE = InnoDB; +ALTER TABLE t1 CHANGE COLUMN a A INT; +SHOW CREATE TABLE t1; +SELECT C.NAME FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS C INNER JOIN + INFORMATION_SCHEMA.INNODB_SYS_TABLES T ON C.TABLE_ID=T.TABLE_ID + WHERE T.NAME='test/t1'; +DROP TABLE t1; + --echo # --echo # BUG 20029625 - HANDLE_FATAL_SIGNAL (SIG=11) IN diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 81a4cd9c6c6..c182aaba676 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -498,9 +498,7 @@ dict_mem_table_col_rename( s += len + 1; } - /* This could fail if the data dictionaries are out of sync. - Proceed with the renaming anyway. */ - ut_ad(!strcmp(from, s)); + ut_ad(!my_strcasecmp(system_charset_info, from, s)); dict_mem_table_col_rename_low(table, nth_col, to, s); } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 8f95772268c..6f43e921099 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -4514,7 +4514,6 @@ innobase_rename_column_try( pars_info_add_ull_literal(info, "tableid", user_table->id); pars_info_add_int4_literal(info, "nth", nth_col); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); trx->op_info = "renaming column in SYS_COLUMNS"; @@ -4524,7 +4523,7 @@ innobase_rename_column_try( "PROCEDURE RENAME_SYS_COLUMNS_PROC () IS\n" "BEGIN\n" "UPDATE SYS_COLUMNS SET NAME=:new\n" - "WHERE TABLE_ID=:tableid AND NAME=:old\n" + "WHERE TABLE_ID=:tableid\n" "AND POS=:nth;\n" "END;\n", FALSE, trx); @@ -4548,8 +4547,10 @@ err_exit: index = dict_table_get_next_index(index)) { for (ulint i = 0; i < dict_index_get_n_fields(index); i++) { - if (strcmp(dict_index_get_nth_field(index, i)->name, - from)) { + if (my_strcasecmp( + system_charset_info, + dict_index_get_nth_field(index, i)->name, + from)) { continue; } @@ -4557,7 +4558,6 @@ err_exit: pars_info_add_ull_literal(info, "indexid", index->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4566,14 +4566,14 @@ err_exit: "BEGIN\n" "UPDATE SYS_FIELDS SET COL_NAME=:new\n" - "WHERE INDEX_ID=:indexid AND COL_NAME=:old\n" + "WHERE INDEX_ID=:indexid\n" "AND POS=:nth;\n" /* Try again, in case there is a prefix_len encoded in SYS_FIELDS.POS */ "UPDATE SYS_FIELDS SET COL_NAME=:new\n" - "WHERE INDEX_ID=:indexid AND COL_NAME=:old\n" + "WHERE INDEX_ID=:indexid\n" "AND POS>=65536*:nth AND POS<65536*(:nth+1);\n" "END;\n", @@ -4599,7 +4599,9 @@ rename_foreign: foreign_modified = false; for (unsigned i = 0; i < foreign->n_fields; i++) { - if (strcmp(foreign->foreign_col_names[i], from)) { + if (my_strcasecmp(system_charset_info, + foreign->foreign_col_names[i], + from)) { continue; } @@ -4607,7 +4609,6 @@ rename_foreign: pars_info_add_str_literal(info, "id", foreign->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4616,8 +4617,7 @@ rename_foreign: "BEGIN\n" "UPDATE SYS_FOREIGN_COLS\n" "SET FOR_COL_NAME=:new\n" - "WHERE ID=:id AND POS=:nth\n" - "AND FOR_COL_NAME=:old;\n" + "WHERE ID=:id AND POS=:nth;\n" "END;\n", FALSE, trx); @@ -4641,7 +4641,9 @@ rename_foreign: dict_foreign_t* foreign = *it; for (unsigned i = 0; i < foreign->n_fields; i++) { - if (strcmp(foreign->referenced_col_names[i], from)) { + if (my_strcasecmp(system_charset_info, + foreign->referenced_col_names[i], + from)) { continue; } @@ -4649,7 +4651,6 @@ rename_foreign: pars_info_add_str_literal(info, "id", foreign->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4658,8 +4659,7 @@ rename_foreign: "BEGIN\n" "UPDATE SYS_FOREIGN_COLS\n" "SET REF_COL_NAME=:new\n" - "WHERE ID=:id AND POS=:nth\n" - "AND REF_COL_NAME=:old;\n" + "WHERE ID=:id AND POS=:nth;\n" "END;\n", FALSE, trx); diff --git a/storage/xtradb/dict/dict0mem.cc b/storage/xtradb/dict/dict0mem.cc index 12cbd2f4a97..fb6895867fe 100644 --- a/storage/xtradb/dict/dict0mem.cc +++ b/storage/xtradb/dict/dict0mem.cc @@ -499,9 +499,7 @@ dict_mem_table_col_rename( s += len + 1; } - /* This could fail if the data dictionaries are out of sync. - Proceed with the renaming anyway. */ - ut_ad(!strcmp(from, s)); + ut_ad(!my_strcasecmp(system_charset_info, from, s)); dict_mem_table_col_rename_low(table, nth_col, to, s); } diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 8235263b305..0b7588918bc 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -4516,7 +4516,6 @@ innobase_rename_column_try( pars_info_add_ull_literal(info, "tableid", user_table->id); pars_info_add_int4_literal(info, "nth", nth_col); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); trx->op_info = "renaming column in SYS_COLUMNS"; @@ -4526,7 +4525,7 @@ innobase_rename_column_try( "PROCEDURE RENAME_SYS_COLUMNS_PROC () IS\n" "BEGIN\n" "UPDATE SYS_COLUMNS SET NAME=:new\n" - "WHERE TABLE_ID=:tableid AND NAME=:old\n" + "WHERE TABLE_ID=:tableid\n" "AND POS=:nth;\n" "END;\n", FALSE, trx); @@ -4550,8 +4549,10 @@ err_exit: index = dict_table_get_next_index(index)) { for (ulint i = 0; i < dict_index_get_n_fields(index); i++) { - if (strcmp(dict_index_get_nth_field(index, i)->name, - from)) { + if (my_strcasecmp( + system_charset_info, + dict_index_get_nth_field(index, i)->name, + from)) { continue; } @@ -4559,7 +4560,6 @@ err_exit: pars_info_add_ull_literal(info, "indexid", index->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4568,14 +4568,14 @@ err_exit: "BEGIN\n" "UPDATE SYS_FIELDS SET COL_NAME=:new\n" - "WHERE INDEX_ID=:indexid AND COL_NAME=:old\n" + "WHERE INDEX_ID=:indexid\n" "AND POS=:nth;\n" /* Try again, in case there is a prefix_len encoded in SYS_FIELDS.POS */ "UPDATE SYS_FIELDS SET COL_NAME=:new\n" - "WHERE INDEX_ID=:indexid AND COL_NAME=:old\n" + "WHERE INDEX_ID=:indexid\n" "AND POS>=65536*:nth AND POS<65536*(:nth+1);\n" "END;\n", @@ -4601,7 +4601,9 @@ rename_foreign: foreign_modified = false; for (unsigned i = 0; i < foreign->n_fields; i++) { - if (strcmp(foreign->foreign_col_names[i], from)) { + if (my_strcasecmp(system_charset_info, + foreign->foreign_col_names[i], + from)) { continue; } @@ -4609,7 +4611,6 @@ rename_foreign: pars_info_add_str_literal(info, "id", foreign->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4618,8 +4619,7 @@ rename_foreign: "BEGIN\n" "UPDATE SYS_FOREIGN_COLS\n" "SET FOR_COL_NAME=:new\n" - "WHERE ID=:id AND POS=:nth\n" - "AND FOR_COL_NAME=:old;\n" + "WHERE ID=:id AND POS=:nth;\n" "END;\n", FALSE, trx); @@ -4643,7 +4643,9 @@ rename_foreign: dict_foreign_t* foreign = *it; for (unsigned i = 0; i < foreign->n_fields; i++) { - if (strcmp(foreign->referenced_col_names[i], from)) { + if (my_strcasecmp(system_charset_info, + foreign->referenced_col_names[i], + from)) { continue; } @@ -4651,7 +4653,6 @@ rename_foreign: pars_info_add_str_literal(info, "id", foreign->id); pars_info_add_int4_literal(info, "nth", i); - pars_info_add_str_literal(info, "old", from); pars_info_add_str_literal(info, "new", to); error = que_eval_sql( @@ -4660,8 +4661,7 @@ rename_foreign: "BEGIN\n" "UPDATE SYS_FOREIGN_COLS\n" "SET REF_COL_NAME=:new\n" - "WHERE ID=:id AND POS=:nth\n" - "AND REF_COL_NAME=:old;\n" + "WHERE ID=:id AND POS=:nth;\n" "END;\n", FALSE, trx); From db55b39fb29979acd3c58581e34f8e51507a1792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 5 Nov 2018 16:47:14 +0200 Subject: [PATCH 121/127] Revert some InnoDB/XtraDB changes The relevant InnoDB/XtraDB fixes up to 5.6.42 had already been applied to MariaDB in commit 30c3d6db328d73f1b30be537aceb55d37936fdb9. Revert some changes that appeared in the merge commit 87d852f102a7f733a774252d9a446377ac51a976. --- storage/innobase/dict/dict0dict.cc | 7 +-- storage/innobase/fts/fts0fts.cc | 20 +++---- storage/innobase/handler/ha_innodb.cc | 66 ++++++++++------------- storage/innobase/handler/handler0alter.cc | 33 +++--------- storage/innobase/include/os0file.h | 6 +-- storage/innobase/row/row0merge.cc | 5 +- storage/innobase/row/row0sel.cc | 4 +- storage/xtradb/dict/dict0dict.cc | 7 +-- storage/xtradb/fts/fts0fts.cc | 20 +++---- storage/xtradb/fts/fts0pars.cc | 4 +- storage/xtradb/fts/fts0pars.y | 4 +- storage/xtradb/handler/ha_innodb.cc | 57 +++++++++----------- storage/xtradb/include/data0type.ic | 4 +- storage/xtradb/include/dict0mem.h | 3 +- storage/xtradb/row/row0import.cc | 9 +--- storage/xtradb/row/row0sel.cc | 5 +- 16 files changed, 102 insertions(+), 152 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 6c5df41e90d..7575a4aed62 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. Copyright (c) 2014, 2017, MariaDB Corporation. @@ -3370,10 +3370,7 @@ dict_foreign_find_index( table, col_names, columns, n_cols, index, types_idx, check_charsets, check_null, - error, err_col_no,err_index) - && (!(index->online_status == - ONLINE_INDEX_ABORTED_DROPPED - ||index->online_status == ONLINE_INDEX_ABORTED))) { + error, err_col_no,err_index)) { if (error) { *error = DB_SUCCESS; } diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index d4ca9cb3660..4891e572741 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -870,19 +870,19 @@ fts_drop_index( err = fts_drop_index_tables(trx, index); while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } - fts_free(table); + fts_free(table); return(err); } while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } current_doc_id = table->fts->cache->next_doc_id; first_doc_id = table->fts->cache->first_doc_id; @@ -901,9 +901,9 @@ fts_drop_index( if (index_cache != NULL) { while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } if (index_cache->words) { fts_words_free(index_cache->words); rbt_free(index_cache->words); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c775a89918c..2092cd113a5 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2015,11 +2015,6 @@ innobase_get_lower_case_table_names(void) { return(lower_case_table_names); } -/** return one of the tmpdir path -@return tmpdir path*/ -UNIV_INTERN -char* -innobase_mysql_tmpdir(void) { return (mysql_tmpdir); } /** Create a temporary file in the location specified by the parameter path. If the path is null, then it will be created in tmpdir. @@ -10908,36 +10903,35 @@ innobase_rename_table( row_mysql_lock_data_dictionary(trx); - dict_table_t* table = NULL; - table = dict_table_open_on_name(norm_from, TRUE, FALSE, - DICT_ERR_IGNORE_NONE); + dict_table_t* table = dict_table_open_on_name(norm_from, TRUE, FALSE, + DICT_ERR_IGNORE_NONE); - /* Since DICT_BG_YIELD has sleep for 250 milliseconds, + /* Since DICT_BG_YIELD has sleep for 250 milliseconds, Convert lock_wait_timeout unit from second to 250 milliseconds */ - long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; - if (table != NULL) { - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; - index = dict_table_get_next_index(index)) { + long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; + if (table != NULL) { + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { - if (index->type & DICT_FTS) { - /* Found */ - while (index->index_fts_syncing - && !trx_is_interrupted(trx) - && (lock_wait_timeout--) > 0) { - DICT_BG_YIELD(trx); - } - } - } - dict_table_close(table, TRUE, FALSE); - } + if (index->type & DICT_FTS) { + /* Found */ + while (index->index_fts_syncing + && !trx_is_interrupted(trx) + && (lock_wait_timeout--) > 0) { + DICT_BG_YIELD(trx); + } + } + } + dict_table_close(table, TRUE, FALSE); + } - /* FTS sync is in progress. We shall timeout this operation */ - if (lock_wait_timeout < 0) { - error = DB_LOCK_WAIT_TIMEOUT; - row_mysql_unlock_data_dictionary(trx); - DBUG_RETURN(error); - } + /* FTS sync is in progress. We shall timeout this operation */ + if (lock_wait_timeout < 0) { + error = DB_LOCK_WAIT_TIMEOUT; + row_mysql_unlock_data_dictionary(trx); + DBUG_RETURN(error); + } /* Transaction must be flagged as a locking transaction or it hasn't been started yet. */ @@ -11092,14 +11086,12 @@ ha_innobase::rename_table( my_error(ER_TABLE_EXISTS_ERROR, MYF(0), to); error = DB_ERROR; + } else if (error == DB_LOCK_WAIT_TIMEOUT) { + my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to); + + error = DB_LOCK_WAIT; } - else if (error == DB_LOCK_WAIT_TIMEOUT) { - my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to); - - error = DB_LOCK_WAIT; - } - DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 6f43e921099..367601e1b33 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -4173,23 +4173,11 @@ oom: table. Either way, we should be seeing and reporting a bogus duplicate key error. */ dup_key = NULL; - } else if (prebuilt->trx->error_key_num == 0) { + } else { + DBUG_ASSERT(prebuilt->trx->error_key_num + < ha_alter_info->key_count); dup_key = &ha_alter_info->key_info_buffer[ prebuilt->trx->error_key_num]; - } else { - /* Check if there is generated cluster index column */ - if (ctx->num_to_add_index > ha_alter_info->key_count) { - DBUG_ASSERT(prebuilt->trx->error_key_num - <= ha_alter_info->key_count); - dup_key = &ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num - 1]; - } - else { - DBUG_ASSERT(prebuilt->trx->error_key_num - < ha_alter_info->key_count); - dup_key = &ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num]; - } } print_keydup_error(altered_table, dup_key, MYF(0)); break; @@ -5117,18 +5105,9 @@ commit_try_rebuild( FTS_DOC_ID. */ dup_key = NULL; } else { - if (ctx->num_to_add_index > ha_alter_info->key_count) { - DBUG_ASSERT(err_key <= - ha_alter_info->key_count); - dup_key = &ha_alter_info - ->key_info_buffer[err_key - 1]; - } - else { - DBUG_ASSERT(err_key < - ha_alter_info->key_count); - dup_key = &ha_alter_info - ->key_info_buffer[err_key]; - } + DBUG_ASSERT(err_key < ha_alter_info->key_count); + dup_key = &ha_alter_info + ->key_info_buffer[err_key]; } print_keydup_error(altered_table, dup_key, MYF(0)); diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index e73aeca6e5a..a78e3d64773 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1,6 +1,6 @@ /*********************************************************************** -Copyright (c) 1995, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. Copyright (c) 2013, 2017, MariaDB Corporation. @@ -1314,10 +1314,6 @@ os_file_get_status( file can be opened in RW mode */ #if !defined(UNIV_HOTBACKUP) - -/** return one of the tmpdir path - @return tmpdir path*/ -char *innobase_mysql_tmpdir(void); /** Create a temporary file in the location specified by the parameter path. If the path is null, then it will be created in tmpdir. @param[in] path location for creating temporary file diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 5d3e2d2cca6..ec9b8f79e49 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2005, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -3138,10 +3138,9 @@ row_merge_file_create_low( file APIs, add instrumentation to register with performance schema */ struct PSI_file_locker* locker = NULL; - PSI_file_locker_state state; if (!path) { - path = innobase_mysql_tmpdir(); + path = mysql_tmpdir; } static const char label[] = "/Innodb Merge Temp File"; char* name = static_cast( diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 9426953d173..c2954742f8e 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2015, 2018, MariaDB Corporation. @@ -4550,7 +4550,7 @@ no_gap_lock: prebuilt->new_rec_locks = 1; } err = DB_SUCCESS; - break; + /* fall through */ case DB_SUCCESS: break; case DB_LOCK_WAIT: diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 04ead82dffa..b76414eeea4 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. Copyright (c) 2014, 2017, MariaDB Corporation. @@ -3387,10 +3387,7 @@ dict_foreign_find_index( table, col_names, columns, n_cols, index, types_idx, check_charsets, check_null, - error, err_col_no,err_index) - && (!(index->online_status == - ONLINE_INDEX_ABORTED_DROPPED - ||index->online_status == ONLINE_INDEX_ABORTED))) { + error, err_col_no,err_index)) { if (error) { *error = DB_SUCCESS; } diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 9e038c2edd5..e2a479bf0ae 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -870,19 +870,19 @@ fts_drop_index( err = fts_drop_index_tables(trx, index); while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } - fts_free(table); + fts_free(table); return(err); } while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } current_doc_id = table->fts->cache->next_doc_id; first_doc_id = table->fts->cache->first_doc_id; @@ -901,9 +901,9 @@ fts_drop_index( if (index_cache != NULL) { while (index->index_fts_syncing - && !trx_is_interrupted(trx)) { - DICT_BG_YIELD(trx); - } + && !trx_is_interrupted(trx)) { + DICT_BG_YIELD(trx); + } if (index_cache->words) { fts_words_free(index_cache->words); rbt_free(index_cache->words); diff --git a/storage/xtradb/fts/fts0pars.cc b/storage/xtradb/fts/fts0pars.cc index b7fef4ea8ce..19917ccd26a 100644 --- a/storage/xtradb/fts/fts0pars.cc +++ b/storage/xtradb/fts/fts0pars.cc @@ -106,8 +106,8 @@ extern int ftserror(const char* p); typedef int (*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner); struct fts_lexer_t { - fts_scanner scanner; - void* yyscanner; + fts_scanner scanner; + void* yyscanner; }; diff --git a/storage/xtradb/fts/fts0pars.y b/storage/xtradb/fts/fts0pars.y index 36dae9f7ceb..65c4189eece 100644 --- a/storage/xtradb/fts/fts0pars.y +++ b/storage/xtradb/fts/fts0pars.y @@ -52,8 +52,8 @@ extern int ftserror(const char* p); typedef int (*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner); struct fts_lexer_struct { - fts_scanner scanner; - void* yyscanner; + fts_scanner scanner; + void* yyscanner; }; %} diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index e7992c3f161..127309a6b65 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11721,36 +11721,35 @@ innobase_rename_table( row_mysql_lock_data_dictionary(trx); - dict_table_t* table = NULL; - table = dict_table_open_on_name(norm_from, TRUE, FALSE, - DICT_ERR_IGNORE_NONE); + dict_table_t* table = dict_table_open_on_name(norm_from, TRUE, FALSE, + DICT_ERR_IGNORE_NONE); - /* Since DICT_BG_YIELD has sleep for 250 milliseconds, + /* Since DICT_BG_YIELD has sleep for 250 milliseconds, Convert lock_wait_timeout unit from second to 250 milliseconds */ - long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; - if (table != NULL) { - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; - index = dict_table_get_next_index(index)) { + long int lock_wait_timeout = thd_lock_wait_timeout(thd) * 4; + if (table != NULL) { + for (dict_index_t* index = dict_table_get_first_index(table); + index != NULL; + index = dict_table_get_next_index(index)) { - if (index->type & DICT_FTS) { - /* Found */ - while (index->index_fts_syncing - && !trx_is_interrupted(trx) - && (lock_wait_timeout--) > 0) { - DICT_BG_YIELD(trx); - } - } - } - dict_table_close(table, TRUE, FALSE); - } + if (index->type & DICT_FTS) { + /* Found */ + while (index->index_fts_syncing + && !trx_is_interrupted(trx) + && (lock_wait_timeout--) > 0) { + DICT_BG_YIELD(trx); + } + } + } + dict_table_close(table, TRUE, FALSE); + } - /* FTS sync is in progress. We shall timeout this operation */ - if (lock_wait_timeout < 0) { - error = DB_LOCK_WAIT_TIMEOUT; - row_mysql_unlock_data_dictionary(trx); - DBUG_RETURN(error); - } + /* FTS sync is in progress. We shall timeout this operation */ + if (lock_wait_timeout < 0) { + error = DB_LOCK_WAIT_TIMEOUT; + row_mysql_unlock_data_dictionary(trx); + DBUG_RETURN(error); + } /* Transaction must be flagged as a locking transaction or it hasn't been started yet. */ @@ -11916,12 +11915,6 @@ ha_innobase::rename_table( error = DB_LOCK_WAIT; } - else if (error == DB_LOCK_WAIT_TIMEOUT) { - my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0), to); - - error = DB_LOCK_WAIT; - } - DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); } diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 63a0785744d..a7e2eb0682c 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -525,7 +525,7 @@ dtype_get_fixed_size_low( return(0); } #endif /* UNIV_DEBUG */ - // fallthrough + /* fall through */ case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: @@ -603,7 +603,7 @@ dtype_get_min_size_low( return(0); } #endif /* UNIV_DEBUG */ - // fallthrough + /* fall through */ case DATA_CHAR: case DATA_FIXBINARY: case DATA_INT: diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 95f9ea0f787..e6c0aa1f252 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -292,7 +292,8 @@ dict_mem_table_add_col( const char* name, /*!< in: column name, or NULL */ ulint mtype, /*!< in: main datatype */ ulint prtype, /*!< in: precise type */ - ulint len); /*!< in: precision */ + ulint len) /*!< in: precision */ + MY_ATTRIBUTE((nonnull(1))); /**********************************************************************//** Renames a column of a table in the data dictionary cache. */ UNIV_INTERN diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index f3163a63754..5c42ab6f2bb 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2012, 2018, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -1969,11 +1969,7 @@ PageConverter::update_index_page( return(DB_SUCCESS); } - if (!page_is_leaf(block->frame)) { - return (DB_SUCCESS); - } - - return(update_records(block)); + return page_is_leaf(block->frame) ? update_records(block) : DB_SUCCESS; } /** @@ -3834,4 +3830,3 @@ row_import_for_mysql( return(row_import_cleanup(prebuilt, trx, err)); } - diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index ce5d74b5a3b..daef291248b 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2736,7 +2736,8 @@ row_sel_field_store_in_mysql_format_func( case DATA_SYS: /* These column types should never be shipped to MySQL. */ ut_ad(0); - break; + /* fall through */ + case DATA_CHAR: case DATA_FIXBINARY: case DATA_FLOAT: @@ -4566,7 +4567,7 @@ no_gap_lock: prebuilt->new_rec_locks = 1; } err = DB_SUCCESS; - break; + /* fall through */ case DB_SUCCESS: break; case DB_LOCK_WAIT: From 03680a9b4fda9fa15675e137d46521628553c0eb Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 22 Oct 2018 15:14:43 +0200 Subject: [PATCH 122/127] MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins --- mysql-test/main/join_nested_jcl6.result | 2 +- mysql-test/main/join_outer.result | 21 ++++ mysql-test/main/join_outer.test | 20 ++++ mysql-test/main/join_outer_jcl6.result | 21 ++++ sql/sql_select.cc | 151 ++++++++++++++++++------ 5 files changed, 181 insertions(+), 34 deletions(-) diff --git a/mysql-test/main/join_nested_jcl6.result b/mysql-test/main/join_nested_jcl6.result index eb59531b7d2..2f8e1712672 100644 --- a/mysql-test/main/join_nested_jcl6.result +++ b/mysql-test/main/join_nested_jcl6.result @@ -2055,7 +2055,7 @@ ON (t5.b=t8.b); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t5 ALL NULL NULL NULL NULL 2 1 SIMPLE t6 ALL NULL NULL NULL NULL 1 Using join buffer (flat, BNL join) -1 SIMPLE t7 ref b_i b_i 5 const 0 Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan +1 SIMPLE t7 ref b_i b_i 5 const 1 Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan 1 SIMPLE t8 ALL b_i NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b FROM t5 LEFT JOIN diff --git a/mysql-test/main/join_outer.result b/mysql-test/main/join_outer.result index 9245111369f..5a123826d79 100644 --- a/mysql-test/main/join_outer.result +++ b/mysql-test/main/join_outer.result @@ -2511,4 +2511,25 @@ ORDER BY tb1.i1; v2 DROP TABLE t1,t2; # end of 5.5 tests +# +# MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins +# +create table t1(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2(a int); +insert into t2 values (0),(1); +create table t3 (a int, b int, key(a)); +insert into t3 select A.a + B.a* 10 + C.a * 100, 12345 from t1 A, t1 B, t1 C; +# Uses range for table t3: +explain select * from t1 left join t3 on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 +1 SIMPLE t3 range a a 5 NULL 5 Using where +# This must use range for table t3, too: +explain select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 +1 SIMPLE t3 range a a 5 NULL 5 Using where +drop table t1,t2,t3; SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_outer.test b/mysql-test/main/join_outer.test index 6d20c089bd9..28f544dec7d 100644 --- a/mysql-test/main/join_outer.test +++ b/mysql-test/main/join_outer.test @@ -2042,4 +2042,24 @@ DROP TABLE t1,t2; --echo # end of 5.5 tests +--echo # +--echo # MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins +--echo # +create table t1(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t2(a int); +insert into t2 values (0),(1); + +create table t3 (a int, b int, key(a)); +insert into t3 select A.a + B.a* 10 + C.a * 100, 12345 from t1 A, t1 B, t1 C; + +--echo # Uses range for table t3: +explain select * from t1 left join t3 on t1.a=t3.b and t3.a<5; + +--echo # This must use range for table t3, too: +explain select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; + +drop table t1,t2,t3; + SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_outer_jcl6.result b/mysql-test/main/join_outer_jcl6.result index cb582b1399f..842ab19e20a 100644 --- a/mysql-test/main/join_outer_jcl6.result +++ b/mysql-test/main/join_outer_jcl6.result @@ -2522,6 +2522,27 @@ ORDER BY tb1.i1; v2 DROP TABLE t1,t2; # end of 5.5 tests +# +# MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins +# +create table t1(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t2(a int); +insert into t2 values (0),(1); +create table t3 (a int, b int, key(a)); +insert into t3 select A.a + B.a* 10 + C.a * 100, 12345 from t1 A, t1 B, t1 C; +# Uses range for table t3: +explain select * from t1 left join t3 on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 +1 SIMPLE t3 range a a 5 NULL 5 Using where; Rowid-ordered scan; Using join buffer (flat, BNL join) +# This must use range for table t3, too: +explain select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 Using join buffer (flat, BNL join) +1 SIMPLE t3 range a a 5 NULL 5 Using where; Rowid-ordered scan; Using join buffer (incremental, BNL join) +drop table t1,t2,t3; SET optimizer_switch=@save_optimizer_switch; set join_cache_level=default; show variables like 'join_cache_level'; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index a3003285f87..91d9326854b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -4300,6 +4300,40 @@ struct SARGABLE_PARAM }; +/* + Mark all tables inside a join nest as constant. + + @detail This is called when there is a local "Impossible WHERE" inside + a multi-table LEFT JOIN. +*/ + +void mark_join_nest_as_const(JOIN *join, + TABLE_LIST *join_nest, + table_map *found_const_table_map, + uint *const_count) +{ + List_iterator it(join_nest->nested_join->join_list); + TABLE_LIST *tbl; + while ((tbl= it++)) + { + if (tbl->nested_join) + { + mark_join_nest_as_const(join, tbl, found_const_table_map, const_count); + continue; + } + JOIN_TAB *tab= tbl->table->reginfo.join_tab; + + tab->type= JT_CONST; + tab->info= ET_IMPOSSIBLE_ON_CONDITION; + tab->table->const_table= 1; + + join->const_table_map|= tab->table->map; + *found_const_table_map|= tab->table->map; + set_position(join,(*const_count)++,tab,(KEYUSE*) 0); + mark_as_null_row(tab->table); // All fields are NULL + } +} + /** Calculate the best possible join and initialize the join structure. @@ -4871,39 +4905,80 @@ make_join_statistics(JOIN *join, List &tables_list, /* Perform range analysis if there are keys it could use (1). - Don't do range analysis if we're on the inner side of an outer join (2). - Do range analysis if we're on the inner side of a semi-join (3). - Don't do range analysis for materialized subqueries (4). - Don't do range analysis for materialized derived tables (5) + Don't do range analysis for materialized subqueries (2). + Don't do range analysis for materialized derived tables (3) */ if ((!s->const_keys.is_clear_all() || !bitmap_is_clear_all(&s->table->cond_set)) && // (1) - (!s->table->pos_in_table_list->embedding || // (2) - (s->table->pos_in_table_list->embedding && // (3) - s->table->pos_in_table_list->embedding->sj_on_expr)) && // (3) - !s->table->is_filled_at_execution() && // (4) - !(s->table->pos_in_table_list->derived && // (5) - s->table->pos_in_table_list->is_materialized_derived())) // (5) + !s->table->is_filled_at_execution() && // (2) + !(s->table->pos_in_table_list->derived && // (3) + s->table->pos_in_table_list->is_materialized_derived())) // (3) { bool impossible_range= FALSE; ha_rows records= HA_POS_ERROR; SQL_SELECT *select= 0; if (!s->const_keys.is_clear_all()) { + Item *sargable_cond; + int cond_source; + /* + Figure out which condition we should use for range analysis. For + INNER JOIN, we use the WHERE, for inner side of LEFT JOIN we should + use the ON expression. + */ + if (*s->on_expr_ref) + { + /* + This is an inner table from a single-table LEFT JOIN, "t1 LEFT JOIN + t2 ON cond". Use the condition cond. + */ + cond_source= 0; + sargable_cond= *s->on_expr_ref; + } + else if (s->table->pos_in_table_list->embedding && + !s->table->pos_in_table_list->embedding->sj_on_expr) + { + /* + This is the inner side of a multi-table outer join. Use the + appropriate ON expression. + */ + cond_source= 1; + sargable_cond= s->table->pos_in_table_list->embedding->on_expr; + } + else + { + /* The table is not inner wrt some LEFT JOIN. Use the WHERE clause */ + cond_source= 2; + sargable_cond= join->conds; + } + select= make_select(s->table, found_const_table_map, found_const_table_map, - *s->on_expr_ref ? *s->on_expr_ref : join->conds, + sargable_cond, (SORT_INFO*) 0, 1, &error); if (!select) goto error; records= get_quick_record_count(join->thd, select, s->table, &s->const_keys, join->row_limit); - /* Range analyzer could modify the condition. */ - if (*s->on_expr_ref) - *s->on_expr_ref= select->cond; - else - join->conds= select->cond; + + /* + Range analyzer might have modified the condition. Put it the new + condition to where we got it from. + */ + switch (cond_source) { + case 0: + *s->on_expr_ref= select->cond; + break; + case 1: + s->table->pos_in_table_list->embedding->on_expr= select->cond; + break; + case 2: + join->conds= select->cond; + break; + default: + DBUG_ASSERT(0); + } s->quick=select->quick; s->needed_reg=select->needed_reg; @@ -4924,23 +4999,33 @@ make_join_statistics(JOIN *join, List &tables_list, } if (impossible_range) { - /* - Impossible WHERE or ON expression - In case of ON, we mark that the we match one empty NULL row. - In case of WHERE, don't set found_const_table_map to get the - caller to abort with a zero row result. - */ - join->const_table_map|= s->table->map; - set_position(join,const_count++,s,(KEYUSE*) 0); - s->type= JT_CONST; - s->table->const_table= 1; - if (*s->on_expr_ref) - { - /* Generate empty row */ - s->info= ET_IMPOSSIBLE_ON_CONDITION; - found_const_table_map|= s->table->map; - mark_as_null_row(s->table); // All fields are NULL - } + /* + Impossible WHERE or ON expression + In case of ON, we mark that the we match one empty NULL row. + In case of WHERE, don't set found_const_table_map to get the + caller to abort with a zero row result. + */ + TABLE_LIST *emb= s->table->pos_in_table_list->embedding; + if (emb && !emb->sj_on_expr) + { + /* Mark all tables in a multi-table join nest as const */ + mark_join_nest_as_const(join, emb, &found_const_table_map, + &const_count); + } + else + { + join->const_table_map|= s->table->map; + set_position(join,const_count++,s,(KEYUSE*) 0); + s->type= JT_CONST; + s->table->const_table= 1; + if (*s->on_expr_ref) + { + /* Generate empty row */ + s->info= ET_IMPOSSIBLE_ON_CONDITION; + found_const_table_map|= s->table->map; + mark_as_null_row(s->table); // All fields are NULL + } + } } if (records != HA_POS_ERROR) { From 3b6d90385230be911e15b9aa93a76e26367bc4bc Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 23 Oct 2018 11:23:34 +0200 Subject: [PATCH 123/127] MDEV-17493: Partition pruning doesn't work for nested outer joins Reuse the fix for MDEV-17518 here, too. --- mysql-test/main/partition_pruning.result | 33 +++++++ mysql-test/main/partition_pruning.test | 31 +++++++ sql/sql_select.cc | 109 +++++++++++------------ sql/sql_select.h | 8 +- 4 files changed, 120 insertions(+), 61 deletions(-) diff --git a/mysql-test/main/partition_pruning.result b/mysql-test/main/partition_pruning.result index 422132dd1c3..00bd24101d6 100644 --- a/mysql-test/main/partition_pruning.result +++ b/mysql-test/main/partition_pruning.result @@ -3484,3 +3484,36 @@ select * from t1 where (a = 1 AND b < 'd' AND (c = 'b' OR (c = 'c' AND d = 1)) O a b c d 1 a b 1 drop table t1; +# +# MDEV-17493: Partition pruning doesn't work for nested outer joins +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int, c int); +insert into t1 select a,a,a from t0; +create table t2 (a int, b int, c int); +insert into t2 select a,a,a from t0; +create table t3 ( +part_id int, +a int +) partition by list (part_id) ( +partition p0 values in (0), +partition p1 values in (1), +partition p2 values in (2), +partition p3 values in (3), +partition p4 values in (4) +); +insert into t3 select mod(a,5), a from t0; +explain partitions +select * from t1 left join t3 on (t1.a=t3.a and t3.part_id=3); +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 10 +1 SIMPLE t3 p3 ALL NULL NULL NULL NULL 2 Using where; Using join buffer (flat, BNL join) +# The following should have partitions="p3", NOT p0,p1,p2,p3,p4: +explain partitions +select * from t1 left join (t3 join t2) on (t1.a=t3.a and t3.a=t2.b and t3.part_id=3); +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 NULL ALL NULL NULL NULL NULL 10 +1 SIMPLE t3 p3 ALL NULL NULL NULL NULL 2 Using where; Using join buffer (flat, BNL join) +1 SIMPLE t2 NULL ALL NULL NULL NULL NULL 10 Using where; Using join buffer (incremental, BNL join) +drop table t0,t1,t2,t3; diff --git a/mysql-test/main/partition_pruning.test b/mysql-test/main/partition_pruning.test index 9d72e9c0d01..2879b0eae6c 100644 --- a/mysql-test/main/partition_pruning.test +++ b/mysql-test/main/partition_pruning.test @@ -1535,4 +1535,35 @@ select * from t1 where (a = 1 AND b < 'd' AND (c = 'b' OR (c = 'c' AND d = 1)) O (a = 1 AND b >= 'a' AND (c = 'c' OR (c = 'd' AND d = 2)))); drop table t1; +--echo # +--echo # MDEV-17493: Partition pruning doesn't work for nested outer joins +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int, c int); +insert into t1 select a,a,a from t0; +create table t2 (a int, b int, c int); +insert into t2 select a,a,a from t0; + +create table t3 ( + part_id int, + a int +) partition by list (part_id) ( + partition p0 values in (0), + partition p1 values in (1), + partition p2 values in (2), + partition p3 values in (3), + partition p4 values in (4) +); +insert into t3 select mod(a,5), a from t0; + +explain partitions +select * from t1 left join t3 on (t1.a=t3.a and t3.part_id=3); + +--echo # The following should have partitions="p3", NOT p0,p1,p2,p3,p4: +explain partitions +select * from t1 left join (t3 join t2) on (t1.a=t3.a and t3.a=t2.b and t3.part_id=3); + +drop table t0,t1,t2,t3; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 91d9326854b..08e9b8daf73 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -292,6 +292,9 @@ static bool find_order_in_list(THD *, Ref_ptr_array, TABLE_LIST *, ORDER *, static double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, table_map rem_tables); void set_postjoin_aggr_write_func(JOIN_TAB *tab); + +static Item **get_sargable_cond(JOIN *join, TABLE *table); + #ifndef DBUG_OFF /* @@ -1770,19 +1773,9 @@ JOIN::optimize_inner() List_iterator_fast li(select_lex->leaf_tables); while ((tbl= li++)) { - /* - If tbl->embedding!=NULL that means that this table is in the inner - part of the nested outer join, and we can't do partition pruning - (TODO: check if this limitation can be lifted) - */ - if (!tbl->embedding || - (tbl->embedding && tbl->embedding->sj_on_expr)) - { - Item *prune_cond= tbl->on_expr? tbl->on_expr : conds; - tbl->table->all_partitions_pruned_away= prune_partitions(thd, - tbl->table, - prune_cond); - } + Item **prune_cond= get_sargable_cond(this, tbl->table); + tbl->table->all_partitions_pruned_away= + prune_partitions(thd, tbl->table, *prune_cond); } } #endif @@ -4334,6 +4327,47 @@ void mark_join_nest_as_const(JOIN *join, } } + +/* + @brief Get the condition that can be used to do range analysis/partition + pruning/etc + + @detail + Figure out which condition we can use: + - For INNER JOIN, we use the WHERE, + - "t1 LEFT JOIN t2 ON ..." uses t2's ON expression + - "t1 LEFT JOIN (...) ON ..." uses the join nest's ON expression. +*/ + +static Item **get_sargable_cond(JOIN *join, TABLE *table) +{ + Item **retval; + if (table->pos_in_table_list->on_expr) + { + /* + This is an inner table from a single-table LEFT JOIN, "t1 LEFT JOIN + t2 ON cond". Use the condition cond. + */ + retval= &table->pos_in_table_list->on_expr; + } + else if (table->pos_in_table_list->embedding && + !table->pos_in_table_list->embedding->sj_on_expr) + { + /* + This is the inner side of a multi-table outer join. Use the + appropriate ON expression. + */ + retval= &(table->pos_in_table_list->embedding->on_expr); + } + else + { + /* The table is not inner wrt some LEFT JOIN. Use the WHERE clause */ + retval= &join->conds; + } + return retval; +} + + /** Calculate the best possible join and initialize the join structure. @@ -4919,42 +4953,11 @@ make_join_statistics(JOIN *join, List &tables_list, SQL_SELECT *select= 0; if (!s->const_keys.is_clear_all()) { - Item *sargable_cond; - int cond_source; - /* - Figure out which condition we should use for range analysis. For - INNER JOIN, we use the WHERE, for inner side of LEFT JOIN we should - use the ON expression. - */ - if (*s->on_expr_ref) - { - /* - This is an inner table from a single-table LEFT JOIN, "t1 LEFT JOIN - t2 ON cond". Use the condition cond. - */ - cond_source= 0; - sargable_cond= *s->on_expr_ref; - } - else if (s->table->pos_in_table_list->embedding && - !s->table->pos_in_table_list->embedding->sj_on_expr) - { - /* - This is the inner side of a multi-table outer join. Use the - appropriate ON expression. - */ - cond_source= 1; - sargable_cond= s->table->pos_in_table_list->embedding->on_expr; - } - else - { - /* The table is not inner wrt some LEFT JOIN. Use the WHERE clause */ - cond_source= 2; - sargable_cond= join->conds; - } + Item **sargable_cond= get_sargable_cond(join, s->table); select= make_select(s->table, found_const_table_map, found_const_table_map, - sargable_cond, + *sargable_cond, (SORT_INFO*) 0, 1, &error); if (!select) @@ -4966,19 +4969,7 @@ make_join_statistics(JOIN *join, List &tables_list, Range analyzer might have modified the condition. Put it the new condition to where we got it from. */ - switch (cond_source) { - case 0: - *s->on_expr_ref= select->cond; - break; - case 1: - s->table->pos_in_table_list->embedding->on_expr= select->cond; - break; - case 2: - join->conds= select->cond; - break; - default: - DBUG_ASSERT(0); - } + *sargable_cond= select->cond; s->quick=select->quick; s->needed_reg=select->needed_reg; diff --git a/sql/sql_select.h b/sql/sql_select.h index 2cc47f6ec3b..0e486c1fbec 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -263,8 +263,12 @@ typedef struct st_join_table { /* Pointer to the associated ON expression. on_expr_ref=!NULL except for degenerate joins. - *on_expr_ref!=NULL for tables that are first inner tables within an outer - join. + + Optimization phase: *on_expr_ref!=NULL for tables that are the single + tables on the inner side of the outer join (t1 LEFT JOIN t2 ON...) + + Execution phase: *on_expr_ref!=NULL for tables that are first inner tables + within an outer join (which may have multiple tables) */ Item **on_expr_ref; COND_EQUAL *cond_equal; /**< multiple equalities for the on expression */ From 14b62b15784233df105a3684d362827b2883bed4 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 23 Oct 2018 13:17:14 +0200 Subject: [PATCH 124/127] MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins Continuation of the fix: Make condition selectivity estimate use the right estimate, too. --- mysql-test/main/join_outer.result | 32 +++++++++++++++++++++++++++++++ mysql-test/main/join_outer.test | 23 ++++++++++++++++++++++ sql/sql_select.cc | 8 +++++--- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/join_outer.result b/mysql-test/main/join_outer.result index 5a123826d79..aa9f30885a9 100644 --- a/mysql-test/main/join_outer.result +++ b/mysql-test/main/join_outer.result @@ -2531,5 +2531,37 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 10 1 SIMPLE t2 ALL NULL NULL NULL NULL 2 1 SIMPLE t3 range a a 5 NULL 5 Using where +# +# .. part 2: make sure condition selectivity can use the condition too. +# +alter table t3 drop key a; +set @tmp1=@@optimizer_use_condition_selectivity; +set @tmp2=@@use_stat_tables; +set @tmp3=@@histogram_size; +set use_stat_tables=preferably; +set optimizer_use_condition_selectivity=4; +set histogram_size=100; +analyze table t3 persistent for all; +Table Op Msg_type Msg_text +test.t3 analyze status Engine-independent statistics collected +test.t3 analyze status OK +# t3.filtered is less than 100%: +explain extended select * from t1 left join t3 on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 100.00 +1 SIMPLE t3 ALL NULL NULL NULL NULL 1000 0.99 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t1` left join `test`.`t3` on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5) where 1 +# t3.filtered must less than 100%, too: +explain extended select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 100.00 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 100.00 +1 SIMPLE t3 ALL NULL NULL NULL NULL 1000 0.99 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t2`.`a` AS `a` from `test`.`t1` left join (`test`.`t3` join `test`.`t2`) on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5) where 1 drop table t1,t2,t3; +set optimizer_use_condition_selectivity= @tmp1; +set use_stat_tables= @tmp2; +set histogram_size= @tmp3; SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_outer.test b/mysql-test/main/join_outer.test index 28f544dec7d..9118febc800 100644 --- a/mysql-test/main/join_outer.test +++ b/mysql-test/main/join_outer.test @@ -2060,6 +2060,29 @@ explain select * from t1 left join t3 on t1.a=t3.b and t3.a<5; --echo # This must use range for table t3, too: explain select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; +--echo # +--echo # .. part 2: make sure condition selectivity can use the condition too. +--echo # +alter table t3 drop key a; +set @tmp1=@@optimizer_use_condition_selectivity; +set @tmp2=@@use_stat_tables; +set @tmp3=@@histogram_size; +set use_stat_tables=preferably; +set optimizer_use_condition_selectivity=4; +set histogram_size=100; + +analyze table t3 persistent for all; + +--echo # t3.filtered is less than 100%: +explain extended select * from t1 left join t3 on t1.a=t3.b and t3.a<5; + +--echo # t3.filtered must less than 100%, too: +explain extended select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; + drop table t1,t2,t3; +set optimizer_use_condition_selectivity= @tmp1; +set use_stat_tables= @tmp2; +set histogram_size= @tmp3; + SET optimizer_switch=@save_optimizer_switch; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 08e9b8daf73..39c77dc4dc2 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -4951,9 +4951,10 @@ make_join_statistics(JOIN *join, List &tables_list, bool impossible_range= FALSE; ha_rows records= HA_POS_ERROR; SQL_SELECT *select= 0; + Item **sargable_cond= NULL; if (!s->const_keys.is_clear_all()) { - Item **sargable_cond= get_sargable_cond(join, s->table); + sargable_cond= get_sargable_cond(join, s->table); select= make_select(s->table, found_const_table_map, found_const_table_map, @@ -4978,10 +4979,11 @@ make_join_statistics(JOIN *join, List &tables_list, } if (!impossible_range) { + if (!sargable_cond) + sargable_cond= get_sargable_cond(join, s->table); if (join->thd->variables.optimizer_use_condition_selectivity > 1) calculate_cond_selectivity_for_table(join->thd, s->table, - *s->on_expr_ref ? - s->on_expr_ref : &join->conds); + sargable_cond); if (s->table->reginfo.impossible_range) { impossible_range= TRUE; From f0cf85fd15509179bf6e33176cb89761fc89e5a9 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 23 Oct 2018 14:12:59 +0200 Subject: [PATCH 125/127] MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins Update .result files after the previous patch --- mysql-test/main/join_outer_jcl6.result | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/mysql-test/main/join_outer_jcl6.result b/mysql-test/main/join_outer_jcl6.result index 842ab19e20a..6f5e4b3e85e 100644 --- a/mysql-test/main/join_outer_jcl6.result +++ b/mysql-test/main/join_outer_jcl6.result @@ -2542,7 +2542,39 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 10 1 SIMPLE t2 ALL NULL NULL NULL NULL 2 Using join buffer (flat, BNL join) 1 SIMPLE t3 range a a 5 NULL 5 Using where; Rowid-ordered scan; Using join buffer (incremental, BNL join) +# +# .. part 2: make sure condition selectivity can use the condition too. +# +alter table t3 drop key a; +set @tmp1=@@optimizer_use_condition_selectivity; +set @tmp2=@@use_stat_tables; +set @tmp3=@@histogram_size; +set use_stat_tables=preferably; +set optimizer_use_condition_selectivity=4; +set histogram_size=100; +analyze table t3 persistent for all; +Table Op Msg_type Msg_text +test.t3 analyze status Engine-independent statistics collected +test.t3 analyze status OK +# t3.filtered is less than 100%: +explain extended select * from t1 left join t3 on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 100.00 +1 SIMPLE t3 hash_ALL NULL #hash#$hj 5 test.t1.a 1000 0.99 Using where; Using join buffer (flat, BNLH join) +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from `test`.`t1` left join `test`.`t3` on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5 and `test`.`t1`.`a` is not null) where 1 +# t3.filtered must less than 100%, too: +explain extended select * from t1 left join (t3 join t2) on t1.a=t3.b and t3.a<5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 100.00 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 100.00 Using where +1 SIMPLE t3 ALL NULL NULL NULL NULL 1000 0.99 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t2`.`a` AS `a` from `test`.`t1` left join (`test`.`t3` join `test`.`t2`) on(`test`.`t3`.`b` = `test`.`t1`.`a` and `test`.`t3`.`a` < 5 and `test`.`t1`.`a` is not null) where 1 drop table t1,t2,t3; +set optimizer_use_condition_selectivity= @tmp1; +set use_stat_tables= @tmp2; +set histogram_size= @tmp3; SET optimizer_switch=@save_optimizer_switch; set join_cache_level=default; show variables like 'join_cache_level'; From e058a251c10350f3727ca1df022dc5786933535b Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 1 Nov 2018 11:35:28 +0300 Subject: [PATCH 126/127] MDEV-17518: Range optimization doesn't use ON expressions from nested outer joins Part#2: take into account that join nest that we are marking as constant might already have constant tables in it. Don't count these tables twice. --- mysql-test/main/join_outer.result | 15 +++++++++++++++ mysql-test/main/join_outer.test | 17 +++++++++++++++++ mysql-test/main/join_outer_jcl6.result | 15 +++++++++++++++ sql/sql_select.cc | 17 ++++++++++------- 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/mysql-test/main/join_outer.result b/mysql-test/main/join_outer.result index aa9f30885a9..6b671a0759f 100644 --- a/mysql-test/main/join_outer.result +++ b/mysql-test/main/join_outer.result @@ -2564,4 +2564,19 @@ drop table t1,t2,t3; set optimizer_use_condition_selectivity= @tmp1; set use_stat_tables= @tmp2; set histogram_size= @tmp3; +# Another test +CREATE TABLE t1 (i1 int) ; +CREATE TABLE t2 (pk int NOT NULL PRIMARY KEY) ; +CREATE TABLE t3 (pk int NOT NULL, i1 int, PRIMARY KEY (pk)) ; +INSERT INTO t3 VALUES (2, NULL); +CREATE TABLE t4 (pk int NOT NULL, i1 int, PRIMARY KEY (pk), KEY i1 (i1)) ; +CREATE VIEW v4 AS SELECT * FROM t4; +SELECT 1 +FROM t3 RIGHT JOIN t1 ON t3.i1 = t1.i1 +LEFT JOIN v4 +RIGHT JOIN t2 ON v4.i1 = t2.pk ON t1.i1 = t2.pk +WHERE t3.pk IN (2); +1 +drop view v4; +drop table t1,t2,t3,t4; SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_outer.test b/mysql-test/main/join_outer.test index 9118febc800..82127397bee 100644 --- a/mysql-test/main/join_outer.test +++ b/mysql-test/main/join_outer.test @@ -2084,5 +2084,22 @@ set optimizer_use_condition_selectivity= @tmp1; set use_stat_tables= @tmp2; set histogram_size= @tmp3; +--echo # Another test +CREATE TABLE t1 (i1 int) ; +CREATE TABLE t2 (pk int NOT NULL PRIMARY KEY) ; +CREATE TABLE t3 (pk int NOT NULL, i1 int, PRIMARY KEY (pk)) ; +INSERT INTO t3 VALUES (2, NULL); + +CREATE TABLE t4 (pk int NOT NULL, i1 int, PRIMARY KEY (pk), KEY i1 (i1)) ; +CREATE VIEW v4 AS SELECT * FROM t4; + +SELECT 1 +FROM t3 RIGHT JOIN t1 ON t3.i1 = t1.i1 + LEFT JOIN v4 + RIGHT JOIN t2 ON v4.i1 = t2.pk ON t1.i1 = t2.pk +WHERE t3.pk IN (2); + +drop view v4; +drop table t1,t2,t3,t4; SET optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/main/join_outer_jcl6.result b/mysql-test/main/join_outer_jcl6.result index 6f5e4b3e85e..f46f3e2df21 100644 --- a/mysql-test/main/join_outer_jcl6.result +++ b/mysql-test/main/join_outer_jcl6.result @@ -2575,6 +2575,21 @@ drop table t1,t2,t3; set optimizer_use_condition_selectivity= @tmp1; set use_stat_tables= @tmp2; set histogram_size= @tmp3; +# Another test +CREATE TABLE t1 (i1 int) ; +CREATE TABLE t2 (pk int NOT NULL PRIMARY KEY) ; +CREATE TABLE t3 (pk int NOT NULL, i1 int, PRIMARY KEY (pk)) ; +INSERT INTO t3 VALUES (2, NULL); +CREATE TABLE t4 (pk int NOT NULL, i1 int, PRIMARY KEY (pk), KEY i1 (i1)) ; +CREATE VIEW v4 AS SELECT * FROM t4; +SELECT 1 +FROM t3 RIGHT JOIN t1 ON t3.i1 = t1.i1 +LEFT JOIN v4 +RIGHT JOIN t2 ON v4.i1 = t2.pk ON t1.i1 = t2.pk +WHERE t3.pk IN (2); +1 +drop view v4; +drop table t1,t2,t3,t4; SET optimizer_switch=@save_optimizer_switch; set join_cache_level=default; show variables like 'join_cache_level'; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 39c77dc4dc2..a99b991af96 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -4316,14 +4316,17 @@ void mark_join_nest_as_const(JOIN *join, } JOIN_TAB *tab= tbl->table->reginfo.join_tab; - tab->type= JT_CONST; - tab->info= ET_IMPOSSIBLE_ON_CONDITION; - tab->table->const_table= 1; + if (!(join->const_table_map & tab->table->map)) + { + tab->type= JT_CONST; + tab->info= ET_IMPOSSIBLE_ON_CONDITION; + tab->table->const_table= 1; - join->const_table_map|= tab->table->map; - *found_const_table_map|= tab->table->map; - set_position(join,(*const_count)++,tab,(KEYUSE*) 0); - mark_as_null_row(tab->table); // All fields are NULL + join->const_table_map|= tab->table->map; + *found_const_table_map|= tab->table->map; + set_position(join,(*const_count)++,tab,(KEYUSE*) 0); + mark_as_null_row(tab->table); // All fields are NULL + } } } From bdfe2784d5b73a1fdcdacb3d9adcc9dc71af344b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 6 Nov 2018 08:42:30 +0200 Subject: [PATCH 127/127] Unify a test with the 10.3 version Remove trailing white space, and use lower case for "eval" and "let". Add an "End of 10.2 tests" marker to ease future merges. --- mysql-test/r/derived_cond_pushdown.result | 459 ++++++------- mysql-test/t/derived_cond_pushdown.test | 751 +++++++++++----------- 2 files changed, 606 insertions(+), 604 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 9d9d1e7613c..f5bc72e91e7 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -1,23 +1,23 @@ create table t1 (a int, b int, c int); create table t2 (a int, b int, c int, d decimal); -insert into t1 values +insert into t1 values (1,21,345), (1,33,7), (8,33,114), (1,21,500), (1,19,107), (5,14,787), (8,33,123), (9,10,211), (5,16,207), (1,33,988), (5,27,132), (1,21,104), (6,20,309), (6,20,315), (1,21,101), (8,33,404), (9,10,800), (1,21,123), (7,11,708), (6,20,214); insert into t2 values -(2,3,207,207.0000), (1,21,909,12.0000), (7,13,312,406.0000), +(2,3,207,207.0000), (1,21,909,12.0000), (7,13,312,406.0000), (8,64,248,107.0000), (6,20,315,279.3333), (1,19,203,107.0000), (8,80,800,314.0000), (3,12,231,190.0000), (6,23,303,909.0000); Warnings: Note 1265 Data truncated for column 'd' at row 5 create table t1_double(a int, b double, c double); -insert into t1_double values +insert into t1_double values (1,23.4,14.3333), (1,12.5,18.9), (3,12.5,18.9), (4,33.4,14.3333), (4,14.3333,13.65), (5,17.89,7.22), (6,33.4,14.3), (10,33.4,13.65), (11,33.4,13.65); create table t2_double(a int, b double, c double); -insert into t2_double values +insert into t2_double values (1,22.4,14.3333), (1,12.5,18.9), (2,22.4,18.9), (4,33.4,14.3333), (5,22.4,13.65), (7,17.89,18.9), (6,33.4,14.3333), (10,31.4,13.65), (12,33.4,13.65); @@ -41,22 +41,22 @@ insert into t2_decimal values (2,1,13),(2,2,11),(3,3,16), (1,3,22),(1,3,14),(2,2,15), (2,1,43),(2,3,11),(2,3,16); -create view v1 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 +create view v1 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 group by a,b having max_c < 707; -create view v2 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 +create view v2 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 where t1.a>5 group by a,b having max_c < 707; create view v3 as select a, b, min(c) as min_c from t1 where t1.a<10 group by a,b having min_c > 109; create view v4 as select a, b, min(max_c) as min_c from v1 where (v1.a<15) group by a,b; -create view v_union as +create view v_union as select a, b, min(c) as c from t1 where t1.a<10 group by a,b having c > 109 union select a, b, max(c) as c from t1 where t1.b>10 group by a,b having c < 300; -create view v2_union as +create view v2_union as select a, b, min(c) as c from t1 where t1.a<10 group by a,b having c > 109 union @@ -65,13 +65,13 @@ where t1.b>10 group by a,b having c < 300 union select a, b, avg(c) as c from t1 where t1.c>300 group by a,b having c < 707; -create view v3_union as +create view v3_union as select a, b, (a+1) as c from t1 where t1.a<10 union select a, b, c from t1 where t1.b>10 and t1.c>100; -create view v4_union as +create view v4_union as select a, b, max(c)-100 as c from t1 where t1.a<10 group by a,b having c > 109 union @@ -80,13 +80,13 @@ where t1.b>10; create view v_double as select a, avg(a/4) as avg_a, b, c from t1_double where (b>12.2) group by b,c having (avg_a<22.333); -create view v_char as -select a, b, max(c) as max_c from t1_char +create view v_char as +select a, b, max(c) as max_c from t1_char group by a,b having max_c < 9; create view v_decimal as select a, b, avg(c) as avg_c from t1_decimal group by a,b having (avg_c>12); -# conjunctive subformula : pushing into HAVING +# conjunctive subformula : pushing into HAVING set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.max_c>214) and (t2.a>v1.a); a b max_c avg_c a b c d 1 21 500 234.6000 2 3 207 207 @@ -161,7 +161,7 @@ EXPLAIN } set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300); a b max_c avg_c a b c d 1 21 500 234.6000 1 21 909 12 @@ -172,7 +172,7 @@ a b max_c avg_c a b c d 6 20 315 279.3333 6 23 303 909 select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300); a b max_c avg_c a b c d 1 21 500 234.6000 1 21 909 12 @@ -183,7 +183,7 @@ a b max_c avg_c a b c d 6 20 315 279.3333 6 23 303 909 explain select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -191,7 +191,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort explain format=json select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300); EXPLAIN { @@ -235,8 +235,8 @@ EXPLAIN } } } -# extracted or formula : pushing into HAVING -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +# extracted or formula : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where ((v1.max_c>400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where ((v1.max_c>400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or +((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or ((v1.max_c<135) and (v1.max_c300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or +((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or ((v1.max_c<135) and (v1.max_c300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or +((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or ((v1.max_c<135) and (v1.max_c ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort explain format=json select * from v1,t2 where -((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or +((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or ((v1.max_c<135) and (v1.max_c7) and (t2.c7) and (t2.c7) and (t2.c ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where ((v1.a>7) and (t2.c7) and (t2.c5) and (t2.b7) and (t2.c5) and (t2.b7) and (t2.c5) and (t2.b ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v2,t2 where +explain format=json select * from v2,t2 where ((v2.a>7) and (t2.c5) and (t2.b4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or ((v1.a<2) and (v1.max_c4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or +select * from v1,t2 where +((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or ((v1.a<2) and (v1.max_c4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or +explain select * from v1,t2 where +((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or ((v1.a<2) and (v1.max_c ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,t2 where -((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or +explain format=json select * from v1,t2 where +((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or ((v1.a<2) and (v1.max_c7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,t2 where -((v1.a>7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c120) and (v1.b=t2.b)) or (v1.a>7); a b max_c avg_c a b c d 1 21 500 234.6000 1 21 909 12 @@ -1118,7 +1118,7 @@ a b max_c avg_c a b c d 8 33 404 213.6667 8 80 800 314 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 6 23 303 909 -select * from v1,t2 where +select * from v1,t2 where ((v1.a<2) and (v1.max_c>120) and (v1.b=t2.b)) or (v1.a>7); a b max_c avg_c a b c d 1 21 500 234.6000 1 21 909 12 @@ -1131,13 +1131,13 @@ a b max_c avg_c a b c d 8 33 404 213.6667 8 80 800 314 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 6 23 303 909 -explain select * from v1,t2 where +explain select * from v1,t2 where ((v1.a<2) and (v1.max_c>120) and (v1.b=t2.b)) or (v1.a>7); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where ((v1.a<2) and (v1.max_c>120) and (v1.b=t2.b)) or (v1.a>7); EXPLAIN { @@ -1250,7 +1250,7 @@ EXPLAIN } } # prepare of a query containing extracted or formula -prepare stmt from "select * from v1,t2 where +prepare stmt from "select * from v1,t2 where ((v1.max_c>400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a20)); a b max_c a b c c Harry 4 d Harry 1 c Harry 4 b Harry 4 -select * from v_char as v,t2_char as t where +select * from v_char as v,t2_char as t where (v.a='c') and (v.b<'Hermes') and ((v.b=t.b) or (v.max_c>20)); a b max_c a b c c Harry 4 d Harry 1 c Harry 4 b Harry 4 -explain select * from v_char as v,t2_char as t where +explain select * from v_char as v,t2_char as t where (v.a='c') and (v.b<'Hermes') and ((v.b=t.b) or (v.max_c>20)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ALL NULL NULL NULL NULL 12 Using where 1 PRIMARY t ALL NULL NULL NULL NULL 12 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1_char ALL NULL NULL NULL NULL 12 Using where; Using temporary; Using filesort -explain format=json select * from v_char as v,t2_char as t where +explain format=json select * from v_char as v,t2_char as t where (v.a='c') and (v.b<'Hermes') and ((v.b=t.b) or (v.max_c>20)); EXPLAIN { @@ -1885,7 +1885,7 @@ EXPLAIN } } } -# extracted or formula : pushing into HAVING using equalities +# extracted or formula : pushing into HAVING using equalities set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where ((t2.a<4) and (v1.a=t2.a)) or ((t2.c>150) and (v1.max_c=t2.c)); a b max_c avg_c a b c d @@ -2014,24 +2014,24 @@ EXPLAIN } } # conjunctive subformulas : pushing into WHERE and HAVING -# pushing equalities +# pushing equalities set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=8) and (v1.a=t2.a) and (v1.max_c=404); a b max_c avg_c a b c d 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 80 800 314 select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=8) and (v1.a=t2.a) and (v1.max_c=404); a b max_c avg_c a b c d 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 80 800 314 explain select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=8) and (v1.a=t2.a) and (v1.max_c=404); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -2039,7 +2039,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort explain format=json select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where (v1.a=8) and (v1.a=t2.a) and (v1.max_c=404); EXPLAIN { @@ -2084,22 +2084,22 @@ EXPLAIN } } } -# conjunctive subformulas : pushing into WHERE and HAVING -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +# conjunctive subformulas : pushing into WHERE and HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a>3) and (v1.max_c>200) and (t2.b3) and (v1.max_c>200) and (t2.b3) and (v1.max_c>200) and (t2.b ref key0 key0 5 test.t2.d 2 Using where 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where (v1.a>3) and (v1.max_c>200) and (t2.b0.2) or (v.b<17)) or (t.c>17)) and (t.c=v.c) and (v.c>18); a avg_a b c a b c @@ -2366,17 +2366,17 @@ EXPLAIN } set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.a=v1.b); a b max_c avg_c a b c d select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.a=v1.b); a b max_c avg_c a b c d explain select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.a=v1.b); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -2384,7 +2384,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort explain format=json select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.a=v1.b); EXPLAIN { @@ -2550,7 +2550,7 @@ EXPLAIN } } } -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); a b max_c avg_c a b c d 1 19 107 107.0000 1 21 909 12 @@ -2561,7 +2561,7 @@ a b max_c avg_c a b c d 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 80 800 314 -select * from v1,t2 where +select * from v1,t2 where (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); a b max_c avg_c a b c d 1 19 107 107.0000 1 21 909 12 @@ -2572,13 +2572,13 @@ a b max_c avg_c a b c d 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 80 800 314 -explain select * from v1,t2 where +explain select * from v1,t2 where (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); EXPLAIN { @@ -2716,7 +2716,7 @@ EXPLAIN } } } -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (t2.a<2) and (t2.c>900) and ((v1.a900) and ((v1.a900) and ((v1.a ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where (t2.a<2) and (t2.c>900) and ((v1.a ref key0 key0 5 test.t2.a 2 Using where 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,v2,t2 where +explain format=json select * from v1,v2,t2 where (v1.a=v2.a) and (v1.a=t2.a) and (v2.b<50); EXPLAIN { @@ -2880,7 +2880,7 @@ EXPLAIN } } } -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 2 3 207 207 @@ -2897,7 +2897,7 @@ a b max_c avg_c a b max_c avg_c a b c d 8 33 404 213.6667 8 33 404 213.6667 1 19 203 107 8 33 404 213.6667 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 8 33 404 213.6667 6 23 303 909 -select * from v1,v2,t2 where +select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 2 3 207 207 @@ -2914,7 +2914,7 @@ a b max_c avg_c a b max_c avg_c a b c d 8 33 404 213.6667 8 33 404 213.6667 1 19 203 107 8 33 404 213.6667 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 8 33 404 213.6667 6 23 303 909 -explain select * from v1,v2,t2 where +explain select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -2922,7 +2922,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ref key0 key0 5 v1.b 2 Using where 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,v2,t2 where +explain format=json select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); EXPLAIN { @@ -2996,21 +2996,21 @@ EXPLAIN } } } -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where ((v1.a=v2.a) and (v1.a=t2.a)) or ((v2.b>13) and (t2.c<115)); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 33 404 213.6667 8 80 800 314 -select * from v1,v2,t2 where +select * from v1,v2,t2 where ((v1.a=v2.a) and (v1.a=t2.a)) or ((v2.b>13) and (t2.c<115)); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 8 64 248 107 8 33 404 213.6667 8 33 404 213.6667 8 80 800 314 -explain select * from v1,v2,t2 where +explain select * from v1,v2,t2 where ((v1.a=v2.a) and (v1.a=t2.a)) or ((v2.b>13) and (t2.c<115)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 @@ -3018,7 +3018,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (incremental, BNL join) 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,v2,t2 where +explain format=json select * from v1,v2,t2 where ((v1.a=v2.a) and (v1.a=t2.a)) or ((v2.b>13) and (t2.c<115)); EXPLAIN { @@ -3094,22 +3094,22 @@ EXPLAIN # using several derived tables : pushing in all tables # conjunctive subformula : pushing into HAVING # extracted or formula : pushing into WHERE -# pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and ((v2.b<50) or (v2.b=19)) and (v1.max_c<300); a b max_c avg_c a b max_c avg_c a b c d 1 19 107 107.0000 6 20 315 279.3333 1 21 909 12 1 19 107 107.0000 6 20 315 279.3333 1 19 203 107 1 19 107 107.0000 8 33 404 213.6667 1 21 909 12 1 19 107 107.0000 8 33 404 213.6667 1 19 203 107 -select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and +select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and ((v2.b<50) or (v2.b=19)) and (v1.max_c<300); a b max_c avg_c a b max_c avg_c a b c d 1 19 107 107.0000 6 20 315 279.3333 1 21 909 12 1 19 107 107.0000 6 20 315 279.3333 1 19 203 107 1 19 107 107.0000 8 33 404 213.6667 1 21 909 12 1 19 107 107.0000 8 33 404 213.6667 1 19 203 107 -explain select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and +explain select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and ((v2.b<50) or (v2.b=19)) and (v1.max_c<300); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 @@ -3117,7 +3117,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (incremental, BNL join) 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and +explain format=json select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and ((v2.b<50) or (v2.b=19)) and (v1.max_c<300); EXPLAIN { @@ -3194,13 +3194,13 @@ EXPLAIN # using several derived tables : pushing only in one table # conjunctive subformula : pushing into WHERE # pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,t2 where (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); a b max_c avg_c a b max_c avg_c a b c d -select * from v1,v2,t2 where +select * from v1,v2,t2 where (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); a b max_c avg_c a b max_c avg_c a b c d -explain select * from v1,v2,t2 where +explain select * from v1,v2,t2 where (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -3208,7 +3208,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort -explain format=json select * from v1,v2,t2 where +explain format=json select * from v1,v2,t2 where (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); EXPLAIN { @@ -3373,7 +3373,7 @@ EXPLAIN # extracted or formula : pushing into WHERE # conjunctive subformula : pushing into WHERE using equalities # pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_char as v,t2_char as t where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_char as v,t2_char as t where (v.a=t.a) and (t.a='b') and ((v.b='Vika') or (v.b='Ali')); a b max_c a b c b Vika 2 b Ivan 1 @@ -3381,7 +3381,7 @@ b Vika 2 b Ali 6 b Vika 2 b Hermes 3 b Vika 2 b Ivan 11 b Vika 2 b Harry 4 -select * from v_char as v,t2_char as t where +select * from v_char as v,t2_char as t where (v.a=t.a) and (t.a='b') and ((v.b='Vika') or (v.b='Ali')); a b max_c a b c b Vika 2 b Ivan 1 @@ -3389,13 +3389,13 @@ b Vika 2 b Ali 6 b Vika 2 b Hermes 3 b Vika 2 b Ivan 11 b Vika 2 b Harry 4 -explain select * from v_char as v,t2_char as t where +explain select * from v_char as v,t2_char as t where (v.a=t.a) and (t.a='b') and ((v.b='Vika') or (v.b='Ali')); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY ALL NULL NULL NULL NULL 12 Using where 1 PRIMARY t ALL NULL NULL NULL NULL 12 Using where; Using join buffer (flat, BNL join) 2 DERIVED t1_char ALL NULL NULL NULL NULL 12 Using where; Using temporary; Using filesort -explain format=json select * from v_char as v,t2_char as t where +explain format=json select * from v_char as v,t2_char as t where (v.a=t.a) and (t.a='b') and ((v.b='Vika') or (v.b='Ali')); EXPLAIN { @@ -3442,10 +3442,10 @@ EXPLAIN } # using several derived tables : pushing in all tables # extracted or formula : pushing into WHERE -# conjunctive subformulas : pushing into HAVING +# conjunctive subformulas : pushing into HAVING # pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,v3,t2 where -((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,v2,v3,t2 where +((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) and (v1.max_c<500) and (v3.a=t2.a) and (v2.max_c>300); a b max_c avg_c a b max_c avg_c a b min_c a b c d 6 20 315 279.3333 6 20 315 279.3333 7 11 708 7 13 312 406 @@ -3462,8 +3462,8 @@ a b max_c avg_c a b max_c avg_c a b min_c a b c d 8 33 404 213.6667 8 33 404 213.6667 6 20 214 6 20 315 279 8 33 404 213.6667 8 33 404 213.6667 8 33 114 8 80 800 314 8 33 404 213.6667 8 33 404 213.6667 6 20 214 6 23 303 909 -select * from v1,v2,v3,t2 where -((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) +select * from v1,v2,v3,t2 where +((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) and (v1.max_c<500) and (v3.a=t2.a) and (v2.max_c>300); a b max_c avg_c a b max_c avg_c a b min_c a b c d 6 20 315 279.3333 6 20 315 279.3333 7 11 708 7 13 312 406 @@ -3480,8 +3480,8 @@ a b max_c avg_c a b max_c avg_c a b min_c a b c d 8 33 404 213.6667 8 33 404 213.6667 6 20 214 6 20 315 279 8 33 404 213.6667 8 33 404 213.6667 8 33 114 8 80 800 314 8 33 404 213.6667 8 33 404 213.6667 6 20 214 6 23 303 909 -explain select * from v1,v2,v3,t2 where -((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) +explain select * from v1,v2,v3,t2 where +((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) and (v1.max_c<500) and (v3.a=t2.a) and (v2.max_c>300); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -3491,8 +3491,8 @@ id select_type table type possible_keys key key_len ref rows Extra 4 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort -explain format=json select * from v1,v2,v3,t2 where -((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) +explain format=json select * from v1,v2,v3,t2 where +((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) and (v1.max_c<500) and (v3.a=t2.a) and (v2.max_c>300); EXPLAIN { @@ -3598,7 +3598,7 @@ EXPLAIN } } # using several derived tables : pushing in all tables -# conjunctive subformulas : pushing into HAVING +# conjunctive subformulas : pushing into HAVING set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 where t1.a>5 group by a,b having max_c < 707) v1, @@ -3706,7 +3706,7 @@ EXPLAIN } # using several derived tables : pushing in all tables # extracted or formulas : pushing into HAVING -# conjunctive subformula : pushing into HAVING +# conjunctive subformula : pushing into HAVING set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 where t1.a>5 group by a,b having max_c < 707) v1, @@ -3875,10 +3875,10 @@ EXPLAIN } } # extracted or formula : pushing into HAVING -# conjunctive subformula : pushing into WHERE +# conjunctive subformula : pushing into WHERE set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); a b max_c avg_c a b c d 5 27 132 132.0000 2 3 207 207 @@ -3892,7 +3892,7 @@ a b max_c avg_c a b c d 5 27 132 132.0000 6 23 303 909 select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); a b max_c avg_c a b c d 5 27 132 132.0000 2 3 207 207 @@ -3906,7 +3906,7 @@ a b max_c avg_c a b c d 5 27 132 132.0000 6 23 303 909 explain select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 @@ -3914,7 +3914,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort explain format=json select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -group by a,b having max_c < 707) v1, +group by a,b having max_c < 707) v1, t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); EXPLAIN { @@ -3960,24 +3960,24 @@ EXPLAIN } } # extracted and formula : pushing into WHERE -# conjunctive subformula : pushing into HAVING +# conjunctive subformula : pushing into HAVING set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279 6 20 315 279.3333 6 23 303 909 select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279 6 20 315 279.3333 6 23 303 909 explain select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -3985,7 +3985,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort explain format=json select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 -where t1.a>5 group by a,b having max_c < 707) v1, +where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); EXPLAIN { @@ -4155,11 +4155,11 @@ EXPLAIN } } # using query with union -# extracted and formula : pushing into WHERE -# extracted or formula : pushing into HAVING +# extracted and formula : pushing into WHERE +# extracted or formula : pushing into HAVING # pushing equalities set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a<5) and (v1.b=t2.b) and (v1.b=19) -union +union select * from v1,t2 where ((v1.max_c>400) or (v1.avg_c>270)) and (v1.a400) or (v1.avg_c>270)) and (v1.a400) or (v1.avg_c>270)) and (v1.a ALL NULL NULL NULL NULL NULL explain format=json select * from v1,t2 where (v1.a<5) and (v1.b=t2.b) and (v1.b=19) -union +union select * from v1,t2 where ((v1.max_c>400) or (v1.avg_c>270)) and (v1.a3) and (v1.b>27)) or (v1.max_c>550); @@ -4320,7 +4320,7 @@ a b max_c avg_c a b c d 8 33 404 213.6667 8 80 800 314 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 6 23 303 909 -select * from v1,t2 where +select * from v1,t2 where ((t2.a=v1.a) or (v1.b=t2.b)) and ((v1.a=1) or (v1.a=6)) union select * from v1,t2 where ((v1.a>3) and (v1.b>27)) or (v1.max_c>550); @@ -4340,7 +4340,7 @@ a b max_c avg_c a b c d 8 33 404 213.6667 8 80 800 314 8 33 404 213.6667 3 12 231 190 8 33 404 213.6667 6 23 303 909 -explain select * from v1,t2 where +explain select * from v1,t2 where ((t2.a=v1.a) or (v1.b=t2.b)) and ((v1.a=1) or (v1.a=6)) union select * from v1,t2 where ((v1.a>3) and (v1.b>27)) or (v1.max_c>550); @@ -4352,7 +4352,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 UNION ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) 4 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort NULL UNION RESULT ALL NULL NULL NULL NULL NULL -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where ((t2.a=v1.a) or (v1.b=t2.b)) and ((v1.a=1) or (v1.a=6)) union select * from v1,t2 where ((v1.a>3) and (v1.b>27)) or (v1.max_c>550); @@ -4451,10 +4451,10 @@ EXPLAIN } } # using query with union -# extracted or formula : pushing into HAVING -# conjunctive subformulas : pushing into WHERE +# extracted or formula : pushing into HAVING +# conjunctive subformulas : pushing into WHERE # pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where ((v1.a=1) and (v1.a=t2.a)) and ((v1.max_c<500) or (v1.avg_c>500)) union select * from v2,t2 where @@ -4469,7 +4469,7 @@ a b max_c avg_c a b c d 6 20 315 279.3333 1 19 203 107 8 33 404 213.6667 1 21 909 12 8 33 404 213.6667 1 19 203 107 -select * from v1,t2 where +select * from v1,t2 where ((v1.a=1) and (v1.a=t2.a)) and ((v1.max_c<500) or (v1.avg_c>500)) union select * from v2,t2 where @@ -4484,7 +4484,7 @@ a b max_c avg_c a b c d 6 20 315 279.3333 1 19 203 107 8 33 404 213.6667 1 21 909 12 8 33 404 213.6667 1 19 203 107 -explain select * from v1,t2 where +explain select * from v1,t2 where ((v1.a=1) and (v1.a=t2.a)) and ((v1.max_c<500) or (v1.avg_c>500)) union select * from v2,t2 where @@ -4503,7 +4503,7 @@ id select_type table type possible_keys key key_len ref rows Extra 3 UNION ref key0 key0 5 test.t2.c 2 Using where 6 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort NULL UNION RESULT ALL NULL NULL NULL NULL NULL -explain format=json select * from v1,t2 where +explain format=json select * from v1,t2 where ((v1.a=1) and (v1.a=t2.a)) and ((v1.max_c<500) or (v1.avg_c>500)) union select * from v2,t2 where @@ -4754,7 +4754,7 @@ EXPLAIN } # using derived table with union # conjunctive subformula : pushing into WHERE -# extracted or formula : pushing into HAVING +# extracted or formula : pushing into HAVING set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_union,t2 where ((v_union.a<2) or (v_union.c>800)) and (v_union.b>12); a b c a b c d @@ -4955,27 +4955,27 @@ EXPLAIN } } } -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_char as v,t2_char as t where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_char as v,t2_char as t where (v.a=t.a) and (v.b='Vika') and (v.max_c>2); a b max_c a b c c Vika 7 c Vinny 3 c Vika 7 c Inga 9 c Vika 7 c Ivan 2 c Vika 7 c Inga 2 -select * from v_char as v,t2_char as t where +select * from v_char as v,t2_char as t where (v.a=t.a) and (v.b='Vika') and (v.max_c>2); a b max_c a b c c Vika 7 c Vinny 3 c Vika 7 c Inga 9 c Vika 7 c Ivan 2 c Vika 7 c Inga 2 -explain select * from v_char as v,t2_char as t where +explain select * from v_char as v,t2_char as t where (v.a=t.a) and (v.b='Vika') and (v.max_c>2); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t ALL NULL NULL NULL NULL 12 Using where 1 PRIMARY ref key0 key0 2 test.t.a 2 Using where 2 DERIVED t1_char ALL NULL NULL NULL NULL 12 Using where; Using temporary; Using filesort -explain format=json select * from v_char as v,t2_char as t where +explain format=json select * from v_char as v,t2_char as t where (v.a=t.a) and (v.b='Vika') and (v.max_c>2); EXPLAIN { @@ -5025,19 +5025,19 @@ EXPLAIN # conjunctive subformula : pushing into WHERE using equalities # pushing equalities set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v_union,v1,t2 where -(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) +(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) and ((v_union.c>800) or (v1.max_c>200)); a b c a b max_c avg_c a b c d 1 19 107 1 21 500 234.6000 1 21 909 12 1 19 107 1 21 500 234.6000 1 19 203 107 select * from v_union,v1,t2 where -(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) +(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) and ((v_union.c>800) or (v1.max_c>200)); a b c a b max_c avg_c a b c d 1 19 107 1 21 500 234.6000 1 21 909 12 1 19 107 1 21 500 234.6000 1 19 203 107 explain select * from v_union,v1,t2 where -(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) +(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) and ((v_union.c>800) or (v1.max_c>200)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -5048,7 +5048,7 @@ id select_type table type possible_keys key key_len ref rows Extra 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort NULL UNION RESULT ALL NULL NULL NULL NULL NULL explain format=json select * from v_union,v1,t2 where -(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) +(v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) and ((v_union.c>800) or (v1.max_c>200)); EXPLAIN { @@ -5152,11 +5152,11 @@ EXPLAIN } } } -# using derived table with union -# extracted or formula : pushing into WHERE +# using derived table with union +# extracted or formula : pushing into WHERE # conjunctive subformula : pushing into HAVING # pushing equalities -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v2_union as v,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v2_union as v,t2 where ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); a b c a b c d 8 33 404.0000 8 64 248 107 @@ -5165,7 +5165,7 @@ a b c a b c d 8 33 404.0000 8 80 800 314 6 20 312.0000 6 23 303 909 6 20 214.0000 6 23 303 909 -select * from v2_union as v,t2 where +select * from v2_union as v,t2 where ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); a b c a b c d 8 33 404.0000 8 64 248 107 @@ -5174,7 +5174,7 @@ a b c a b c d 8 33 404.0000 8 80 800 314 6 20 312.0000 6 23 303 909 6 20 214.0000 6 23 303 909 -explain select * from v2_union as v,t2 where +explain select * from v2_union as v,t2 where ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -5183,7 +5183,7 @@ id select_type table type possible_keys key key_len ref rows Extra 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 4 UNION t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort NULL UNION RESULT ALL NULL NULL NULL NULL NULL -explain format=json select * from v2_union as v,t2 where +explain format=json select * from v2_union as v,t2 where ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); EXPLAIN { @@ -5530,21 +5530,21 @@ EXPLAIN } # using derived table with union of selects without aggregation # extracted the whole condition: in WHERE of both selects -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v3_union as v,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v3_union as v,t2 where (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); a b c a b c d 1 19 2 1 21 909 12 1 21 2 1 21 909 12 1 19 2 1 19 203 107 1 21 2 1 19 203 107 -select * from v3_union as v,t2 where +select * from v3_union as v,t2 where (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); a b c a b c d 1 19 2 1 21 909 12 1 21 2 1 21 909 12 1 19 2 1 19 203 107 1 21 2 1 19 203 107 -explain select * from v3_union as v,t2 where +explain select * from v3_union as v,t2 where (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -5552,7 +5552,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where NULL UNION RESULT ALL NULL NULL NULL NULL NULL -explain format=json select * from v3_union as v,t2 where +explain format=json select * from v3_union as v,t2 where (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); EXPLAIN { @@ -5613,7 +5613,7 @@ EXPLAIN } } } -# using derived table with union of +# using derived table with union of # a select without aggregation and a select with aggregation # extracted conjunctive predicate: pushing in WHERE of both selects set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v4_union as v,t2 where (v.a=t2.a) and (v.b<20); @@ -5701,13 +5701,13 @@ EXPLAIN } } } -# using derived table with union of +# using derived table with union of # a select without aggregation and a select with aggregation # extracted subformula: pushing in WHERE of one select # extracted subformula: pushing in HAVING of the other select # extracted sub-subformula: pushing in WHERE of the other select # using an equality in all pushdowns -set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v4_union as v,t2 where +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v4_union as v,t2 where (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); a b c a b c d 1 33 1088 1 21 909 12 @@ -5720,7 +5720,7 @@ a b c a b c d 1 21 600 1 19 203 107 1 33 888 1 19 203 107 8 33 504 8 80 800 314 -select * from v4_union as v,t2 where +select * from v4_union as v,t2 where (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); a b c a b c d 1 33 1088 1 21 909 12 @@ -5733,7 +5733,7 @@ a b c a b c d 1 21 600 1 19 203 107 1 33 888 1 19 203 107 8 33 504 8 80 800 314 -explain select * from v4_union as v,t2 where +explain select * from v4_union as v,t2 where (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where @@ -5741,7 +5741,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where NULL UNION RESULT ALL NULL NULL NULL NULL NULL -explain format=json select * from v4_union as v,t2 where +explain format=json select * from v4_union as v,t2 where (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); EXPLAIN { @@ -6757,7 +6757,7 @@ EXPLAIN # using several derived tables : pushing in all tables # extracted or formula : pushing into WHERE # conjunctive subformula : pushing into WHERE -# pushing equalities +# pushing equalities set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v4,v2 where (((v4.a<12) and (v4.b>13)) or (v4.a>10)) and (v4.min_c=v2.max_c) and (v4.min_c>100); @@ -7051,7 +7051,7 @@ select * from (select a from t1 order by a limit 5) t where t.a not in (1,2,3); a 0 4 -set statement optimizer_switch='condition_pushdown_for_derived=on' for +set statement optimizer_switch='condition_pushdown_for_derived=on' for select * from (select a from t1 order by a limit 5) t where t.a not in (1,2,3); a 0 @@ -7065,7 +7065,7 @@ a 3 6 set statement optimizer_switch='condition_pushdown_for_derived=off' for -select * from +select * from (select a from t1 where a < 4 union select a from t1 where a > 5 order by a limit 5) t where t.a not in (2,9); a @@ -7073,8 +7073,8 @@ a 1 3 6 -set statement optimizer_switch='condition_pushdown_for_derived=on' for -select * from +set statement optimizer_switch='condition_pushdown_for_derived=on' for +select * from (select a from t1 where a < 4 union select a from t1 where a > 5 order by a limit 5) t where t.a not in (2,9); a @@ -7092,27 +7092,27 @@ CREATE TABLE t2 (b INT) ENGINE=MyISAM; CREATE OR REPLACE VIEW v2 AS SELECT * FROM t2; CREATE TABLE t3 (c INT) ENGINE=MyISAM; CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v3 AS SELECT * FROM t3; -SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT c FROM v3 WHERE c = a -) +SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT c FROM v3 WHERE c = a +) ); a INSERT INTO t1 VALUES (2); INSERT INTO t2 VALUES (3), (2); INSERT INTO t3 VALUES (4), (1), (2), (7); -SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT c FROM v3 WHERE c = a -) +SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT c FROM v3 WHERE c = a +) ); a 2 EXPLAIN FORMAT=JSON -SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT c FROM v3 WHERE c = a -) +SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT c FROM v3 WHERE c = a +) ); EXPLAIN { @@ -7167,36 +7167,36 @@ EXPLAIN } CREATE TABLE t4 (d INT, e INT) ENGINE=MyISAM; INSERT INTO t4 VALUES (1,10),(3,11),(2,10),(2,20),(3,21); -CREATE OR REPLACE VIEW v4 AS +CREATE OR REPLACE VIEW v4 AS SELECT d, sum(e) AS s FROM t4 GROUP BY d; -set statement optimizer_switch='condition_pushdown_for_derived=off' for SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT d FROM v4 WHERE s > a -) +set statement optimizer_switch='condition_pushdown_for_derived=off' for SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT d FROM v4 WHERE s > a +) ); a 2 -SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT d FROM v4 WHERE s > a -) +SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT d FROM v4 WHERE s > a +) ); a 2 -explain SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT d FROM v4 WHERE s > a -) +explain SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT d FROM v4 WHERE s > a +) ); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 system NULL NULL NULL NULL 1 1 PRIMARY t2 ALL NULL NULL NULL NULL 2 Using where; FirstMatch(t1) 3 DEPENDENT SUBQUERY index_subquery key0 key0 5 func 2 Using where 5 DERIVED t4 ALL NULL NULL NULL NULL 5 Using temporary; Using filesort -explain format=json SELECT * FROM t1 WHERE a IN ( -SELECT b FROM v2 WHERE b < a OR b IN ( -SELECT d FROM v4 WHERE s > a -) +explain format=json SELECT * FROM t1 WHERE a IN ( +SELECT b FROM v2 WHERE b < a OR b IN ( +SELECT d FROM v4 WHERE s > a +) ); EXPLAIN { @@ -7265,12 +7265,12 @@ INSERT INTO t1 VALUES (1); CREATE TABLE t2 (b INT) ENGINE=MyISAM; INSERT INTO t2 VALUES (3),(4); CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v2 AS SELECT * FROM t2; -SELECT * FROM +SELECT * FROM ( SELECT * FROM t1 WHERE EXISTS ( SELECT * FROM v2 WHERE b = a ) ) AS sq; a EXPLAIN FORMAT=JSON -SELECT * FROM +SELECT * FROM ( SELECT * FROM t1 WHERE EXISTS ( SELECT * FROM v2 WHERE b = a ) ) AS sq; EXPLAIN @@ -7315,8 +7315,8 @@ EXPLAIN DROP VIEW v2; DROP TABLE t1,t2; # -# MDEV-11102: condition pushdown into materialized inner table -# of outer join is not applied as not being valid +# MDEV-11102: condition pushdown into materialized inner table +# of outer join is not applied as not being valid # CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (0),(2); @@ -7415,19 +7415,19 @@ DROP TABLE t1; CREATE TABLE t1 (pk1 INT PRIMARY KEY, a INT, b INT) ENGINE=MyISAM; INSERT INTO t1 VALUES (10,7,1),(11,0,2); CREATE TABLE t2 (pk2 INT PRIMARY KEY, c INT, d DATETIME) ENGINE=MyISAM; -INSERT INTO t2 VALUES +INSERT INTO t2 VALUES (1,4,'2008-09-27 00:34:58'), (2,5,'2007-05-28 00:00:00'), (3,6,'2009-07-25 09:21:20'); CREATE VIEW v1 AS SELECT * FROM t1; CREATE ALGORITHM=TEMPTABLE VIEW v2 AS SELECT * FROM t2; -SELECT * FROM v1 AS sq +SELECT * FROM v1 AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; pk1 a b 10 7 1 11 0 2 EXPLAIN FORMAT=JSON -SELECT * FROM v1 AS sq +SELECT * FROM v1 AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; EXPLAIN { @@ -7471,13 +7471,13 @@ EXPLAIN ] } } -SELECT * FROM ( SELECT * FROM t1 ) AS sq +SELECT * FROM ( SELECT * FROM t1 ) AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; pk1 a b 10 7 1 11 0 2 EXPLAIN FORMAT=JSON -SELECT * FROM ( SELECT * FROM t1 ) AS sq +SELECT * FROM ( SELECT * FROM t1 ) AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; EXPLAIN { @@ -8249,18 +8249,18 @@ d DROP VIEW v1; DROP TABLE t1,t2; # -# MDEV-11820: second execution of PS for query -# with false subquery predicate in WHERE +# MDEV-11820: second execution of PS for query +# with false subquery predicate in WHERE # CREATE TABLE t1 (c VARCHAR(3)) ENGINE=MyISAM; INSERT INTO t1 VALUES ('foo'),('bar'); CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; CREATE TABLE t2 (a INT); INSERT INTO t2 VALUES (3), (4); -PREPARE stmt1 FROM +PREPARE stmt1 FROM " SELECT * FROM v1 WHERE 1 IN (SELECT a FROM t2) OR c = 'foo'"; -PREPARE stmt2 FROM -"EXPLAIN FORMAT=JSON +PREPARE stmt2 FROM +"EXPLAIN FORMAT=JSON SELECT * FROM v1 WHERE 1 IN (SELECT a FROM t2) OR c = 'foo'"; EXECUTE stmt1; c @@ -8358,7 +8358,7 @@ DEALLOCATE PREPARE stmt2; DROP VIEW v1; DROP TABLE t1,t2; # -# MDEV-12373: pushdown into derived with side effects is prohibited +# MDEV-12373: pushdown into derived with side effects is prohibited # CREATE TABLE sales_documents ( id int NOT NULL AUTO_INCREMENT, @@ -8374,9 +8374,9 @@ synced_at int NOT NULL DEFAULT '0', sum decimal(13,2) NOT NULL DEFAULT '0', PRIMARY KEY (id) ); -INSERT INTO sales_documents -(id, sale_id, type, order_number, data, created_at, -updated_at, date, generated, synced_at, sum) +INSERT INTO sales_documents +(id, sale_id, type, order_number, data, created_at, +updated_at, date, generated, synced_at, sum) VALUES (555, 165, 3, 5, '{}', 1486538300, 1486722835, '2017-02-17', 0, 1486538313, 2320.00), (556, 165, 2, 3, '{}', 1486538304, 1486563125, '2017-02-08', 1, 1486538302, 2320.00), @@ -8455,7 +8455,7 @@ row order_number 14 51 DROP TABLE sales_documents; # -# MDEV-12845: pushdown from merged derived using equalities +# MDEV-12845: pushdown from merged derived using equalities # create table t1 (a int); insert into t1 values @@ -9810,7 +9810,7 @@ EXPLAIN DROP TABLE t1; # # MDEV-15087: error from inexpensive subquery before check -# for condition pushdown into derived +# for condition pushdown into derived # CREATE TABLE t1 (i1 int, v1 varchar(1)); INSERT INTO t1 VALUES (7,'x'); @@ -10387,3 +10387,4 @@ f 3 DROP VIEW v1; DROP TABLE t1; +# End of 10.2 tests diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 476a3d795b9..46aa898c960 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1,46 +1,46 @@ -let $no_pushdown= set statement optimizer_switch='condition_pushdown_for_derived=off' for; +let $no_pushdown= set statement optimizer_switch='condition_pushdown_for_derived=off' for; create table t1 (a int, b int, c int); create table t2 (a int, b int, c int, d decimal); -insert into t1 values +insert into t1 values (1,21,345), (1,33,7), (8,33,114), (1,21,500), (1,19,107), (5,14,787), (8,33,123), (9,10,211), (5,16,207), (1,33,988), (5,27,132), (1,21,104), (6,20,309), (6,20,315), (1,21,101), (8,33,404), (9,10,800), (1,21,123), (7,11,708), (6,20,214); insert into t2 values - (2,3,207,207.0000), (1,21,909,12.0000), (7,13,312,406.0000), + (2,3,207,207.0000), (1,21,909,12.0000), (7,13,312,406.0000), (8,64,248,107.0000), (6,20,315,279.3333), (1,19,203,107.0000), (8,80,800,314.0000), (3,12,231,190.0000), (6,23,303,909.0000); - + create table t1_double(a int, b double, c double); -insert into t1_double values +insert into t1_double values (1,23.4,14.3333), (1,12.5,18.9), (3,12.5,18.9), (4,33.4,14.3333), (4,14.3333,13.65), (5,17.89,7.22), (6,33.4,14.3), (10,33.4,13.65), (11,33.4,13.65); - + create table t2_double(a int, b double, c double); -insert into t2_double values +insert into t2_double values (1,22.4,14.3333), (1,12.5,18.9), (2,22.4,18.9), (4,33.4,14.3333), (5,22.4,13.65), (7,17.89,18.9), - (6,33.4,14.3333), (10,31.4,13.65), (12,33.4,13.65); - + (6,33.4,14.3333), (10,31.4,13.65), (12,33.4,13.65); + create table t1_char(a char, b char(8), c int); insert into t1_char values ('a','Ivan',1), ('b','Vika',2), ('b','Inga',6), ('c','Vika',7), ('b','Ivan',7), ('a','Alex',6), ('b','Inga',5), ('d','Ron',9), - ('d','Harry',2), ('d','Hermione',3), ('c','Ivan',3), ('c','Harry',4); - + ('d','Harry',2), ('d','Hermione',3), ('c','Ivan',3), ('c','Harry',4); + create table t2_char(a char, b char(8), c int); insert into t2_char values ('b','Ivan',1), ('c','Vinny',3), ('c','Inga',9), ('a','Vika',1), ('c','Ivan',2), ('b','Ali',6), ('c','Inga',2), ('a','Ron',9), ('d','Harry',1), ('b','Hermes',3), ('b','Ivan',11), ('b','Harry',4); - + create table t1_decimal (a decimal(3,1), b decimal(3,1), c int); insert into t1_decimal values (1,1,23),(2,2,11),(3,3,16), (1,1,12),(1,1,14),(2,3,15), - (2,1,13),(2,3,11),(3,3,16); + (2,1,13),(2,3,11),(3,3,16); create table t2_decimal (a decimal(3,1), b decimal(3,1), c int); insert into t2_decimal values @@ -48,27 +48,27 @@ insert into t2_decimal values (1,3,22),(1,3,14),(2,2,15), (2,1,43),(2,3,11),(2,3,16); -create view v1 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 +create view v1 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 group by a,b having max_c < 707; - -create view v2 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 - where t1.a>5 group by a,b having max_c < 707; - + +create view v2 as select a, b, max(c) as max_c, avg(c) as avg_c from t1 + where t1.a>5 group by a,b having max_c < 707; + create view v3 as select a, b, min(c) as min_c from t1 where t1.a<10 group by a,b having min_c > 109; - + create view v4 as select a, b, min(max_c) as min_c from v1 - where (v1.a<15) group by a,b; - -create view v_union as + where (v1.a<15) group by a,b; + +create view v_union as select a, b, min(c) as c from t1 where t1.a<10 group by a,b having c > 109 union select a, b, max(c) as c from t1 - where t1.b>10 group by a,b having c < 300; - -create view v2_union as + where t1.b>10 group by a,b having c < 300; + +create view v2_union as select a, b, min(c) as c from t1 where t1.a<10 group by a,b having c > 109 union @@ -76,117 +76,117 @@ create view v2_union as where t1.b>10 group by a,b having c < 300 union select a, b, avg(c) as c from t1 - where t1.c>300 group by a,b having c < 707; + where t1.c>300 group by a,b having c < 707; -create view v3_union as +create view v3_union as select a, b, (a+1) as c from t1 where t1.a<10 union select a, b, c from t1 - where t1.b>10 and t1.c>100; - -create view v4_union as + where t1.b>10 and t1.c>100; + +create view v4_union as select a, b, max(c)-100 as c from t1 where t1.a<10 group by a,b having c > 109 union select a, b, (c+100) as c from t1 - where t1.b>10; + where t1.b>10; create view v_double as select a, avg(a/4) as avg_a, b, c from t1_double - where (b>12.2) group by b,c having (avg_a<22.333); - -create view v_char as - select a, b, max(c) as max_c from t1_char - group by a,b having max_c < 9; - + where (b>12.2) group by b,c having (avg_a<22.333); + +create view v_char as + select a, b, max(c) as max_c from t1_char + group by a,b having max_c < 9; + create view v_decimal as select a, b, avg(c) as avg_c from t1_decimal - group by a,b having (avg_c>12); + group by a,b having (avg_c>12); ---echo # conjunctive subformula : pushing into HAVING +--echo # conjunctive subformula : pushing into HAVING let $query= select * from v1,t2 where (v1.max_c>214) and (t2.a>v1.a); eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; -let $query= +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 - group by a,b having max_c < 707) v1, + group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; ---echo # extracted or formula : pushing into HAVING -let $query= - select * from v1,t2 where +--echo # extracted or formula : pushing into HAVING +let $query= + select * from v1,t2 where ((v1.max_c>400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or + ((v1.max_c>300) and (v1.avg_c>t2.d) and (v1.b=t2.b)) or ((v1.max_c<135) and (v1.max_c6) and (t2.b>v1.b); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; let $query= select * from v2,t2 where (v2.b>25) and (t2.a7) and (t2.c7) and (t2.c5) and (t2.b4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or +eval explain format=json $query; + +let $query= + select * from v1,t2 where + ((v1.a>4) and (v1.b>t2.b) and (v1.max_c=t2.d)) or ((v1.a<2) and (v1.max_c400) and (t2.b>v1.b); +let $query= + select * from v1,t2 where (v1.a<2) and (v1.max_c>400) and (t2.b>v1.b); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; -let $query= +let $query= select * from v_double as v,t2_double as t where (v.a=t.a) and (v.avg_a>0.45) and (v.b>10); eval $no_pushdown $query; @@ -194,57 +194,57 @@ eval $query; eval explain $query; eval explain format=json $query; -let $query= +let $query= select * from v_decimal as v,t2_decimal as t where (v.a=t.a) and (v.avg_c>15) and (v.b>1); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # extracted or formula : pushing into HAVING and WHERE -let $query= - select * from v1,t2 where - ((v1.a>7) and (v1.max_c>300) and (t2.c7) and (v1.max_c>300) and (t2.c120)) or (v1.a>7); eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; - ---echo # extracted or formulas : pushing into WHERE and HAVING -let $query= - select * from v1,t2 where + +--echo # extracted or formulas : pushing into WHERE and HAVING +let $query= + select * from v1,t2 where ((v1.a<2) and (v1.max_c>120) and (v1.b=t2.b)) or (v1.a>7); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - -let $query= +eval explain format=json $query; + +let $query= select * from v1,t2 where ((v1.a<2) and (v1.max_c<200) and (t2.c>v1.max_c) and (v1.max_c=t2.d)) or - ((v1.a>4) and (v1.max_c<500) and (t2.b4) and (v1.max_c<500) and (t2.b400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a400) and (t2.a>v1.a)) or ((v1.max_c<135) and (t2.a150) and (v1.max_c=t2.c); eval $no_pushdown $query; @@ -286,73 +286,73 @@ eval $query; eval explain $query; eval explain format=json $query; ---echo # extracted and formula : pushing into WHERE +--echo # extracted and formula : pushing into WHERE --echo # pushing equalities -let $query= +let $query= select * from v1,t2 where (v1.a=v1.b) and (v1.a=t2.a) and (v1.a=3); eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; - -let $query= + +let $query= select * from v1,t2 where (v1.a=1) and (v1.b=21) and (t2.a=2); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; -let $query= - select * from v_char as v,t2_char as t where - (v.a='c') and (v.b<'Hermes') and ((v.b=t.b) or (v.max_c>20)); +let $query= + select * from v_char as v,t2_char as t where + (v.a='c') and (v.b<'Hermes') and ((v.b=t.b) or (v.max_c>20)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # extracted and formula : pushing into WHERE using equalities --echo # pushing equalities -let $query= +let $query= select * from v_decimal as v,t2_decimal as t where (v.a=v.b) and (v.b=t.b) and ((t.b>1) or (v.a=1)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; ---echo # extracted or formula : pushing into HAVING using equalities -let $query= +--echo # extracted or formula : pushing into HAVING using equalities +let $query= select * from v1,t2 - where ((t2.a<4) and (v1.a=t2.a)) or ((t2.c>150) and (v1.max_c=t2.c)); + where ((t2.a<4) and (v1.a=t2.a)) or ((t2.c>150) and (v1.max_c=t2.c)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # conjunctive subformulas : pushing into WHERE and HAVING using equalities -let $query= +let $query= select * from v1,t2 where ((t2.a>5) and (v1.a=t2.a)) and ((t2.c>250) and (v1.max_c=t2.c)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # conjunctive subformulas : pushing into WHERE and HAVING ---echo # pushing equalities -let $query= +--echo # pushing equalities +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 - group by a,b having max_c < 707) v1, + group by a,b having max_c < 707) v1, t2 where (v1.a=8) and (v1.a=t2.a) and (v1.max_c=404); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - ---echo # conjunctive subformulas : pushing into WHERE and HAVING -let $query= - select * from v1,t2 where +eval explain format=json $query; + +--echo # conjunctive subformulas : pushing into WHERE and HAVING +let $query= + select * from v1,t2 where (v1.a>3) and (v1.max_c>200) and (t2.b10) or (v.a=1)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # conjunctive subformula : pushing into WHERE ---echo # extracted or formula : pushing into HAVING using equalities -let $query= +--echo # extracted or formula : pushing into HAVING using equalities +let $query= select * from v_double as v,t2_double as t where (((v.a>0.2) or (v.b<17)) or (t.c>17)) and (t.c=v.c) and (v.c>18); eval $no_pushdown $query; @@ -383,146 +383,146 @@ eval explain format=json $query; --echo # extracted or formula : pushing into WHERE --echo # conjunctive subformula : pushing into HAVING --echo # pushing equalities -let $query= +let $query= select * from v_decimal as v,t2_decimal as t where - (((v.a>4) or (v.a=2)) or (v.b>3)) and (v.avg_c=13); + (((v.a>4) or (v.a=2)) or (v.b>3)) and (v.avg_c=13); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; -let $query= +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 - where t1.a>5 group by a,b having max_c < 707) v1, + where t1.a>5 group by a,b having max_c < 707) v1, t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.a=v1.b); eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; - + --echo # nothing to push -let $query= +let $query= select * from v1,t2 where (t2.a<2) and (t2.c>900); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.b=t2.b); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - -let $query= - select * from v1,t2 where - (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); +eval explain format=json $query; + +let $query= + select * from v1,t2 where + (t2.a=v1.a) or (v1.b=t2.b) and ((v1.a=1) or (v1.a=6)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - -let $query= +eval explain format=json $query; + +let $query= select * from v1,t2 where (v1.a=1) or (v1.b=21) or (t2.a=2); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; -let $query= - select * from v1,t2 where +let $query= + select * from v1,t2 where (t2.a<2) and (t2.c>900) and ((v1.a13) and (t2.c<115)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using several derived tables : pushing in all tables --echo # conjunctive subformula : pushing into HAVING --echo # extracted or formula : pushing into WHERE ---echo # pushing equalities -let $query= - select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and +--echo # pushing equalities +let $query= + select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and ((v2.b<50) or (v2.b=19)) and (v1.max_c<300); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # using several derived tables : pushing only in one table --echo # conjunctive subformula : pushing into WHERE --echo # pushing equalities -let $query= - select * from v1,v2,t2 where - (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); +let $query= + select * from v1,v2,t2 where + (v1.a=t2.a) and (v1.a=v1.b) and (v1.a=v2.a) and (v2.max_c<300); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using several derived tables : pushing only in one table --echo # extracted and formula : pushing into WHERE --echo # conjunctive subformula : pushing into WHERE using equalities --echo # pushing equalities -let $query= +let $query= select * from v1,v2,t2 where (v1.a=1) and (v1.b>10) and (v1.b=v2.b); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # extracted or formula : pushing into WHERE --echo # conjunctive subformula : pushing into WHERE using equalities --echo # pushing equalities -let $query= - select * from v_char as v,t2_char as t where +let $query= + select * from v_char as v,t2_char as t where (v.a=t.a) and (t.a='b') and ((v.b='Vika') or (v.b='Ali')); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using several derived tables : pushing in all tables --echo # extracted or formula : pushing into WHERE ---echo # conjunctive subformulas : pushing into HAVING +--echo # conjunctive subformulas : pushing into HAVING --echo # pushing equalities -let $query= - select * from v1,v2,v3,t2 where - ((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) +let $query= + select * from v1,v2,v3,t2 where + ((v1.a=v2.a) or (v1.a=t2.a)) and ((v3.b<50) or (v3.b=33)) and (v1.max_c<500) and (v3.a=t2.a) and (v2.max_c>300); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using several derived tables : pushing in all tables ---echo # conjunctive subformulas : pushing into HAVING -let $query= +--echo # conjunctive subformulas : pushing into HAVING +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 where t1.a>5 group by a,b having max_c < 707) v1, @@ -532,11 +532,11 @@ let $query= eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using several derived tables : pushing in all tables --echo # extracted or formulas : pushing into HAVING ---echo # conjunctive subformula : pushing into HAVING +--echo # conjunctive subformula : pushing into HAVING let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 @@ -550,51 +550,51 @@ let $query= eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # extracted or formula : pushing into HAVING ---echo # conjunctive subformula : pushing into WHERE -let $query= +--echo # conjunctive subformula : pushing into WHERE +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 - group by a,b having max_c < 707) v1, - t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); + group by a,b having max_c < 707) v1, + t2 where ((v1.a=1) or (v1.max_c<300)) and (v1.b>25); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # extracted and formula : pushing into WHERE ---echo # conjunctive subformula : pushing into HAVING -let $query= +--echo # conjunctive subformula : pushing into HAVING +let $query= select * from (select a, b, max(c) as max_c, avg(c) as avg_c from t1 - where t1.a>5 group by a,b having max_c < 707) v1, - t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); + where t1.a>5 group by a,b having max_c < 707) v1, + t2 where (v1.a=t2.a) and (v1.max_c>300) and (v1.b<30); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using query with union --echo # conjunctive subformula : pushing into WHERE --echo # conjunctive subformulas : pushing into HAVING and WHERE -let $query= +let $query= select * from v1,t2 where (v1.a<5) and (v1.b=t2.b) and (t2.c>800) union - select * from v1,t2 where (v1.max_c>100) and (v1.a>7) and (t2.d>800); + select * from v1,t2 where (v1.max_c>100) and (v1.a>7) and (t2.d>800); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using query with union ---echo # extracted and formula : pushing into WHERE ---echo # extracted or formula : pushing into HAVING +--echo # extracted and formula : pushing into WHERE +--echo # extracted or formula : pushing into HAVING --echo # pushing equalities -let $query= +let $query= select * from v1,t2 where (v1.a<5) and (v1.b=t2.b) and (v1.b=19) - union + union select * from v1,t2 where ((v1.max_c>400) or (v1.avg_c>270)) and (v1.a3) and (v1.b>27)) or (v1.max_c>550); + select * from v1,t2 where ((v1.a>3) and (v1.b>27)) or (v1.max_c>550); eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; --echo # using query with union ---echo # extracted or formula : pushing into HAVING ---echo # conjunctive subformulas : pushing into WHERE +--echo # extracted or formula : pushing into HAVING +--echo # conjunctive subformulas : pushing into WHERE --echo # pushing equalities -let $query= - select * from v1,t2 where +let $query= + select * from v1,t2 where ((v1.a=1) and (v1.a=t2.a)) and ((v1.max_c<500) or (v1.avg_c>500)) union select * from v2,t2 where @@ -643,8 +643,8 @@ eval explain format=json $query; --echo # using derived table with union --echo # conjunctive subformula : pushing into WHERE ---echo # extracted or formula : pushing into HAVING -let $query= +--echo # extracted or formula : pushing into HAVING +let $query= select * from v_union,t2 where ((v_union.a<2) or (v_union.c>800)) and (v_union.b>12); eval $no_pushdown $query; @@ -656,7 +656,7 @@ eval explain format=json $query; --echo # conjunctive subformula : pushing into HAVING --echo # conjunctive subformula : pushing into WHERE --echo # pushing equalities -let $query= +let $query= select * from v_union,t2 where (v_union.a=1) and (v_union.a=t2.a) and (v_union.c<200); eval $no_pushdown $query; @@ -664,220 +664,220 @@ eval $query; eval explain $query; eval explain format=json $query; -let $query= - select * from v_char as v,t2_char as t where +let $query= + select * from v_char as v,t2_char as t where (v.a=t.a) and (v.b='Vika') and (v.max_c>2); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - ---echo # using derived table with union ---echo # using several derived tables : pushing in all tables ---echo # conjunctive subformula : pushing into WHERE using equalities ---echo # pushing equalities -let $query= - select * from v_union,v1,t2 where - (v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) - and ((v_union.c>800) or (v1.max_c>200)); -eval $no_pushdown $query; -eval $query; -eval explain $query; eval explain format=json $query; ---echo # using derived table with union ---echo # extracted or formula : pushing into WHERE +--echo # using derived table with union +--echo # using several derived tables : pushing in all tables +--echo # conjunctive subformula : pushing into WHERE using equalities +--echo # pushing equalities +let $query= + select * from v_union,v1,t2 where + (v_union.a=v1.a) and (v1.a=t2.a) and (t2.a=1) + and ((v_union.c>800) or (v1.max_c>200)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using derived table with union +--echo # extracted or formula : pushing into WHERE --echo # conjunctive subformula : pushing into HAVING --echo # pushing equalities -let $query= - select * from v2_union as v,t2 where +let $query= + select * from v2_union as v,t2 where ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using derived table with union of selects without aggregation --echo # extracted conjunctive predicate: pushing in WHERE of both selects -let $query= +let $query= select * from v3_union as v,t2 where (v.a=t2.a) and (v.c>6); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using derived table with union of selects without aggregation --echo # extracted conjunctive OR subformula: pushing in WHERE using equalities -let $query= +let $query= select * from v3_union as v,t2 where (v.a=t2.a) and ((t2.a>1) or (v.b<20)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using derived table with union of selects without aggregation --echo # extracted the whole condition: in WHERE of both selects -let $query= - select * from v3_union as v,t2 where +let $query= + select * from v3_union as v,t2 where (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; ---echo # using derived table with union of +--echo # using derived table with union of --echo # a select without aggregation and a select with aggregation --echo # extracted conjunctive predicate: pushing in WHERE of both selects -let $query= +let $query= select * from v4_union as v,t2 where (v.a=t2.a) and (v.b<20); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; ---echo # using derived table with union of +--echo # using derived table with union of --echo # a select without aggregation and a select with aggregation --echo # extracted subformula: pushing in WHERE of one select --echo # extracted subformula: pushing in HAVING of the other select --echo # extracted sub-subformula: pushing in WHERE of the other select --echo # using an equality in all pushdowns -let $query= - select * from v4_union as v,t2 where +let $query= + select * from v4_union as v,t2 where (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded derived table : pushing the same conditions --echo # using several derived tables : pushing in all tables --echo # conjunctive subformula : pushing into WHERE --echo # extracted and formula : pushing into WHERE -let $query= +let $query= select * from v4,v1 where - (v4.a<13) and (v1.a>5) and (v1.b>12); + (v4.a<13) and (v1.a>5) and (v1.b>12); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded view : nothing to push --echo # using several derived tables : pushing only in one table --echo # conjunctive subformula : pushing into WHERE -let $query= +let $query= select * from v4,v1,t2 where (v4.a=t2.a) and (v4.a=v1.a) and (v1.b>30); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded view : pushing different conditions --echo # using several derived tables : pushing in all tables --echo # conjunctive subformula : pushing into WHERE using equalities --echo # extracted and formula : pushing into WHERE using equalities --echo # conjunctive subformula : pushing into HAVING -let $query= +let $query= select * from v4,v1,t2 where - (v4.a=t2.a) and (v4.a>1) and (v4.a=v1.a) and (v4.min_c>100) and (v1.b<30); + (v4.a=t2.a) and (v4.a>1) and (v4.a=v1.a) and (v4.min_c>100) and (v1.b<30); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded view : pushing different conditions --echo # using several derived tables : pushing in all tables --echo # extracted or formula : pushing into WHERE --echo # conjunctive subformula : pushing into HAVING -let $query= +let $query= select * from v4,v1,t2 where (((v4.b>10) and (v4.a>1)) or (v4.b<20)) and (v1.max_c>200) and (v1.a=v4.a); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # using embedded view : pushing different conditions --echo # using several derived tables : pushing only in one table --echo # extracted or formula : pushing into WHERE --echo # extracted or formula : pushing into HAVING -let $query= +let $query= select * from v4,v2 where ((v4.a>12) and (v4.min_c<300) and (v4.b>13)) or (v4.a<1); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # using embedded view : pushing different conditions --echo # using several derived tables : pushing only in one table --echo # conjunctive subformula : pushing into WHERE --echo # conjunctive subformula : pushing into HAVING --echo # pushing equalities -let $query= +let $query= select * from v4,v2 where - (v4.a=v2.b) and (v4.a=v4.b) and (v4.min_c<100); + (v4.a=v2.b) and (v4.a=v4.b) and (v4.min_c<100); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # using embedded view : pushing the same conditions --echo # using several derived tables : pushing in all tables --echo # extracted and formula : pushing into WHERE using equalities --echo # conjunctive subformula : pushing into WHERE --echo # pushing equalities -let $query= +let $query= select * from v4,v2 where - (v4.a=v2.b) and (v4.a=v4.b) and (v2.b<30); + (v4.a=v2.b) and (v4.a=v4.b) and (v2.b<30); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded view : pushing the same conditions --echo # using several derived tables : pushing in all tables --echo # extracted or formula : pushing into WHERE using equalities --echo # extracted and formula : pushing into WHERE using equalities --echo # pushing equalities -let $query= +let $query= select * from v4,v2 where (v4.a=v2.b) and (v4.a=v4.b) and ((v2.b<30) or (v4.a>2)); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + --echo # using embedded view : pushing the same conditions --echo # using several derived tables : pushing in all tables --echo # extracted or formula : pushing into WHERE --echo # conjunctive subformula : pushing into WHERE ---echo # pushing equalities -let $query= +--echo # pushing equalities +let $query= select * from v4,v2 where (((v4.a<12) and (v4.b>13)) or (v4.a>10)) and (v4.min_c=v2.max_c) and (v4.min_c>100); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; --echo # using embedded view : pushing the same conditions --echo # using several derived tables : pushing only in one table --echo # extracted or formula : pushing into WHERE -let $query= +let $query= select * from v4,v2,t2 where (((v4.a<12) and (t2.b>13)) or (v4.a>10)) and (v4.min_c=t2.c) and (t2.c>100); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; - +eval explain format=json $query; + drop view v1,v2,v3,v4; drop view v_union,v2_union,v3_union,v4_union; drop view v_double,v_char,v_decimal; -drop table t1,t2,t1_double,t2_double,t1_char,t2_char,t1_decimal,t2_decimal; +drop table t1,t2,t1_double,t2_double,t1_char,t2_char,t1_decimal,t2_decimal; --echo # --echo # MDEV-10782: condition extracted from a multiple equality @@ -906,8 +906,8 @@ CREATE TABLE t1 (i int) ENGINE=MyISAM; CREATE VIEW v AS SELECT 5; SELECT * FROM t1 WHERE 1 IN ( SELECT * FROM v ); DROP VIEW v; -DROP TABLE t1; - +DROP TABLE t1; + --echo # --echo # MDEV-10785: second execution of a query with condition --echo # pushed into view @@ -936,17 +936,17 @@ select a from t1 order by a limit 5; set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from (select a from t1 order by a limit 5) t where t.a not in (1,2,3); -set statement optimizer_switch='condition_pushdown_for_derived=on' for +set statement optimizer_switch='condition_pushdown_for_derived=on' for select * from (select a from t1 order by a limit 5) t where t.a not in (1,2,3); select a from t1 where a < 4 union select a from t1 where a > 5 order by a limit 5; set statement optimizer_switch='condition_pushdown_for_derived=off' for -select * from +select * from (select a from t1 where a < 4 union select a from t1 where a > 5 order by a limit 5) t where t.a not in (2,9); -set statement optimizer_switch='condition_pushdown_for_derived=on' for -select * from +set statement optimizer_switch='condition_pushdown_for_derived=on' for +select * from (select a from t1 where a < 4 union select a from t1 where a > 5 order by a limit 5) t where t.a not in (2,9); @@ -963,45 +963,45 @@ CREATE OR REPLACE VIEW v2 AS SELECT * FROM t2; CREATE TABLE t3 (c INT) ENGINE=MyISAM; CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v3 AS SELECT * FROM t3; -SELECT * FROM t1 WHERE a IN ( - SELECT b FROM v2 WHERE b < a OR b IN ( - SELECT c FROM v3 WHERE c = a - ) +SELECT * FROM t1 WHERE a IN ( + SELECT b FROM v2 WHERE b < a OR b IN ( + SELECT c FROM v3 WHERE c = a + ) ); INSERT INTO t1 VALUES (2); INSERT INTO t2 VALUES (3), (2); INSERT INTO t3 VALUES (4), (1), (2), (7); -SELECT * FROM t1 WHERE a IN ( - SELECT b FROM v2 WHERE b < a OR b IN ( - SELECT c FROM v3 WHERE c = a - ) +SELECT * FROM t1 WHERE a IN ( + SELECT b FROM v2 WHERE b < a OR b IN ( + SELECT c FROM v3 WHERE c = a + ) ); EXPLAIN FORMAT=JSON -SELECT * FROM t1 WHERE a IN ( - SELECT b FROM v2 WHERE b < a OR b IN ( - SELECT c FROM v3 WHERE c = a - ) +SELECT * FROM t1 WHERE a IN ( + SELECT b FROM v2 WHERE b < a OR b IN ( + SELECT c FROM v3 WHERE c = a + ) ); CREATE TABLE t4 (d INT, e INT) ENGINE=MyISAM; INSERT INTO t4 VALUES (1,10),(3,11),(2,10),(2,20),(3,21); -CREATE OR REPLACE VIEW v4 AS +CREATE OR REPLACE VIEW v4 AS SELECT d, sum(e) AS s FROM t4 GROUP BY d; -let $query = -SELECT * FROM t1 WHERE a IN ( - SELECT b FROM v2 WHERE b < a OR b IN ( - SELECT d FROM v4 WHERE s > a - ) +let $query = +SELECT * FROM t1 WHERE a IN ( + SELECT b FROM v2 WHERE b < a OR b IN ( + SELECT d FROM v4 WHERE s > a + ) ); eval $no_pushdown $query; eval $query; eval explain $query; -eval explain format=json $query; +eval explain format=json $query; DROP VIEW v2,v3,v4; DROP TABLE t1,t2,t3,t4; @@ -1014,17 +1014,17 @@ DROP TABLE t1,t2,t3,t4; CREATE TABLE t1 (a INT) ENGINE=MyISAM; INSERT INTO t1 VALUES (1); - + CREATE TABLE t2 (b INT) ENGINE=MyISAM; INSERT INTO t2 VALUES (3),(4); CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v2 AS SELECT * FROM t2; - -SELECT * FROM + +SELECT * FROM ( SELECT * FROM t1 WHERE EXISTS ( SELECT * FROM v2 WHERE b = a ) ) AS sq; EXPLAIN FORMAT=JSON -SELECT * FROM +SELECT * FROM ( SELECT * FROM t1 WHERE EXISTS ( SELECT * FROM v2 WHERE b = a ) ) AS sq; @@ -1032,18 +1032,18 @@ DROP VIEW v2; DROP TABLE t1,t2; --echo # ---echo # MDEV-11102: condition pushdown into materialized inner table ---echo # of outer join is not applied as not being valid +--echo # MDEV-11102: condition pushdown into materialized inner table +--echo # of outer join is not applied as not being valid --echo # CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (0),(2); - + CREATE TABLE t2 (b INT); INSERT INTO t2 VALUES (1),(2); - + CREATE OR REPLACE ALGORITHM=TEMPTABLE VIEW v2 AS SELECT * FROM t2; - + SELECT * FROM t1 LEFT JOIN t2 ON a = b WHERE b IS NULL; SELECT * FROM t1 LEFT JOIN v2 ON a = b WHERE b IS NULL; @@ -1076,26 +1076,26 @@ DROP TABLE t1; CREATE TABLE t1 (pk1 INT PRIMARY KEY, a INT, b INT) ENGINE=MyISAM; INSERT INTO t1 VALUES (10,7,1),(11,0,2); - + CREATE TABLE t2 (pk2 INT PRIMARY KEY, c INT, d DATETIME) ENGINE=MyISAM; -INSERT INTO t2 VALUES +INSERT INTO t2 VALUES (1,4,'2008-09-27 00:34:58'), (2,5,'2007-05-28 00:00:00'), (3,6,'2009-07-25 09:21:20'); - + CREATE VIEW v1 AS SELECT * FROM t1; CREATE ALGORITHM=TEMPTABLE VIEW v2 AS SELECT * FROM t2; -SELECT * FROM v1 AS sq +SELECT * FROM v1 AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; EXPLAIN FORMAT=JSON -SELECT * FROM v1 AS sq +SELECT * FROM v1 AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; -SELECT * FROM ( SELECT * FROM t1 ) AS sq +SELECT * FROM ( SELECT * FROM t1 ) AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; EXPLAIN FORMAT=JSON -SELECT * FROM ( SELECT * FROM t1 ) AS sq +SELECT * FROM ( SELECT * FROM t1 ) AS sq WHERE b IN ( SELECT pk2 FROM v2 WHERE c > sq.b ) OR b = 100; DROP VIEW v1,v2; @@ -1155,7 +1155,7 @@ DROP TABLE t1,t2; CREATE TABLE t (pk INT PRIMARY KEY, f INT) ENGINE=MyISAM; CREATE ALGORITHM=TEMPTABLE VIEW v AS SELECT * FROM t; INSERT INTO t VALUES (1,1),(3,2); - + SELECT * FROM v AS v1, v AS v2 WHERE v2.pk > v1.f AND v1.f IN ( SELECT COUNT(pk) FROM t ); @@ -1297,10 +1297,10 @@ DROP TABLE t1; CREATE TABLE t1 (c VARCHAR(3)); CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; INSERT INTO t1 VALUES ('foo'),('bar'); - + CREATE TABLE t2 (c VARCHAR(3)); INSERT INTO t2 VALUES ('foo'),('xyz'); - + SELECT * FROM v1 WHERE v1.c IN ( SELECT MIN(c) FROM t2 WHERE 0 ); EXPLAIN FORMAT=JSON SELECT * FROM v1 WHERE v1.c IN ( SELECT MIN(c) FROM t2 WHERE 0 ); @@ -1311,18 +1311,18 @@ DROP TABLE t1,t2; CREATE TABLE t1 (d DECIMAL(10,2)); CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; INSERT INTO t1 VALUES (5.37),(1.1); - + CREATE TABLE t2 (d DECIMAL(10,2)); INSERT INTO t2 VALUES ('1.1'),('2.23'); - + SELECT * FROM v1 WHERE v1.d IN ( SELECT MIN(d) FROM t2 WHERE 0 ); DROP VIEW v1; DROP TABLE t1,t2; --echo # ---echo # MDEV-11820: second execution of PS for query ---echo # with false subquery predicate in WHERE +--echo # MDEV-11820: second execution of PS for query +--echo # with false subquery predicate in WHERE --echo # CREATE TABLE t1 (c VARCHAR(3)) ENGINE=MyISAM; @@ -1331,10 +1331,10 @@ CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; CREATE TABLE t2 (a INT); INSERT INTO t2 VALUES (3), (4); -PREPARE stmt1 FROM +PREPARE stmt1 FROM " SELECT * FROM v1 WHERE 1 IN (SELECT a FROM t2) OR c = 'foo'"; -PREPARE stmt2 FROM -"EXPLAIN FORMAT=JSON +PREPARE stmt2 FROM +"EXPLAIN FORMAT=JSON SELECT * FROM v1 WHERE 1 IN (SELECT a FROM t2) OR c = 'foo'"; EXECUTE stmt1; EXECUTE stmt2; @@ -1347,14 +1347,14 @@ INSERT INTO t2 SELECT a+1 FROM t2; EXECUTE stmt1; EXECUTE stmt2; DEALLOCATE PREPARE stmt1; -# the result here will change after the merge with the fix for mdev-11859 +# the result here will change after the merge with the fix for mdev-11859 DEALLOCATE PREPARE stmt2; DROP VIEW v1; DROP TABLE t1,t2; --echo # ---echo # MDEV-12373: pushdown into derived with side effects is prohibited +--echo # MDEV-12373: pushdown into derived with side effects is prohibited --echo # CREATE TABLE sales_documents ( @@ -1372,9 +1372,9 @@ CREATE TABLE sales_documents ( PRIMARY KEY (id) ); -INSERT INTO sales_documents -(id, sale_id, type, order_number, data, created_at, - updated_at, date, generated, synced_at, sum) +INSERT INTO sales_documents +(id, sale_id, type, order_number, data, created_at, + updated_at, date, generated, synced_at, sum) VALUES (555, 165, 3, 5, '{}', 1486538300, 1486722835, '2017-02-17', 0, 1486538313, 2320.00), (556, 165, 2, 3, '{}', 1486538304, 1486563125, '2017-02-08', 1, 1486538302, 2320.00), @@ -1450,11 +1450,11 @@ SELECT * FROM GROUP BY t.order_number ) a, (SELECT @row := 0) r) t WHERE row <> order_number; - + DROP TABLE sales_documents; --echo # ---echo # MDEV-12845: pushdown from merged derived using equalities +--echo # MDEV-12845: pushdown from merged derived using equalities --echo # create table t1 (a int); @@ -1602,7 +1602,7 @@ CREATE TABLE t2 (x INT, y INT, z INT); INSERT INTO t1 VALUES (1,1,66,1), (1,1,56,2), (3,2,42,3); INSERT INTO t2 VALUES (1,1,66), (1,12,32); -LET $query= +let $query= SELECT * FROM t2, ( @@ -1613,11 +1613,11 @@ FROM t2, ) AS v1 WHERE (v1.a=1) AND (v1.b=v1.a) AND (v1.a=t2.x) AND (v1.max_c>30); -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM t2, ( @@ -1628,9 +1628,9 @@ FROM t2, ) AS v1 WHERE (v1.a=1) AND (v1.b=v1.a) AND (v1.b=v1.d) AND (v1.a=t2.x) AND (v1.max_c>30); -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; DROP TABLE t1,t2; @@ -1656,7 +1656,7 @@ CREATE TABLE t2 (e INT, f INT, g INT); INSERT INTO t1 VALUES (1,14),(2,13),(1,19),(2,32),(3,24); INSERT INTO t2 VALUES (1,19,2),(3,24,1),(1,12,2),(3,11,3),(2,32,1); -LET $query= +let $query= SELECT * FROM t1 WHERE (t1.a,t1.b) IN ( @@ -1670,11 +1670,11 @@ WHERE (t1.a,t1.b) IN WHERE d_tab.e>1 ) ; -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM t1 WHERE (t1.a,t1.b) IN ( @@ -1688,11 +1688,11 @@ WHERE (t1.a,t1.b) IN WHERE d_tab.max_f<25 ) ; -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM t1 WHERE (t1.a,t1.b) IN ( @@ -1706,11 +1706,11 @@ WHERE (t1.a,t1.b) IN GROUP BY d_tab.g ) ; -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM t1 WHERE (t1.a,t1.b) IN ( @@ -1724,9 +1724,9 @@ WHERE (t1.a,t1.b) IN GROUP BY d_tab.g ) ; -EVAL $query; -EVAL EXPLAIN $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN $query; +eval EXPLAIN FORMAT=JSON $query; DROP TABLE t1,t2; @@ -1755,7 +1755,7 @@ CREATE TABLE t1 (a INT, b INT); INSERT INTO t1 VALUES (1,2),(2,3),(3,4); -LET $query= +let $query= SELECT * FROM ( SELECT t1.b AS a @@ -1763,10 +1763,10 @@ SELECT * FROM GROUP BY t1.a ) dt WHERE (dt.a=2); -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM ( SELECT t1.b AS a @@ -1775,10 +1775,10 @@ SELECT * FROM HAVING (t1.a<3) ) dt WHERE (dt.a>1); -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM ( SELECT 'ab' AS a @@ -1786,10 +1786,10 @@ SELECT * FROM GROUP BY t1.a ) dt WHERE (dt.a='ab'); -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM ( SELECT 1 AS a @@ -1797,8 +1797,8 @@ SELECT * FROM GROUP BY t1.a ) dt WHERE (dt.a=1); -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN FORMAT=JSON $query; DROP TABLE t1; @@ -1810,7 +1810,7 @@ DROP TABLE t1; CREATE TABLE t1 (a INT, b INT); INSERT INTO t1 VALUES (1,2),(1,3); -LET $query= +let $query= SELECT * FROM ( SELECT t1.a @@ -1825,10 +1825,10 @@ JOIN WHERE 1 IN (0,t1.a) ) AS dt2 ON dt1.a = dt2.a; -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; +eval $query; +eval EXPLAIN FORMAT=JSON $query; -LET $query= +let $query= SELECT * FROM ( SELECT t1.a,MAX(t1.b) @@ -1836,14 +1836,13 @@ SELECT * FROM GROUP BY t1.a ) AS dt, t1 WHERE dt.a=t1.a AND dt.a IN (1,t1.a); -EVAL $query; -EVAL EXPLAIN FORMAT=JSON $query; - +eval $query; +eval EXPLAIN FORMAT=JSON $query; DROP TABLE t1; --echo # --echo # MDEV-15087: error from inexpensive subquery before check ---echo # for condition pushdown into derived +--echo # for condition pushdown into derived --echo # CREATE TABLE t1 (i1 int, v1 varchar(1)); @@ -2075,3 +2074,5 @@ SELECT * FROM t1; DROP VIEW v1; DROP TABLE t1; + +--echo # End of 10.2 tests