From 214023aa0e6ec00dcac386167a2b2cf9394b6c7e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 18 Jan 2019 13:45:06 +1100 Subject: [PATCH 01/32] systemd: mariadb@bootstrap doesn't bootstrap, galera_new_cluster does Closes #1106 --- support-files/mariadb@.service.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 4838620b8cd..8afae3bd09b 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -1,5 +1,5 @@ # Multi instance version of mariadb. For if you run multiple versions at once. -# Also used for mariadb@bootstrap to bootstrap Galera. +# Also used for mariadb@bootstrap to tell you to use galera_new_cluster. # # create config file @sysconf2dir@/my{instancename}.cnf # From 2cb4b152c8fc7bc5156961c2f87e5274d228377d Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 16 Nov 2019 14:59:54 +0100 Subject: [PATCH 02/32] ===================================================================== This new CONNECT version 1.07 fully implements NOSQL support. It allows working on JSON or XML data retrieved as REST query results from all binary distributions of MariaDB when cpprestsdk is installed and the GetRest library is available. ===================================================================== - Make Rest available for MariaDB binary distributed versions. Change RestGet function so it can be called from a library. modified: storage/connect/CMakeLists.txt modified: storage/connect/restget.cpp modified: storage/connect/tabrest.cpp - Make column FLAG option available to discovery functions. modified: storage/connect/ha_connect.cc modified: storage/connect/plgdbsem.h - Update CONNECT version number and date. modified: storage/connect/ha_connect.cc - Move OEMColumns function from mycat.cc to reldef.cpp. modified: storage/connect/mycat.cc modified: storage/connect/reldef.cpp - Allocate tables as TABREF (was RELDEF) modified: storage/connect/mycat.cc modified: storage/connect/mycat.h - Fix MDEV-20845 by commenting out TIMEOUT setting. modified: storage/connect/myconn.cpp - Call DefineAM before calling GetColCatInfo. Column offset is now based on record format instead of table type. The RECFM_VCT format was added. This enables tables to specify the record format and is useful in particular for OEM tables. modified: storage/connect/plgdbsem.h modified: storage/connect/reldef.cpp modified: storage/connect/reldef.h modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfix.cpp modified: storage/connect/tabfmt.cpp modified: storage/connect/tabmysql.cpp modified: storage/connect/tabutil.cpp modified: storage/connect/tabutil.h modified: storage/connect/tabvct.cpp modified: storage/connect/xindex.cpp --- storage/connect/CMakeLists.txt | 6 +- storage/connect/ha_connect.cc | 17 +- storage/connect/mycat.cc | 111 +------------ storage/connect/mycat.h | 4 +- storage/connect/myconn.cpp | 10 +- storage/connect/plgdbsem.h | 29 ++-- storage/connect/reldef.cpp | 275 +++++++++++++++++++++++---------- storage/connect/reldef.h | 9 +- storage/connect/restget.cpp | 40 ++--- storage/connect/tabdos.cpp | 23 ++- storage/connect/tabdos.h | 1 - storage/connect/tabfix.cpp | 2 +- storage/connect/tabfmt.cpp | 5 +- storage/connect/tabmysql.cpp | 9 +- storage/connect/tabrest.cpp | 101 ++++++++++-- storage/connect/tabutil.cpp | 16 +- storage/connect/tabutil.h | 3 +- storage/connect/tabvct.cpp | 3 + storage/connect/xindex.cpp | 14 +- 19 files changed, 412 insertions(+), 266 deletions(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index be5c7034c3b..66b658b036b 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -312,6 +312,8 @@ OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON) IF(CONNECT_WITH_REST) MESSAGE(STATUS "=====> REST support is ON") + SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h) + add_definitions(-DREST_SUPPORT) FIND_PACKAGE(cpprestsdk) IF (cpprestsdk_FOUND) MESSAGE(STATUS "=====> cpprestsdk found") @@ -326,8 +328,8 @@ IF(CONNECT_WITH_REST) # Comment it out if not needed depending on your cpprestsdk installation. SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") ENDIF(UNIX) - SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h) - add_definitions(-DREST_SUPPORT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp) + add_definitions(-DREST_SOURCE) ELSE(NOT cpprestsdk_FOUND) MESSAGE(STATUS "=====> cpprestsdk package not found") ENDIF (cpprestsdk_FOUND) diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index be7c6548181..beb7c5c7e74 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,9 +170,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.06.0010 June 01, 2019"; + char version[]= "Version 1.07.0001 November 12, 2019"; #if defined(__WIN__) - char compver[]= "Version 1.06.0010 " __DATE__ " " __TIME__; + char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -6062,7 +6062,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } // endif !nblin for (i= 0; !rc && i < qrp->Nblin; i++) { - typ= len= prec= dec= 0; + typ= len= prec= dec= flg= 0; tm= NOT_NULL_FLAG; cnm= (char*)"noname"; dft= xtra= key= fmt= tn= NULL; @@ -6102,6 +6102,9 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (crp->Kdata->GetIntValue(i)) tm= 0; // Nullable + break; + case FLD_FLAG: + flg = crp->Kdata->GetIntValue(i); break; case FLD_FORMAT: fmt= (crp->Kdata) ? crp->Kdata->GetCharValue(i) : NULL; @@ -6233,7 +6236,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, // Now add the field if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra, - fmt, 0, dbf, v)) + fmt, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor i @@ -7379,14 +7382,14 @@ maria_declare_plugin(connect) &connect_storage_engine, "CONNECT", "Olivier Bertrand", - "Management of External Data (SQL/NOSQL/MED), including many file formats", + "Management of External Data (SQL/NOSQL/MED), including Rest query results", PLUGIN_LICENSE_GPL, connect_init_func, /* Plugin Init */ connect_done_func, /* Plugin Deinit */ - 0x0106, /* version number (1.06) */ + 0x0107, /* version number (1.07) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.06.0010", /* string version */ + "1.07.0001", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b1d23b8c5e2..f8b3dc03aa5 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -95,7 +95,7 @@ #endif // ZIP_SUPPORT #if defined(REST_SUPPORT) #include "tabrest.h" -#endif // Rest_SUPPORT +#endif // REST_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -107,7 +107,6 @@ extern "C" HINSTANCE s_hModule; // Saved module handle #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) bool MongoEnabled(void); #endif // JAVA_SUPPORT || CMGO_SUPPORT -PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info); /***********************************************************************/ /* Get the plugin directory. */ @@ -349,100 +348,6 @@ uint GetFuncID(const char *func) return fnc; } // end of GetFuncID -/***********************************************************************/ -/* OEMColumn: Get table column info for an OEM table. */ -/***********************************************************************/ -PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info) - { - typedef PQRYRES (__stdcall *XCOLDEF) (PGLOBAL, void*, char*, char*, bool); - const char *module, *subtype; - char c, soname[_MAX_PATH], getname[40] = "Col"; -#if defined(__WIN__) - HANDLE hdll; /* Handle to the external DLL */ -#else // !__WIN__ - void *hdll; /* Handle for the loaded shared library */ -#endif // !__WIN__ - XCOLDEF coldef = NULL; - PQRYRES qrp = NULL; - - module = topt->module; - subtype = topt->subtype; - - if (!module || !subtype) - return NULL; - - /*********************************************************************/ - /* Ensure that the .dll doesn't have a path. */ - /* This is done to ensure that only approved dll from the system */ - /* directories are used (to make this even remotely secure). */ - /*********************************************************************/ - if (check_valid_path(module, strlen(module))) { - strcpy(g->Message, "Module cannot contain a path"); - return NULL; - } else - PlugSetPath(soname, module, GetPluginDir()); - - // The exported name is always in uppercase - for (int i = 0; ; i++) { - c = subtype[i]; - getname[i + 3] = toupper(c); - if (!c) break; - } // endfor i - -#if defined(__WIN__) - // Load the Dll implementing the table - if (!(hdll = LoadLibrary(soname))) { - char buf[256]; - DWORD rc = GetLastError(); - - sprintf(g->Message, MSG(DLL_LOAD_ERROR), rc, soname); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - return NULL; - } // endif hDll - - // Get the function returning an instance of the external DEF class - if (!(coldef = (XCOLDEF)GetProcAddress((HINSTANCE)hdll, getname))) { - sprintf(g->Message, MSG(PROCADD_ERROR), GetLastError(), getname); - FreeLibrary((HMODULE)hdll); - return NULL; - } // endif coldef -#else // !__WIN__ - const char *error = NULL; - - // Load the desired shared library - if (!(hdll = dlopen(soname, RTLD_LAZY))) { - error = dlerror(); - sprintf(g->Message, MSG(SHARED_LIB_ERR), soname, SVP(error)); - return NULL; - } // endif Hdll - - // Get the function returning an instance of the external DEF class - if (!(coldef = (XCOLDEF)dlsym(hdll, getname))) { - error = dlerror(); - sprintf(g->Message, MSG(GET_FUNC_ERR), getname, SVP(error)); - dlclose(hdll); - return NULL; - } // endif coldef -#endif // !__WIN__ - - // Just in case the external Get function does not set error messages - sprintf(g->Message, "Error getting column info from %s", subtype); - - // Get the table column definition - qrp = coldef(g, topt, tab, db, info); - -#if defined(__WIN__) - FreeLibrary((HMODULE)hdll); -#else // !__WIN__ - dlclose(hdll); -#endif // !__WIN__ - - return qrp; - } // end of OEMColumns - /* ------------------------- Class CATALOG --------------------------- */ /***********************************************************************/ @@ -483,10 +388,10 @@ void MYCAT::Reset(void) /* GetTableDesc: retrieve a table descriptor. */ /* Look for a table descriptor matching the name and type. */ /***********************************************************************/ -PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, +PTABDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR type, PRELDEF *) { - PRELDEF tdp= NULL; + PTABDEF tdp= NULL; if (trace(1)) htrc("GetTableDesc: name=%s am=%s\n", tablep->GetName(), SVP(type)); @@ -507,12 +412,12 @@ PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, /* MakeTableDesc: make a table/view description. */ /* Note: caller must check if name already exists before calling it. */ /***********************************************************************/ -PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) +PTABDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) { TABTYPE tc; LPCSTR name= (PSZ)PlugDup(g, tablep->GetName()); LPCSTR schema= (PSZ)PlugDup(g, tablep->GetSchema()); - PRELDEF tdp= NULL; + PTABDEF tdp= NULL; if (trace(1)) htrc("MakeTableDesc: name=%s schema=%s am=%s\n", @@ -580,8 +485,8 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) } // endswitch // Do make the table/view definition - if (tdp && tdp->Define(g, this, name, schema, am)) - tdp= NULL; + if (tdp && tdp->Define(g, this, name, schema, am)) + tdp = NULL; if (trace(1)) htrc("Table %s made\n", am); @@ -594,7 +499,7 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) /***********************************************************************/ PTDB MYCAT::GetTable(PGLOBAL g, PTABLE tablep, MODE mode, LPCSTR type) { - PRELDEF tdp; + PTABDEF tdp; PTDB tdbp= NULL; // LPCSTR name= tablep->GetName(); diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 818e535b32d..6473f7a5c11 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -102,14 +102,14 @@ class MYCAT : public CATALOG { // Methods void Reset(void); bool StoreIndex(PGLOBAL, PTABDEF) {return false;} // Temporary - PRELDEF GetTableDesc(PGLOBAL g, PTABLE tablep, + PTABDEF GetTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR type, PRELDEF *prp = NULL); PTDB GetTable(PGLOBAL g, PTABLE tablep, MODE mode = MODE_READ, LPCSTR type = NULL); void ClearDB(PGLOBAL g); protected: - PRELDEF MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am); + PTABDEF MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am); // Members ha_connect *Hc; // The Connect handler diff --git a/storage/connect/myconn.cpp b/storage/connect/myconn.cpp index 253c42bb002..e7500ff2c31 100644 --- a/storage/connect/myconn.cpp +++ b/storage/connect/myconn.cpp @@ -472,7 +472,7 @@ int MYSQLC::Open(PGLOBAL g, const char *host, const char *db, int pt, const char *csname) { const char *pipe = NULL; - uint cto = 10, nrt = 20; + //uint cto = 10, nrt = 20; my_bool my_true= 1; m_DB = mysql_init(NULL); @@ -485,11 +485,11 @@ int MYSQLC::Open(PGLOBAL g, const char *host, const char *db, if (trace(1)) htrc("MYSQLC Open: m_DB=%.4X size=%d\n", m_DB, (int)sizeof(*m_DB)); - // Removed to do like FEDERATED do + // Removed to do like FEDERATED does //mysql_options(m_DB, MYSQL_READ_DEFAULT_GROUP, "client-mariadb"); - mysql_options(m_DB, MYSQL_OPT_USE_REMOTE_CONNECTION, NULL); - mysql_options(m_DB, MYSQL_OPT_CONNECT_TIMEOUT, &cto); - mysql_options(m_DB, MYSQL_OPT_READ_TIMEOUT, &nrt); +//mysql_options(m_DB, MYSQL_OPT_USE_REMOTE_CONNECTION, NULL); +//mysql_options(m_DB, MYSQL_OPT_CONNECT_TIMEOUT, &cto); +//mysql_options(m_DB, MYSQL_OPT_READ_TIMEOUT, &nrt); //mysql_options(m_DB, MYSQL_OPT_WRITE_TIMEOUT, ...); #if defined(__WIN__) diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index f10ae209e9d..a40e32bcfb2 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -149,16 +149,22 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MGO = 194, /* MGO access method type no */ TYPE_AM_OUT = 200}; /* Output relations (storage) */ -enum RECFM {RECFM_NAF = -2, /* Not a file */ - RECFM_OEM = -1, /* OEM file access method */ - RECFM_VAR = 0, /* Varying length DOS files */ - RECFM_FIX = 1, /* Fixed length DOS files */ - RECFM_BIN = 2, /* Binary DOS files (also fixed) */ - RECFM_VCT = 3, /* VCT formatted files */ - RECFM_ODBC = 4, /* Table accessed via ODBC */ - RECFM_JDBC = 5, /* Table accessed via JDBC */ - RECFM_PLG = 6, /* Table accessed via PLGconn */ - RECFM_DBF = 7}; /* DBase formatted file */ +enum RECFM {RECFM_DFLT = 0, /* Default table type */ + RECFM_NAF = 1, /* Not a file table */ + RECFM_OEM = 2, /* OEM table */ + RECFM_VAR = 3, /* Varying length DOS files */ + RECFM_FIX = 4, /* Fixed length DOS files */ + RECFM_BIN = 5, /* Binary DOS files (also fixed) */ + RECFM_DBF = 6, /* DBase formatted file */ + RECFM_CSV = 7, /* CSV file */ + RECFM_FMT = 8, /* FMT formatted file */ + RECFM_VCT = 9, /* VCT formatted files */ + RECFM_XML = 10, /* XML formatted files */ + RECFM_JASON = 11, /* JASON formatted files */ + RECFM_DIR = 12, /* DIR table */ + RECFM_ODBC = 13, /* Table accessed via ODBC */ + RECFM_JDBC = 14, /* Table accessed via JDBC */ + RECFM_PLG = 15}; /* Table accessed via PLGconn */ enum MISC {DB_TABNO = 1, /* DB routines in Utility Table */ MAX_MULT_KEY = 10, /* Max multiple key number */ @@ -537,7 +543,8 @@ enum XFLD {FLD_NO = 0, /* Not a field definition item */ FLD_FORMAT = 16, /* Field format */ FLD_CAT = 17, /* Table catalog */ FLD_SCHEM = 18, /* Table schema */ - FLD_TABNAME = 19}; /* Column Table name */ + FLD_TABNAME = 19, /* Column Table name */ + FLD_FLAG = 20}; /* Field flag (CONNECT specific) */ /***********************************************************************/ /* Result of last SQL noconv query. */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 8ba8aac3621..88c28757588 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,11 +1,11 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.6 */ +/* Version 1.7 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2004-2016 */ +/* (C) Copyright to the author Olivier BERTRAND 2004-2019 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -61,6 +61,102 @@ extern handlerton *connect_hton; /***********************************************************************/ USETEMP UseTemp(void); char *GetPluginDir(void); +PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char* tab, char* db, bool info); + +/***********************************************************************/ +/* OEMColumns: Get table column info for an OEM table. */ +/***********************************************************************/ +PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char* tab, char* db, bool info) +{ + typedef PQRYRES(__stdcall* XCOLDEF) (PGLOBAL, void*, char*, char*, bool); + const char* module, * subtype; + char c, soname[_MAX_PATH], getname[40] = "Col"; +#if defined(__WIN__) + HANDLE hdll; /* Handle to the external DLL */ +#else // !__WIN__ + void* hdll; /* Handle for the loaded shared library */ +#endif // !__WIN__ + XCOLDEF coldef = NULL; + PQRYRES qrp = NULL; + + module = topt->module; + subtype = topt->subtype; + + if (!module || !subtype) + return NULL; + + /*********************************************************************/ + /* Ensure that the .dll doesn't have a path. */ + /* This is done to ensure that only approved dll from the system */ + /* directories are used (to make this even remotely secure). */ + /*********************************************************************/ + if (check_valid_path(module, strlen(module))) { + strcpy(g->Message, "Module cannot contain a path"); + return NULL; + } + else + PlugSetPath(soname, module, GetPluginDir()); + + // The exported name is always in uppercase + for (int i = 0; ; i++) { + c = subtype[i]; + getname[i + 3] = toupper(c); + if (!c) break; + } // endfor i + +#if defined(__WIN__) + // Load the Dll implementing the table + if (!(hdll = LoadLibrary(soname))) { + char buf[256]; + DWORD rc = GetLastError(); + + sprintf(g->Message, MSG(DLL_LOAD_ERROR), rc, soname); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, + (LPTSTR)buf, sizeof(buf), NULL); + strcat(strcat(g->Message, ": "), buf); + return NULL; + } // endif hDll + +// Get the function returning an instance of the external DEF class + if (!(coldef = (XCOLDEF)GetProcAddress((HINSTANCE)hdll, getname))) { + sprintf(g->Message, MSG(PROCADD_ERROR), GetLastError(), getname); + FreeLibrary((HMODULE)hdll); + return NULL; + } // endif coldef +#else // !__WIN__ + const char* error = NULL; + + // Load the desired shared library + if (!(hdll = dlopen(soname, RTLD_LAZY))) { + error = dlerror(); + sprintf(g->Message, MSG(SHARED_LIB_ERR), soname, SVP(error)); + return NULL; + } // endif Hdll + +// Get the function returning an instance of the external DEF class + if (!(coldef = (XCOLDEF)dlsym(hdll, getname))) { + error = dlerror(); + sprintf(g->Message, MSG(GET_FUNC_ERR), getname, SVP(error)); + dlclose(hdll); + return NULL; + } // endif coldef +#endif // !__WIN__ + + // Just in case the external Get function does not set error messages + sprintf(g->Message, "Error getting column info from %s", subtype); + + // Get the table column definition + qrp = coldef(g, topt, tab, db, info); + +#if defined(__WIN__) + FreeLibrary((HMODULE)hdll); +#else // !__WIN__ + dlclose(hdll); +#endif // !__WIN__ + + return qrp; +} // end of OEMColumns /* --------------------------- Class RELDEF -------------------------- */ @@ -208,6 +304,7 @@ TABDEF::TABDEF(void) { Schema = NULL; Desc = NULL; + Recfm = RECFM_DFLT; Catfunc = FNC_NO; Card = 0; Elemt = 0; @@ -220,12 +317,41 @@ TABDEF::TABDEF(void) csname = NULL; } // end of TABDEF constructor +/***********************************************************************/ +/* Return the table format. */ +/***********************************************************************/ +RECFM TABDEF::GetTableFormat(const char* type) +{ + RECFM recfm = Recfm; + + if (recfm == RECFM_DFLT) { + // Default format depends on the table type + TABTYPE tc = (Catfunc == FNC_NO) ? GetTypeID(type) : TAB_PRX; + + switch (tc) { + case TAB_DOS: recfm = RECFM_VAR; break; + case TAB_CSV: recfm = RECFM_CSV; break; + case TAB_FMT: recfm = RECFM_FMT; break; + case TAB_FIX: recfm = RECFM_FIX; break; + case TAB_BIN: recfm = RECFM_BIN; break; + case TAB_VEC: recfm = RECFM_VCT; break; + case TAB_DBF: recfm = RECFM_DBF; break; + case TAB_XML: recfm = RECFM_XML; break; + case TAB_DIR: recfm = RECFM_DIR; break; + default: recfm = RECFM_NAF; break; + } // endswitch type + + } // endif recfm + + return recfm; +} // end of GetTableFormat + /***********************************************************************/ /* Define: initialize the table definition block from XDB file. */ /***********************************************************************/ bool TABDEF::Define(PGLOBAL g, PCATLG cat, LPCSTR name, LPCSTR schema, LPCSTR am) - { +{ int poff = 0; Hc = ((MYCAT*)cat)->GetHandler(); @@ -243,13 +369,17 @@ bool TABDEF::Define(PGLOBAL g, PCATLG cat, NULL; csname = GetStringCatInfo(g, "Table_charset", NULL); - // Get The column definitions - if ((poff = GetColCatInfo(g)) < 0) - return true; + // Do the definition of AM specific fields + if (DefineAM(g, am, 0)) + return true; - // Do the definition of AM specific fields - return DefineAM(g, am, poff); - } // end of Define + // Get The column definitions + if (stricmp(am, "OEM") && GetColCatInfo(g) < 0) + return true; + + Hc->tshp = NULL; // TO BE CHECKED + return false; +} // end of Define /***********************************************************************/ /* This function returns the database data path. */ @@ -264,71 +394,71 @@ PCSZ TABDEF::GetPath(void) /***********************************************************************/ int TABDEF::GetColCatInfo(PGLOBAL g) { - char *type= GetStringCatInfo(g, "Type", "*"); + char *type = GetStringCatInfo(g, "Type", "*"); char c, fty, eds; int i, n, loff, poff, nof, nlg; - void *field= NULL; - TABTYPE tc; - PCOLDEF cdp, lcdp= NULL, tocols= NULL; + void *field = NULL; + RECFM trf; + PCOLDEF cdp, lcdp = NULL, tocols= NULL; PCOLINFO pcf= (PCOLINFO)PlugSubAlloc(g, NULL, sizeof(COLINFO)); memset(pcf, 0, sizeof(COLINFO)); - // Get a unique char identifier for type - tc= (Catfunc == FNC_NO) ? GetTypeID(type) : TAB_PRX; + // Get the table format + trf = GetTableFormat(type); // Take care of the column definitions i= poff= nof= nlg= 0; #if defined(__WIN__) // Offsets of HTML and DIR tables start from 0, DBF at 1 - loff= (tc == TAB_DBF) ? 1 : (tc == TAB_XML || tc == TAB_DIR) ? -1 : 0; + loff= (trf == RECFM_DBF) ? 1 : (trf == RECFM_XML || trf == RECFM_DIR) ? -1 : 0; #else // !__WIN__ // Offsets of HTML tables start from 0, DIR and DBF at 1 - loff = (tc == TAB_DBF || tc == TAB_DIR) ? 1 : (tc == TAB_XML) ? -1 : 0; + loff = (trf == RECFM_DBF || trf == RECFM_DIR) ? 1 : (trf == RECFM_XML) ? -1 : 0; #endif // !__WIN__ while (true) { - // Default Offset depends on table type - switch (tc) { - case TAB_DOS: - case TAB_FIX: - case TAB_BIN: - case TAB_VEC: - case TAB_DBF: + // Default Offset depends on table format + switch (trf ) { + case RECFM_VAR: + case RECFM_FIX: + case RECFM_BIN: + case RECFM_VCT: + case RECFM_DBF: poff= loff + nof; // Default next offset nlg= MY_MAX(nlg, poff); // Default lrecl break; - case TAB_CSV: - case TAB_FMT: + case RECFM_CSV: + case RECFM_FMT: nlg+= nof; - case TAB_DIR: - case TAB_XML: + case RECFM_DIR: + case RECFM_XML: poff= loff + (pcf->Flags & U_VIRTUAL ? 0 : 1); break; - case TAB_INI: - case TAB_MAC: - case TAB_TBL: - case TAB_XCL: - case TAB_OCCUR: - case TAB_PRX: - case TAB_OEM: + //case RECFM_INI: + //case RECFM_MAC: + //case RECFM_TBL: + //case RECFM_XCL: + //case RECFM_OCCUR: + //case RECFM_PRX: + case RECFM_OEM: poff = 0; // Offset represents an independant flag break; - default: // VCT PLG ODBC JDBC MYSQL WMI... + default: // PLG ODBC JDBC MYSQL WMI... poff = 0; // NA break; - } // endswitch tc + } // endswitch trf // do { field= Hc->GetColumnOption(g, field, pcf); // } while (field && (*pcf->Name =='*' /*|| pcf->Flags & U_VIRTUAL*/)); - if (tc == TAB_DBF && pcf->Type == TYPE_DATE && !pcf->Datefmt) { + if (trf == RECFM_DBF && pcf->Type == TYPE_DATE && !pcf->Datefmt) { // DBF date format defaults to 'YYYMMDD' pcf->Datefmt= "YYYYMMDD"; pcf->Length= 8; - } // endif tc + } // endif trf if (!field) break; @@ -341,10 +471,10 @@ int TABDEF::GetColCatInfo(PGLOBAL g) else loff= cdp->GetOffset(); - switch (tc) { - case TAB_VEC: + switch (trf ) { + case RECFM_VCT: cdp->SetOffset(0); // Not to have shift - case TAB_BIN: + case RECFM_BIN: // BIN/VEC are packed by default if (nof) { // Field width is the internal representation width @@ -395,7 +525,7 @@ int TABDEF::GetColCatInfo(PGLOBAL g) default: break; - } // endswitch tc + } // endswitch trf if (lcdp) lcdp->SetNext(cdp); @@ -413,21 +543,15 @@ int TABDEF::GetColCatInfo(PGLOBAL g) if (GetDefType() == TYPE_AM_DOS) { int ending, recln= 0; - // Was commented because sometimes ending is 0 even when - // not specified (for instance if quoted is specified) -// if ((ending= Hc->GetIntegerOption("Ending")) < 0) { - if ((ending= Hc->GetIntegerOption("Ending")) <= 0) { - ending= (tc == TAB_BIN || tc == TAB_VEC) ? 0 : CRLF; - Hc->SetIntegerOption("Ending", ending); - } // endif ending + ending = Hc->GetIntegerOption("Ending"); // Calculate the default record size - switch (tc) { - case TAB_FIX: - case TAB_BIN: + switch (trf ) { + case RECFM_FIX: + case RECFM_BIN: recln= nlg + ending; // + length of line ending break; - case TAB_VEC: + case RECFM_VCT: recln= nlg; // if ((k= (pak < 0) ? 8 : pak) > 1) @@ -436,18 +560,18 @@ int TABDEF::GetColCatInfo(PGLOBAL g) // recln= ((recln + k - 1) / k) * k; break; - case TAB_DOS: - case TAB_DBF: + case RECFM_VAR: + case RECFM_DBF: recln= nlg; break; - case TAB_CSV: - case TAB_FMT: + case RECFM_CSV: + case RECFM_FMT: // The number of separators (assuming an extra one can exist) // recln= poff * ((qotd) ? 3 : 1); to be investigated recln= nlg + poff * 3; // To be safe default: break; - } // endswitch tc + } // endswitch trf // lrecl must be at least recln to avoid buffer overflow if (trace(1)) @@ -461,7 +585,7 @@ int TABDEF::GetColCatInfo(PGLOBAL g) if (trace(1)) htrc("Lrecl set to %d\n", recln); - } // endif Lrecl + } // endif TYPE // Attach the column definition to the tabdef SetCols(tocols); @@ -596,10 +720,6 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) cat->Cbuf = (char*)PlugSubAlloc(g, NULL, cat->Cblen); } // endif Cbuf - // Here "OEM" should be replace by a more useful value - if (xdefp->Define(g, cat, Name, Schema, "OEM")) - return NULL; - // Ok, return external block return xdefp; } // end of GetXdef @@ -622,7 +742,7 @@ bool OEMDEF::DeleteTableFile(PGLOBAL g) /***********************************************************************/ bool OEMDEF::DefineAM(PGLOBAL g, LPCSTR, int) { - Module = GetStringCatInfo(g, "Module", ""); + Module = GetStringCatInfo(g, "Module", ""); Subtype = GetStringCatInfo(g, "Subtype", Module); if (!*Module) @@ -632,7 +752,13 @@ bool OEMDEF::DefineAM(PGLOBAL g, LPCSTR, int) + strlen(Subtype) + 3); sprintf(desc, "%s(%s)", Module, Subtype); Desc = desc; - return false; + + // If define block not here yet, get it now + if (!Pxdef && !(Pxdef = GetXdef(g))) + return true; // Error + + // Here "OEM" should be replace by a more useful value + return Pxdef->Define(g, Cat, Name, Schema, Subtype); } // end of DefineAM /***********************************************************************/ @@ -640,7 +766,6 @@ bool OEMDEF::DefineAM(PGLOBAL g, LPCSTR, int) /***********************************************************************/ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) { - RECFM rfm; PTDB tdbp = NULL; // If define block not here yet, get it now @@ -653,18 +778,10 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!(tdbp = Pxdef->GetTable(g, mode))) return NULL; - else - rfm = tdbp->GetFtype(); - - if (rfm == RECFM_NAF) - return tdbp; - else if (rfm == RECFM_OEM) { - if (Multiple) - tdbp = new(g) TDBMUL(tdbp); // No block optimization yet - - return tdbp; - } // endif OEM + else if (Multiple && tdbp->GetFtype() == RECFM_OEM) + tdbp = new(g) TDBMUL(tdbp); // No block optimization yet +#if 0 /*********************************************************************/ /* The OEM table is based on a file type (currently DOS+ only) */ /*********************************************************************/ @@ -723,7 +840,7 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) if (Multiple) tdbp = new(g) TDBMUL(tdbp); - +#endif // 0 return tdbp; } // end of GetTable diff --git a/storage/connect/reldef.h b/storage/connect/reldef.h index f8256a59b3d..73e178ed51c 100644 --- a/storage/connect/reldef.h +++ b/storage/connect/reldef.h @@ -84,10 +84,12 @@ public: void SetNext(PTABDEF tdfp) {Next = tdfp;} int GetMultiple(void) {return Multiple;} int GetPseudo(void) {return Pseudo;} - PCSZ GetPath(void); + RECFM GetRecfm(void) {return Recfm;} + PCSZ GetPath(void); //PSZ GetPath(void) // {return (Database) ? (PSZ)Database : Cat->GetDataPath();} - bool SepIndex(void) {return GetBoolCatInfo("SepIndex", false);} + RECFM GetTableFormat(const char* type); + bool SepIndex(void) {return GetBoolCatInfo("SepIndex", false);} bool IsReadOnly(void) {return Read_Only;} virtual AMT GetDefType(void) {return TYPE_AM_TAB;} virtual PIXDEF GetIndx(void) {return NULL;} @@ -108,7 +110,8 @@ public: // Members PCSZ Schema; /* Table schema (for ODBC) */ PCSZ Desc; /* Table description */ - uint Catfunc; /* Catalog function ID */ + RECFM Recfm; /* File or table format */ + uint Catfunc; /* Catalog function ID */ int Card; /* (max) number of rows in table */ int Elemt; /* Number of rows in blocks or rowset */ int Sort; /* Table already sorted ??? */ diff --git a/storage/connect/restget.cpp b/storage/connect/restget.cpp index 6b184ae6926..d85ec6ae693 100644 --- a/storage/connect/restget.cpp +++ b/storage/connect/restget.cpp @@ -4,12 +4,6 @@ /***********************************************************************/ #include #include -#if defined(MARIADB) -#include -#else -#include "mini-global.h" -#define _OS_H_INCLUDED // Prevent os.h to be called -#endif using namespace utility::conversions; // String conversions utilities using namespace web; // Common features like URIs. @@ -17,24 +11,24 @@ using namespace web::http; // Common HTTP functionality using namespace web::http::client; // HTTP client features using namespace concurrency::streams; // Asynchronous streams -#include "global.h" +typedef const char* PCSZ; /***********************************************************************/ /* Make a local copy of the requested file. */ /***********************************************************************/ -int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) +int restGetFile(char *m, bool xt, PCSZ http, PCSZ uri, PCSZ fn) { int rc = 0; - bool xt = trace(515); auto fileStream = std::make_shared(); if (!http || !fn) { - strcpy(g->Message, "Missing http or filename"); - return 2; + //strcpy(g->Message, "Missing http or filename"); + strcpy(m, "Missing http or filename"); + return 2; } // endif if (xt) - htrc("restGetFile: fn=%s\n", fn); + fprintf(stderr, "restGetFile: fn=%s\n", fn); // Open stream to output file. pplx::task requestTask = fstream::open_ostream(to_string_t(fn)) @@ -42,7 +36,7 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) *fileStream= outFile; if (xt) - htrc("Outfile isopen=%d\n", outFile.is_open()); + fprintf(stderr, "Outfile isopen=%d\n", outFile.is_open()); // Create http_client to send the request. http_client client(to_string_t(http)); @@ -58,8 +52,8 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) // Handle response headers arriving. .then([=](http_response response) { if (xt) - htrc("Received response status code:%u\n", - response.status_code()); + fprintf(stderr, "Received response status code:%u\n", + response.status_code()); // Write response body into the file. return response.body().read_to_end(fileStream->streambuf()); @@ -68,27 +62,27 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) // Close the file stream. .then([=](size_t n) { if (xt) - htrc("Return size=%u\n", n); + fprintf(stderr, "Return size=%zu\n", n); return fileStream->close(); }); // Wait for all the outstanding I/O to complete and handle any exceptions try { - requestTask.wait(); - if (xt) - htrc("In Wait\n"); + fprintf(stderr, "Waiting\n"); + requestTask.wait(); } catch (const std::exception &e) { if (xt) - htrc("Error exception: %s\n", e.what()); - sprintf(g->Message, "Error exception: %s", e.what()); - rc= 1; + fprintf(stderr, "Error exception: %s\n", e.what()); + + sprintf(m, "Error exception: %s", e.what()); + rc= 1; } // end try/catch if (xt) - htrc("restget done: rc=%d\n", rc); + fprintf(stderr, "restget done: rc=%d\n", rc); return rc; } // end of restGetFile diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 4f6e2c81744..8efe2aad702 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -45,7 +45,7 @@ #include "global.h" #include "osutil.h" #include "plgdbsem.h" -#include "catalog.h" +//#include "catalog.h" #include "mycat.h" #include "xindex.h" #include "filamap.h" @@ -161,7 +161,12 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) //Last = GetIntCatInfo("Last", 0); Ending = GetIntCatInfo("Ending", CRLF); - if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { + if (Ending <= 0) { + Ending = (Recfm == RECFM_BIN || Recfm == RECFM_VCT) ? 0 : CRLF; + SetIntCatInfo("Ending", Ending); + } // endif ending + + if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { Huge = GetBoolCatInfo("Huge", Cat->GetDefHuge()); Padded = GetBoolCatInfo("Padded", false); Blksize = GetIntCatInfo("Blksize", 0); @@ -191,7 +196,8 @@ bool DOSDEF::GetOptFileName(PGLOBAL g, char *filename) case RECFM_FIX: ftype = ".fop"; break; case RECFM_BIN: ftype = ".bop"; break; case RECFM_VCT: ftype = ".vop"; break; - case RECFM_DBF: ftype = ".dbp"; break; + case RECFM_CSV: ftype = ".cop"; break; + case RECFM_DBF: ftype = ".dbp"; break; default: sprintf(g->Message, MSG(INVALID_FTYPE), Recfm); return true; @@ -261,7 +267,8 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; - case RECFM_DBF: ftype = ".dbx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; default: sprintf(g->Message, MSG(BAD_RECFM_VAL), Recfm); return true; @@ -2257,7 +2264,7 @@ int TDBDOS::ReadDB(PGLOBAL g) /***********************************************************************/ bool TDBDOS::PrepareWriting(PGLOBAL) { - if (!Ftype && (Mode == MODE_INSERT || Txfp->GetUseTemp())) { + if (Ftype == RECFM_VAR && (Mode == MODE_INSERT || Txfp->GetUseTemp())) { char *p; /*******************************************************************/ @@ -2542,7 +2549,8 @@ void DOSCOL::ReadColumn(PGLOBAL g) /*********************************************************************/ /* For a variable length file, check if the field exists. */ /*********************************************************************/ - if (tdbp->Ftype == RECFM_VAR && strlen(tdbp->To_Line) < (unsigned)Deplac) + if ((tdbp->Ftype == RECFM_VAR || tdbp->Ftype == RECFM_CSV) + && strlen(tdbp->To_Line) < (unsigned)Deplac) field = 0; else if (Dsp) for(i = 0; i < field; i++) @@ -2552,7 +2560,8 @@ void DOSCOL::ReadColumn(PGLOBAL g) switch (tdbp->Ftype) { case RECFM_VAR: case RECFM_FIX: // Fixed length text file - case RECFM_DBF: // Fixed length DBase file + case RECFM_CSV: // Variable length CSV or FMT file + case RECFM_DBF: // Fixed length DBase file if (Nod) switch (Buf_Type) { case TYPE_INT: case TYPE_SHORT: diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index bdde37adaad..207a1277fce 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -80,7 +80,6 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ PCSZ Entry; /* Zip entry name or pattern */ PCSZ Pwd; /* Zip password */ PIXDEF To_Indx; /* To index definitions blocks */ - RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ bool Zipped; /* true for zipped table file */ bool Mulentries; /* true for multiple entries */ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 1969fd4465f..4a0a75460cd 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -84,7 +84,7 @@ PTDB TDBFIX::Clone(PTABS t) tp = new(g) TDBFIX(g, this); - if (Ftype < 2) { + if (Ftype == RECFM_VAR || Ftype == RECFM_FIX) { // File is text PDOSCOL cp1, cp2; diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 746382178fb..b395c49c95d 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9.2 */ +/* Version 3.9.3 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2019 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -477,6 +477,7 @@ bool CSVDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) if (DOSDEF::DefineAM(g, "CSV", poff)) return true; + Recfm = RECFM_CSV; GetCharCatInfo("Separator", ",", buf, sizeof(buf)); Sep = (strlen(buf) == 2 && buf[0] == '\\' && buf[1] == 't') ? '\t' : *buf; Quoted = GetIntCatInfo("Quoted", -1); diff --git a/storage/connect/tabmysql.cpp b/storage/connect/tabmysql.cpp index ceffafac02c..7e165fb5a80 100644 --- a/storage/connect/tabmysql.cpp +++ b/storage/connect/tabmysql.cpp @@ -342,11 +342,13 @@ bool MYSQLDEF::DefineAM(PGLOBAL g, LPCSTR am, int) Delayed = !!GetIntCatInfo("Delayed", 0); } else { // MYSQL access from a PROXY table - Tabschema = GetStringCatInfo(g, "Database", Tabschema ? Tabschema : PlugDup(g, "*")); + TABLE_SHARE* s; + + Tabschema = GetStringCatInfo(g, "Database", Tabschema ? Tabschema : PlugDup(g, "*")); Isview = GetBoolCatInfo("View", false); // We must get other connection parms from the calling table - Remove_tshp(Cat); + s = Remove_tshp(Cat); url = GetStringCatInfo(g, "Connect", NULL); if (!url || !*url) { @@ -365,6 +367,9 @@ bool MYSQLDEF::DefineAM(PGLOBAL g, LPCSTR am, int) } // endif url Tabname = Name; + + // Needed for column description + Restore_tshp(Cat, s); } // endif am if ((Srcdef = GetStringCatInfo(g, "Srcdef", NULL))) { diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 9e1a643c89f..dde0411592e 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -1,5 +1,5 @@ /*************** Rest C++ Program Source Code File (.CPP) **************/ -/* PROGRAM NAME: Rest Version 1.5 */ +/* PROGRAM NAME: Rest Version 1.6 */ /* (C) Copyright to the author Olivier BERTRAND 2018 - 2019 */ /* This program is the REST Web API support for MariaDB. */ /* When compiled without MARIADB defined, it is the EOM module code. */ @@ -39,7 +39,7 @@ /***********************************************************************/ /* Get the file from the Web. */ /***********************************************************************/ -int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn); +int restGetFile(char *m, bool x, PCSZ http, PCSZ uri, PCSZ fn); #if defined(__WIN__) static PCSZ slash = "\\"; @@ -48,6 +48,10 @@ static PCSZ slash = "/"; #define stricmp strcasecmp #endif // !__WIN__ +typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); +static XGETREST getRestFnc = NULL; + + #if !defined(MARIADB) /***********************************************************************/ /* DB static variables. */ @@ -75,6 +79,74 @@ PTABDEF __stdcall GetREST(PGLOBAL g, void *memp) } // end of GetREST #endif // !MARIADB +/***********************************************************************/ +/* GetREST: get the external TABDEF from OEM module. */ +/***********************************************************************/ +XGETREST GetRestFunction(PGLOBAL g) +{ + if (getRestFnc) + return getRestFnc; + +#if !defined(REST_SOURCE) + if (trace(515)) + htrc("Looking for GetRest library\n"); + +#if defined(__WIN__) + HANDLE Hdll; + const char* soname = "GetRest.dll"; // Module name + + if (!(Hdll = LoadLibrary(soname))) { + char buf[256]; + DWORD rc = GetLastError(); + + sprintf(g->Message, MSG(DLL_LOAD_ERROR), rc, soname); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, + (LPTSTR)buf, sizeof(buf), NULL); + strcat(strcat(g->Message, ": "), buf); + return NULL; + } // endif Hdll + +// Get the function returning an instance of the external DEF class + if (!(getRestFnc = (XGETREST)GetProcAddress((HINSTANCE)Hdll, "restGetFile"))) { + char buf[256]; + DWORD rc = GetLastError(); + + sprintf(g->Message, MSG(PROCADD_ERROR), rc, "restGetFile"); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, + (LPTSTR)buf, sizeof(buf), NULL); + strcat(strcat(g->Message, ": "), buf); + FreeLibrary((HMODULE)Hdll); + return NULL; + } // endif getRestFnc +#else // !__WIN__ + void* Hso; + const char* error = NULL; + const char* soname = "GetRest.so"; // Module name + + // Load the desired shared library + if (!(Hso = dlopen(soname, RTLD_LAZY))) { + error = dlerror(); + sprintf(g->Message, MSG(SHARED_LIB_ERR), soname, SVP(error)); + return NULL; + } // endif Hdll + +// Get the function returning an instance of the external DEF class + if (!(getRestFnc = (XGETREST)dlsym(Hdll, "restGetFile"))) { + error = dlerror(); + sprintf(g->Message, MSG(GET_FUNC_ERR), "restGetFile", SVP(error)); + dlclose(Hso); + return NULL; + } // endif getdef +#endif // !__WIN__ +#else + getRestFnc = restGetFile; +#endif + + return getRestFnc; +} // end of GetRestFunction + /***********************************************************************/ /* Return the columns definition to MariaDB. */ /***********************************************************************/ @@ -87,6 +159,10 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) PQRYRES qrp= NULL; char filename[_MAX_PATH + 1]; // MAX PATH ??? PCSZ http, uri, fn, ftype; + XGETREST grf = GetRestFunction(g); + + if (!grf) + return NULL; http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); @@ -103,8 +179,8 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) strncat(filename, fn, _MAX_PATH); // Retrieve the file from the web and copy it locally - if (http && restGetFile(g, http, uri, filename)) { - // sprintf(g->Message, "Failed to get file at %s", http); + if (http && grf(g->Message, trace(515), http, uri, filename)) { + // sprintf(g->Message, "Failed to get file at %s", http); } else if (!stricmp(ftype, "XML")) qrp = XMLColumns(g, db, tab, tp, info); else if (!stricmp(ftype, "JSON")) @@ -124,9 +200,14 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) /***********************************************************************/ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - char filename[_MAX_PATH + 1]; + char filename[_MAX_PATH + 1]; int rc = 0, n; - LPCSTR ftype; + bool xt = trace(515); + LPCSTR ftype; + XGETREST grf = GetRestFunction(g); + + if (!grf) + return true; #if defined(MARIADB) ftype = GetStringCatInfo(g, "Type", "JSON"); @@ -135,7 +216,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) ftype = GetStringCatInfo(g, "Ftype", "JSON"); #endif // !MARIADB - if (trace(515)) + if (xt) htrc("ftype = %s am = %s\n", ftype, SVP(am)); n = (!stricmp(ftype, "JSON")) ? 1 @@ -157,9 +238,9 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) strncat(strcpy(filename, GetPath()), Fn, _MAX_PATH); // Retrieve the file from the web and copy it locally - rc = restGetFile(g, Http, Uri, filename); + rc = grf(g->Message, xt, Http, Uri, filename); - if (trace(515)) + if (xt) htrc("Return from restGetFile: rc=%d\n", rc); if (rc) @@ -175,7 +256,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) if (Tdp && Tdp->Define(g, Cat, Name, Schema, "REST")) Tdp = NULL; // Error occured - if (trace(515)) + if (xt) htrc("Tdp defined\n", rc); // Return true in case of error diff --git a/storage/connect/tabutil.cpp b/storage/connect/tabutil.cpp index 462a6fcd839..e187e6eaec5 100644 --- a/storage/connect/tabutil.cpp +++ b/storage/connect/tabutil.cpp @@ -59,11 +59,23 @@ int GetConvSize(void); /* Used by MYSQL tables to get MySQL parameters from the calling proxy */ /* table (PROXY, TBL, XCL, or OCCUR) when used by one of these. */ /************************************************************************/ -void Remove_tshp(PCATLG cat) +TABLE_SHARE *Remove_tshp(PCATLG cat) { - ((MYCAT*)cat)->GetHandler()->tshp = NULL; + TABLE_SHARE *s = ((MYCAT*)cat)->GetHandler()->tshp; + + ((MYCAT*)cat)->GetHandler()->tshp = NULL; + return s; } // end of Remove_thsp +/************************************************************************/ +/* Used by MYSQL tables to get MySQL parameters from the calling proxy */ +/* table (PROXY, TBL, XCL, or OCCUR) when used by one of these. */ +/************************************************************************/ +void Restore_tshp(PCATLG cat, TABLE_SHARE *s) +{ + ((MYCAT*)cat)->GetHandler()->tshp = s; +} // end of Restore_thsp + /************************************************************************/ /* GetTableShare: allocates and open a table share. */ /************************************************************************/ diff --git a/storage/connect/tabutil.h b/storage/connect/tabutil.h index 62678508ca1..c8e7e75106f 100644 --- a/storage/connect/tabutil.h +++ b/storage/connect/tabutil.h @@ -18,7 +18,8 @@ TABLE_SHARE *GetTableShare(PGLOBAL g, THD *thd, const char *db, PQRYRES TabColumns(PGLOBAL g, THD *thd, const char *db, const char *name, bool& info); -void Remove_tshp(PCATLG cat); +TABLE_SHARE *Remove_tshp(PCATLG cat); +void Restore_tshp(PCATLG cat, TABLE_SHARE *s); /* -------------------------- PROXY classes -------------------------- */ diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 40d020202ea..0ed466f6ffb 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -115,11 +115,14 @@ bool VCTDEF::DefineAM(PGLOBAL g, LPCSTR, int poff) Recfm = RECFM_VCT; + // poff is no more in use; This will have to be revisited +#if 0 // For packed files the logical record length is calculated in poff if (poff != Lrecl) { Lrecl = poff; SetIntCatInfo("Lrecl", poff); } // endif poff +#endif // 0 Padded = false; Blksize = 0; diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index efefc17b5f5..db43f800a91 100644 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -659,7 +659,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /* Not true for DBF tables because of eventual soft deleted lines. */ /* Note: for Num_K = 1 any non null value is Ok. */ /*********************************************************************/ - if (Srtd && !filp && Tdbp->Ftype != RECFM_VAR + if (Srtd && !filp && Tdbp->Ftype != RECFM_VAR && Tdbp->Ftype != RECFM_CSV && Tdbp->Txfp->GetAmType() != TYPE_AM_DBF) { Incr = (Num_K > 1) ? To_Rec[1] : Num_K; PlgDBfree(Record); @@ -837,7 +837,8 @@ bool XINDEX::SaveIndex(PGLOBAL g, PIXDEF sxp) case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; - case RECFM_DBF: ftype = ".dbx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; default: sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype); return true; @@ -990,7 +991,8 @@ bool XINDEX::Init(PGLOBAL g) case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; - case RECFM_DBF: ftype = ".dbx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; default: sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype); return true; @@ -1243,7 +1245,8 @@ bool XINDEX::MapInit(PGLOBAL g) case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; - case RECFM_DBF: ftype = ".dbx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; default: sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype); return true; @@ -1457,7 +1460,8 @@ bool XINDEX::GetAllSizes(PGLOBAL g,/* int &ndif,*/ int &numk) case RECFM_FIX: ftype = ".fnx"; break; case RECFM_BIN: ftype = ".bnx"; break; case RECFM_VCT: ftype = ".vnx"; break; - case RECFM_DBF: ftype = ".dbx"; break; + case RECFM_CSV: ftype = ".cnx"; break; + case RECFM_DBF: ftype = ".dbx"; break; default: sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype); return true; From 9c0e462ff28b93127406eeb2db4b763423fabbe3 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 16 Nov 2019 15:59:16 +0100 Subject: [PATCH 03/32] Fix missing declaration in tabrest.cpp causing compiling failure on Linux --- storage/connect/tabrest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index dde0411592e..de8383a838d 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -133,7 +133,7 @@ XGETREST GetRestFunction(PGLOBAL g) } // endif Hdll // Get the function returning an instance of the external DEF class - if (!(getRestFnc = (XGETREST)dlsym(Hdll, "restGetFile"))) { + if (!(getRestFnc = (XGETREST)dlsym(Hso, "restGetFile"))) { error = dlerror(); sprintf(g->Message, MSG(GET_FUNC_ERR), "restGetFile", SVP(error)); dlclose(Hso); From 290972563673a7de32e10bb0e4bdb37295be1371 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Mon, 18 Nov 2019 11:50:58 +0530 Subject: [PATCH 04/32] MDEV-21044: Wrong result when using a smaller size for sort buffer Make sure that the sort buffers can store atleast one sort key. This is needed to make sure that all merge buffers are read else with no sort keys some merge buffers are skipped because the code makes a conclusion there is no data to be read. --- mysql-test/r/order_by.result | 30 ++++++++++++++++++++++++++++++ mysql-test/t/order_by.test | 16 ++++++++++++++++ sql/filesort.cc | 1 + 3 files changed, 47 insertions(+) diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 4cd9aebdf49..380687554d7 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -3207,3 +3207,33 @@ pk 2 3 DROP TABLE t1; +# +# MDEV-21044: Wrong result when using a smaller size for sort buffer +# +create table t1(a varchar(765),b int); +insert into t1 values ("a",1),("b",2),("c",3),("e",4); +insert into t1 values ("d",5),("f",6),("g",7),("h",8); +insert into t1 values ("k",11),("l",12),("i",9),("j",10); +insert into t1 values ("m",13),("n",14),("o",15),("p",16); +set @save_sort_buffer_size= @@sort_buffer_size; +set sort_buffer_size=1024; +select * from t1 order by b; +a b +a 1 +b 2 +c 3 +e 4 +d 5 +f 6 +g 7 +h 8 +i 9 +j 10 +k 11 +l 12 +m 13 +n 14 +o 15 +p 16 +set @@sort_buffer_size= @save_sort_buffer_size; +drop table t1; diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index 1ca258d1d48..999c7314139 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -2141,3 +2141,19 @@ INSERT INTO t1 VALUES (1),(2),(3); SELECT DISTINCT pk FROM t1 GROUP BY 'foo'; SELECT DISTINCT pk FROM t1; DROP TABLE t1; + +--echo # +--echo # MDEV-21044: Wrong result when using a smaller size for sort buffer +--echo # + +create table t1(a varchar(765),b int); +insert into t1 values ("a",1),("b",2),("c",3),("e",4); +insert into t1 values ("d",5),("f",6),("g",7),("h",8); +insert into t1 values ("k",11),("l",12),("i",9),("j",10); +insert into t1 values ("m",13),("n",14),("o",15),("p",16); +set @save_sort_buffer_size= @@sort_buffer_size; +set sort_buffer_size=1024; +select * from t1 order by b; +set @@sort_buffer_size= @save_sort_buffer_size; +drop table t1; + diff --git a/sql/filesort.cc b/sql/filesort.cc index 4f195f68059..bb3e73343ad 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -343,6 +343,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length, param.max_keys_per_buffer=((param.max_keys_per_buffer * (param.rec_length + sizeof(char*))) / param.rec_length - 1); + set_if_bigger(param.max_keys_per_buffer, 1); maxbuffer--; // Offset from 0 if (merge_many_buff(¶m, (uchar*) table_sort.get_sort_keys(), From 5c68343db713b9a2881809f6e21107d73a7d7b46 Mon Sep 17 00:00:00 2001 From: seppo Date: Mon, 18 Nov 2019 15:18:00 +0200 Subject: [PATCH 05/32] MDEV-18497 CTAS async replication from mariadb master crashes galera nodes (#1410) This PR contains a mtr test for reproducing a failure with replicating create table as select statement (CTAS) through asynchronous mariadb replication to mariadb galera cluster. The problem happens when CTAS replication contains both create table statement followed by row events for populating the table. In such situation, the galera node operating as mariadb replication slave, will first replicate only the create table part into the cluster, and then perform another replication containing both the create table and row events. This will lead all other nodes to fail for duplicate table create attempt, and crash due to this failure. PR contains also a fix, which identifies the situation when CTAS has been replicated, and makes further scan in async replication stream to see if there are following row events. The slave node will replicate either single TOI in case the CTAS table is empty, or if CTAS table contains rows, then single bundled write set with create table and row events is replicated to galera cluster. This fix should keep master server's GTID's for CTAS replication in sync with GTID's in galera cluster. --- .../galera/r/galera_as_slave_ctas.result | 14 ++++ .../suite/galera/t/galera_as_slave_ctas.cnf | 5 ++ .../suite/galera/t/galera_as_slave_ctas.test | 75 +++++++++++++++++++ sql/log_event.h | 3 + sql/slave.cc | 26 +++++++ sql/wsrep_mysqld.cc | 34 ++++++++- 6 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/galera_as_slave_ctas.result create mode 100644 mysql-test/suite/galera/t/galera_as_slave_ctas.cnf create mode 100644 mysql-test/suite/galera/t/galera_as_slave_ctas.test diff --git a/mysql-test/suite/galera/r/galera_as_slave_ctas.result b/mysql-test/suite/galera/r/galera_as_slave_ctas.result new file mode 100644 index 00000000000..79114824008 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_as_slave_ctas.result @@ -0,0 +1,14 @@ +START SLAVE; +SHOW VARIABLES LIKE 'binlog_format'; +Variable_name Value +binlog_format ROW +CREATE TABLE source (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; +CREATE TABLE target AS SELECT * FROM source; +DROP TABLE target; +INSERT INTO source VALUES(1); +CREATE TABLE target AS SELECT * FROM source; +DROP TABLE source; +DROP TABLE target; +STOP SLAVE; +RESET SLAVE ALL; +RESET MASTER; diff --git a/mysql-test/suite/galera/t/galera_as_slave_ctas.cnf b/mysql-test/suite/galera/t/galera_as_slave_ctas.cnf new file mode 100644 index 00000000000..eab2a6de90d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_as_slave_ctas.cnf @@ -0,0 +1,5 @@ +!include ../galera_2nodes_as_slave.cnf + +# make sure master server uses ROW format for replication +[mysqld] +binlog-format=row diff --git a/mysql-test/suite/galera/t/galera_as_slave_ctas.test b/mysql-test/suite/galera/t/galera_as_slave_ctas.test new file mode 100644 index 00000000000..7336449a3d0 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_as_slave_ctas.test @@ -0,0 +1,75 @@ +# +# Test Galera as a slave to a MySQL master +# +# The galera/galera_2node_slave.cnf describes the setup of the nodes +# also, for this test, master server must have binlog_format=ROW +# + +--source include/have_innodb.inc + +# As node #1 is not a Galera node, we connect to node #2 in order to run include/galera_cluster.inc +--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--source include/galera_cluster.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 + +--connection node_2 +--disable_query_log +--eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_1; +--enable_query_log +START SLAVE; + + +# make sure master server has binlog_format=ROW +--connection node_1 +SHOW VARIABLES LIKE 'binlog_format'; + +# +# test phase one, issue CTAS with empty source table +# +--connection node_1 +CREATE TABLE source (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; + +CREATE TABLE target AS SELECT * FROM source; + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'target'; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'target'; +--source include/wait_condition.inc + +# +# test phase two, issue CTAS with populated source table +# +--connection node_1 +DROP TABLE target; +INSERT INTO source VALUES(1); + +CREATE TABLE target AS SELECT * FROM source; + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM target; +--source include/wait_condition.inc + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM target; +--source include/wait_condition.inc + +--connection node_1 +DROP TABLE source; +DROP TABLE target; + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'target'; +--source include/wait_condition.inc + + +--connection node_2 +STOP SLAVE; +RESET SLAVE ALL; + +--connection node_1 +RESET MASTER; +--sleep 20 diff --git a/sql/log_event.h b/sql/log_event.h index 9c555c54005..2c8dc3d7353 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -4967,6 +4967,9 @@ bool event_that_should_be_ignored(const char *buf); bool event_checksum_test(uchar *buf, ulong event_len, enum_binlog_checksum_alg alg); enum enum_binlog_checksum_alg get_checksum_alg(const char* buf, ulong len); extern TYPELIB binlog_checksum_typelib; +#ifdef WITH_WSREP +enum Log_event_type wsrep_peak_event(rpl_group_info *rgi, ulonglong* event_size); +#endif /* WITH_WSREP */ /** @} (end of group Replication) diff --git a/sql/slave.cc b/sql/slave.cc index 0ebed343a3d..bf8b8a43b15 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7220,7 +7220,33 @@ err: sql_print_error("Error reading relay log event: %s", errmsg); DBUG_RETURN(0); } +#ifdef WITH_WSREP +enum Log_event_type wsrep_peak_event(rpl_group_info *rgi, ulonglong* event_size) +{ + mysql_mutex_lock(&rgi->rli->data_lock); + unsigned long long event_pos= rgi->event_relay_log_pos; + unsigned long long future_pos= rgi->future_event_relay_log_pos; + + /* scan the log to read next event */ + my_b_seek(rgi->rli->cur_log, future_pos); + rgi->rli->event_relay_log_pos= future_pos; + rgi->event_relay_log_pos= future_pos; + + Log_event* ev = next_event(rgi, event_size); + enum Log_event_type ev_type= (ev) ? ev->get_type_code() : UNKNOWN_EVENT; + delete ev; + + /* scan the log back and re-set the positions to original values */ + rgi->rli->event_relay_log_pos= event_pos; + rgi->event_relay_log_pos= event_pos; + my_b_seek(rgi->rli->cur_log, future_pos); + + mysql_mutex_unlock(&rgi->rli->data_lock); + + return ev_type; +} +#endif /* WITH_WSREP */ /* Rotate a relay log (this is used only by FLUSH LOGS; the automatic rotation because of size is simpler because when we do it we already have all relevant diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 0a519be073c..0627127c6e0 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -37,7 +37,6 @@ #include #include #include "log_event.h" -#include #include "sql_plugin.h" /* wsrep_plugins_pre_init() */ wsrep_t *wsrep = NULL; @@ -1502,6 +1501,39 @@ static bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table, { return false; } + /* + If mariadb master has replicated a CTAS, we should not replicate the create table + part separately as TOI, but to replicate both create table and following inserts + as one write set. + Howver, if CTAS creates empty table, we should replicate the create table alone + as TOI. We have to do relay log event lookup to see if row events follow the + create table event. + */ + if (thd->slave_thread && !(thd->rgi_slave->gtid_ev_flags2 & Gtid_log_event::FL_STANDALONE)) + { + /* this is CTAS, either empty or populated table */ + ulonglong event_size = 0; + enum Log_event_type ev_type= wsrep_peak_event(thd->rgi_slave, &event_size); + switch (ev_type) + { + case QUERY_EVENT: + /* CTAS with empty table, we replicate create table as TOI */ + break; + + case TABLE_MAP_EVENT: + WSREP_DEBUG("replicating CTAS of empty table as TOI"); + // fall through + case WRITE_ROWS_EVENT: + /* CTAS with populated table, we replicate later at commit time */ + WSREP_DEBUG("skipping create table of CTAS replication"); + return false; + + default: + WSREP_WARN("unexpected async replication event: %d", ev_type); + } + return true; + } + /* no next async replication event */ return true; case SQLCOM_CREATE_VIEW: From c6b097ab3702fe993ee81899fefa9779b8b969e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 18 Nov 2019 15:22:01 +0200 Subject: [PATCH 06/32] Remove excessive sleep from test. --- mysql-test/suite/galera/t/galera_as_slave_ctas.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/t/galera_as_slave_ctas.test b/mysql-test/suite/galera/t/galera_as_slave_ctas.test index 7336449a3d0..1a5e023b981 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_ctas.test +++ b/mysql-test/suite/galera/t/galera_as_slave_ctas.test @@ -72,4 +72,4 @@ RESET SLAVE ALL; --connection node_1 RESET MASTER; ---sleep 20 + From 8fbfcce911d43091f34e1412269db785a0273641 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 19 Nov 2019 17:09:43 +0700 Subject: [PATCH 07/32] cleanup: remove always true condition to fix clang warning --- storage/connect/inihandl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/connect/inihandl.cpp b/storage/connect/inihandl.cpp index 8e79aeac7ef..dacab3c485c 100644 --- a/storage/connect/inihandl.cpp +++ b/storage/connect/inihandl.cpp @@ -194,7 +194,7 @@ static void PROFILE_Save( FILE *file, PROFILESECTION *section ) } for (key = section->key; key; key = key->next) - if (key->name && key->name[0]) { + if (key->name[0]) { fprintf(file, "%s", SVP(key->name)); if (key->value) From a51f3b09bb99f2927ce107727316caac2c426eb1 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Thu, 21 Nov 2019 00:32:27 +0700 Subject: [PATCH 08/32] cleanup DBUG DbugParse(): removed mutex lock/unlock which should protect file writes only. And no file writes happen in this function. DbugFlush(): move mutex_unlock out of this method because fflush() doesn't need any locking. Slow stuff like mutex lock/unlock and accessing errno (TLS) is moved to a more narrow scope. --- dbug/dbug.c | 67 ++++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index 54eebfe55a5..78605983ccb 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -322,6 +322,17 @@ static void DbugVfprintf(FILE *stream, const char* format, va_list args); #include static pthread_mutex_t THR_LOCK_dbug; +static void LockMutex(CODE_STATE *cs) +{ + if (!cs->locked) + pthread_mutex_lock(&THR_LOCK_dbug); +} +static void UnlockMutex(CODE_STATE *cs) +{ + if (!cs->locked) + pthread_mutex_unlock(&THR_LOCK_dbug); +} + static CODE_STATE *code_state(void) { CODE_STATE *cs, **cs_ptr; @@ -449,16 +460,9 @@ static int DbugParse(CODE_STATE *cs, const char *control) const char *end; int rel, f_used=0; struct settings *stack; - int org_cs_locked; stack= cs->stack; - if (!(org_cs_locked= cs->locked)) - { - cs->locked= 1; - pthread_mutex_lock(&THR_LOCK_dbug); - } - if (control[0] == '-' && control[1] == '#') control+=2; @@ -661,11 +665,6 @@ static int DbugParse(CODE_STATE *cs, const char *control) control=end+1; end= DbugStrTok(control); } - if (!org_cs_locked) - { - pthread_mutex_unlock(&THR_LOCK_dbug); - cs->locked= 0; - } return !rel || f_used; } @@ -1093,7 +1092,6 @@ int _db_explain_init_(char *buf, size_t len) void _db_enter_(const char *_func_, const char *_file_, uint _line_, struct _db_stack_frame_ *_stack_frame_) { - int save_errno; CODE_STATE *cs; if (!((cs=code_state()))) { @@ -1101,7 +1099,6 @@ void _db_enter_(const char *_func_, const char *_file_, _stack_frame_->prev= 0; return; } - save_errno= errno; _stack_frame_->line= -1; _stack_frame_->func= cs->func; @@ -1122,12 +1119,14 @@ void _db_enter_(const char *_func_, const char *_file_, cs->stack->flags &= ~SANITY_CHECK_ON; if (TRACING) { - if (!cs->locked) - pthread_mutex_lock(&THR_LOCK_dbug); + int save_errno= errno; + LockMutex(cs); DoPrefix(cs, _line_); Indent(cs, cs->level); (void) fprintf(cs->stack->out_file->file, ">%s\n", cs->func); - DbugFlush(cs); /* This does a unlock */ + UnlockMutex(cs); + DbugFlush(cs); + errno=save_errno; } break; case DISABLE_TRACE: @@ -1136,7 +1135,6 @@ void _db_enter_(const char *_func_, const char *_file_, case DONT_TRACE: break; } - errno=save_errno; } /* @@ -1161,7 +1159,6 @@ void _db_enter_(const char *_func_, const char *_file_, void _db_return_(struct _db_stack_frame_ *_stack_frame_) { - int save_errno=errno; uint _slevel_= _stack_frame_->level & ~TRACE_ON; CODE_STATE *cs; get_code_state_or_return; @@ -1182,12 +1179,14 @@ void _db_return_(struct _db_stack_frame_ *_stack_frame_) cs->stack->flags &= ~SANITY_CHECK_ON; if (TRACING) { - if (!cs->locked) - pthread_mutex_lock(&THR_LOCK_dbug); + int save_errno=errno; + LockMutex(cs); DoPrefix(cs, _stack_frame_->line); Indent(cs, cs->level); (void) fprintf(cs->stack->out_file->file, "<%s\n", cs->func); + UnlockMutex(cs); DbugFlush(cs); + errno=save_errno; } } /* @@ -1199,7 +1198,6 @@ void _db_return_(struct _db_stack_frame_ *_stack_frame_) cs->file= _stack_frame_->file; if (cs->framep != NULL) cs->framep= cs->framep->prev; - errno=save_errno; } @@ -1264,27 +1262,24 @@ void _db_doprnt_(const char *format,...) CODE_STATE *cs; get_code_state_or_return; - va_start(args,format); - if (!cs->locked) - pthread_mutex_lock(&THR_LOCK_dbug); if (_db_keyword_(cs, cs->u_keyword, 0)) { int save_errno=errno; + LockMutex(cs); DoPrefix(cs, cs->u_line); if (TRACING) Indent(cs, cs->level + 1); else (void) fprintf(cs->stack->out_file->file, "%s: ", cs->func); (void) fprintf(cs->stack->out_file->file, "%s: ", cs->u_keyword); + va_start(args,format); DbugVfprintf(cs->stack->out_file->file, format, args); + UnlockMutex(cs); + va_end(args); DbugFlush(cs); errno=save_errno; } - else if (!cs->locked) - pthread_mutex_unlock(&THR_LOCK_dbug); - - va_end(args); } /* @@ -1325,10 +1320,9 @@ void _db_dump_(uint _line_, const char *keyword, CODE_STATE *cs; get_code_state_or_return; - if (!cs->locked) - pthread_mutex_lock(&THR_LOCK_dbug); if (_db_keyword_(cs, keyword, 0)) { + LockMutex(cs); DoPrefix(cs, _line_); if (TRACING) { @@ -1356,10 +1350,9 @@ void _db_dump_(uint _line_, const char *keyword, fputc(' ',cs->stack->out_file->file); } (void) fputc('\n',cs->stack->out_file->file); + UnlockMutex(cs); DbugFlush(cs); } - else if (!cs->locked) - pthread_mutex_unlock(&THR_LOCK_dbug); } @@ -1938,16 +1931,16 @@ static void DBUGCloseFile(CODE_STATE *cs, sFILE *new_value) sFILE *fp; if (!cs || !cs->stack || !cs->stack->out_file) return; - if (!cs->locked) - pthread_mutex_lock(&THR_LOCK_dbug); fp= cs->stack->out_file; if (--fp->used == 0) { if (fclose(fp->file) == EOF) { + LockMutex(cs); (void) fprintf(stderr, ERR_CLOSE, cs->process); perror(""); + UnlockMutex(cs); } else { @@ -1955,8 +1948,6 @@ static void DBUGCloseFile(CODE_STATE *cs, sFILE *new_value) } } cs->stack->out_file= new_value; - if (!cs->locked) - pthread_mutex_unlock(&THR_LOCK_dbug); } @@ -2128,8 +2119,6 @@ static void DbugFlush(CODE_STATE *cs) if (cs->stack->delay) (void) Delay(cs->stack->delay); } - if (!cs->locked) - pthread_mutex_unlock(&THR_LOCK_dbug); } /* DbugFlush */ From 7a9eca1191c315b6009e4356d96bd3c4b9fa2415 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 21 Nov 2019 18:33:52 +0100 Subject: [PATCH 09/32] Fix compile error (imcomplete switch) in ha_connect.cc --- storage/connect/ha_connect.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 2bc9537f81b..79175724158 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1045,6 +1045,8 @@ TABTYPE ha_connect::GetRealType(PTOS pos) case TAB_REST: type = TAB_NIY; break; + default: + break; } // endswitch type #endif // REST_SUPPORT @@ -5634,6 +5636,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_CSV: ttp = TAB_REST; break; + default: + break; } // endswitch type #endif // REST_SUPPORT } // endif ttp From 420789512fd0f90b150e5ccb61d127e6c80d712a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 21 Nov 2019 19:38:17 +0100 Subject: [PATCH 10/32] Fix compile error (missing declaration) in tabrest.cpp --- storage/connect/tabrest.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index de8383a838d..d070368cfd2 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -36,11 +36,6 @@ #include "tabfmt.h" #include "tabrest.h" -/***********************************************************************/ -/* Get the file from the Web. */ -/***********************************************************************/ -int restGetFile(char *m, bool x, PCSZ http, PCSZ uri, PCSZ fn); - #if defined(__WIN__) static PCSZ slash = "\\"; #else // !__WIN__ @@ -51,6 +46,11 @@ static PCSZ slash = "/"; typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); static XGETREST getRestFnc = NULL; +/***********************************************************************/ +/* Some stupid compilers need these. */ +/***********************************************************************/ +XGETREST GetRestFunction(PGLOBAL g); +int restGetFile(char* m, bool x, PCSZ http, PCSZ uri, PCSZ fn); #if !defined(MARIADB) /***********************************************************************/ From b4bfa12b0044f7e4659e919784a6ed2252ca21b8 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 21 Nov 2019 20:12:06 +0100 Subject: [PATCH 11/32] Fix compile error (missing declaration) in tabrest.cpp --- storage/connect/tabrest.cpp | 14 -------------- storage/connect/tabrest.h | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index d070368cfd2..fe81c0a65be 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -36,22 +36,8 @@ #include "tabfmt.h" #include "tabrest.h" -#if defined(__WIN__) -static PCSZ slash = "\\"; -#else // !__WIN__ -static PCSZ slash = "/"; -#define stricmp strcasecmp -#endif // !__WIN__ - -typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); static XGETREST getRestFnc = NULL; -/***********************************************************************/ -/* Some stupid compilers need these. */ -/***********************************************************************/ -XGETREST GetRestFunction(PGLOBAL g); -int restGetFile(char* m, bool x, PCSZ http, PCSZ uri, PCSZ fn); - #if !defined(MARIADB) /***********************************************************************/ /* DB static variables. */ diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h index 1725f256079..d945c97e5f1 100644 --- a/storage/connect/tabrest.h +++ b/storage/connect/tabrest.h @@ -5,6 +5,25 @@ /***********************************************************************/ #pragma once +#if defined(__WIN__) +static PCSZ slash = "\\"; +#else // !__WIN__ +static PCSZ slash = "/"; +#define stricmp strcasecmp +#endif // !__WIN__ + +typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ); + +/***********************************************************************/ +/* Functions used by REST. */ +/***********************************************************************/ +XGETREST GetRestFunction(PGLOBAL g); +int restGetFile(char* m, bool x, PCSZ http, PCSZ uri, PCSZ fn); +#if defined(MARIADB) +PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char* tab, char* db, bool info); +#endif // !MARIADB + + /***********************************************************************/ /* Restest table. */ /***********************************************************************/ From d8ace23d260033fc6588599e0a03e2832d20dae9 Mon Sep 17 00:00:00 2001 From: Hashir Sarwar Date: Wed, 23 Oct 2019 17:40:24 +0500 Subject: [PATCH 12/32] Fixed some typos in mysql.cc Closes #1403 --- client/mysql.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 50d45696a5c..c872968b78d 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -3177,7 +3177,7 @@ static int com_go(String *buffer,char *line __attribute__((unused))) { char buff[200]; /* about 110 chars used so far */ - char time_buff[52+3+1]; /* time max + space&parens + NUL */ + char time_buff[52+3+1]; /* time max + space & parens + NUL */ MYSQL_RES *result; ulong timer, warnings= 0; uint error= 0; @@ -3196,7 +3196,7 @@ com_go(String *buffer,char *line __attribute__((unused))) if (buffer->is_empty()) { - if (status.batch) // Ignore empty quries + if (status.batch) // Ignore empty queries. return 0; return put_info("No query specified\n",INFO_ERROR); @@ -3261,7 +3261,7 @@ com_go(String *buffer,char *line __attribute__((unused))) else time_buff[0]= '\0'; - /* Every branch must truncate buff . */ + /* Every branch must truncate buff. */ if (result) { if (!mysql_num_rows(result) && ! quick && !column_types_flag) From fb91774e4e6451298af51e31356a8b8fb1b19487 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 24 Nov 2019 22:58:55 +0100 Subject: [PATCH 13/32] These changed were made after pulling 10.2.30 from origin --------------------------------------------------------- - Temporarily fix MDEV-13782 by commenting out LIKE_FUNC in CondFilter modified: storage/connect/ha_connect.cc - Remove use of hack tables modified: storage/connect/connect.cc modified: storage/connect/connect.h modified: storage/connect/ha_connect.cc modified: storage/connect/xtable.h --- storage/connect/connect.cc | 64 +++++++++++++++++------------------ storage/connect/connect.h | 2 ++ storage/connect/ha_connect.cc | 14 ++++---- storage/connect/xtable.h | 5 ++- 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index 461391d2896..dfc619cf4af 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -566,7 +566,7 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) rc = tdbp->DeleteDB(g, RC_EF); // Specific A.M. delete routine } else if (tdbp->GetMode() == MODE_UPDATE && tdbp->IsIndexed()) - rc = ((PTDBDOX)tdbp)->Txfp->UpdateSortedRows(g); + rc = ((PTDBDOS)tdbp)->GetTxfp()->UpdateSortedRows(g); switch (rc) { case RC_FX: @@ -593,7 +593,7 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) if (!tdbp->IsRemote()) { // Make all the eventual indexes - PTDBDOX tbxp = (PTDBDOX)tdbp; + PTDBDOS tbxp = (PTDBDOS)tdbp; tbxp->ResetKindex(g, NULL); tbxp->SetKey_Col(NULL); rc = tbxp->ResetTableOpt(g, true, tbxp->GetDef()->Indexable() == 1); @@ -622,8 +622,8 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp, bool nox, bool abort) int CntIndexInit(PGLOBAL g, PTDB ptdb, int id, bool sorted) { PIXDEF xdp; - PTDBDOX tdbp; - DOXDEF *dfp; + PTDBDOS tdbp; + DOSDEF *dfp; if (!ptdb) return -1; @@ -633,9 +633,9 @@ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id, bool sorted) } else if (ptdb->GetDef()->Indexable() == 3) { return 1; } else - tdbp= (PTDBDOX)ptdb; + tdbp= (PTDBDOS)ptdb; - dfp= (DOXDEF*)tdbp->To_Def; + dfp= (DOSDEF*)tdbp->GetDef(); //if (!(k= colp->GetKey())) // if (colp->GetOpt() >= 2) { @@ -645,16 +645,16 @@ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id, bool sorted) // This is a pseudo indexed sorted block optimized column // return 0; - if (tdbp->To_Kindex) - if (((XXBASE*)tdbp->To_Kindex)->GetID() == id) { - tdbp->To_Kindex->Reset(); // Same index - return (tdbp->To_Kindex->IsMul()) ? 2 : 1; + if (tdbp->GetKindex()) + if (((XXBASE*)tdbp->GetKindex())->GetID() == id) { + tdbp->GetKindex()->Reset(); // Same index + return (tdbp->GetKindex()->IsMul()) ? 2 : 1; } else { - tdbp->To_Kindex->Close(); - tdbp->To_Kindex= NULL; + tdbp->GetKindex()->Close(); + tdbp->SetKindex(NULL); } // endif colp - for (xdp= dfp->To_Indx; xdp; xdp= xdp->GetNext()) + for (xdp= dfp->GetIndx(); xdp; xdp= xdp->GetNext()) if (xdp->GetID() == id) break; @@ -676,7 +676,7 @@ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id, bool sorted) if (tdbp->InitialyzeIndex(g, xdp, sorted)) return 0; - return (tdbp->To_Kindex->IsMul()) ? 2 : 1; + return (tdbp->GetKindex()->IsMul()) ? 2 : 1; } // end of CntIndexInit #if defined(WORDS_BIGENDIAN) @@ -710,7 +710,7 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, int n, x; RCODE rc; XXBASE *xbp; - PTDBDOX tdbp; + PTDBDOS tdbp; if (!ptdb) return RC_FX; @@ -736,12 +736,12 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, goto rnd; } else - tdbp= (PTDBDOX)ptdb; + tdbp= (PTDBDOS)ptdb; // Set reference values and index operator - if (!tdbp->To_Link || !tdbp->To_Kindex) { + if (!tdbp->GetLink() || !tdbp->GetKindex()) { // if (!tdbp->To_Xdp) { - sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); + sprintf(g->Message, "Index not initialized for table %s", tdbp->GetName()); return RC_FX; #if 0 } // endif !To_Xdp @@ -754,7 +754,7 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, #endif // 0 } // endif !To_Kindex - xbp= (XXBASE*)tdbp->To_Kindex; + xbp= (XXBASE*)tdbp->GetKindex(); if (kr) { char *kp= (char*)kr->key; @@ -764,13 +764,13 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, PVAL valp; PCOL colp; - for (n= 0; n < tdbp->Knum; n++) { - colp= (PCOL)tdbp->To_Key_Col[n]; + for (n= 0; n < tdbp->GetKnum(); n++) { + colp= (PCOL)tdbp->Key(n); if (colp->GetColUse(U_NULLS)) kp++; // Skip null byte - valp= tdbp->To_Link[n]->GetValue(); + valp= tdbp->Link(n)->GetValue(); if (!valp->IsTypeNum()) { if (colp->GetColUse(U_VAR)) { @@ -840,7 +840,7 @@ int CntIndexRange(PGLOBAL g, PTDB ptdb, const uchar* *key, uint *len, bool b, rcb; PVAL valp; PCOL colp; - PTDBDOX tdbp; + PTDBDOS tdbp; XXBASE *xbp; if (!ptdb) @@ -865,35 +865,35 @@ int CntIndexRange(PGLOBAL g, PTDB ptdb, const uchar* *key, uint *len, return k[1] - k[0] + 1; } else - tdbp= (PTDBDOX)ptdb; + tdbp= (PTDBDOS)ptdb; - if (!tdbp->To_Kindex || !tdbp->To_Link) { - if (!tdbp->To_Xdp) { - sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); + if (!tdbp->GetKindex() || !tdbp->GetLink()) { + if (!tdbp->GetXdp()) { + sprintf(g->Message, "Index not initialized for table %s", tdbp->GetName()); DBUG_PRINT("Range", ("%s", g->Message)); return -1; } else // Dynamic index - return tdbp->To_Xdp->GetMaxSame(); // TODO a better estimate + return tdbp->GetXdp()->GetMaxSame(); // TODO a better estimate } else - xbp= (XXBASE*)tdbp->To_Kindex; + xbp= (XXBASE*)tdbp->GetKindex(); for (b= false, i= 0; i < 2; i++) { p= kp= key[i]; if (kp) { - for (n= 0; n < tdbp->Knum; n++) { + for (n= 0; n < tdbp->GetKnum(); n++) { if (kmap[i] & (key_part_map)(1 << n)) { if (b == true) // Cannot do indexing with missing intermediate key return -1; - colp= (PCOL)tdbp->To_Key_Col[n]; + colp= (PCOL)tdbp->Key(n); if (colp->GetColUse(U_NULLS)) p++; // Skip null byte ??? - valp= tdbp->To_Link[n]->GetValue(); + valp= tdbp->Link(n)->GetValue(); if (!valp->IsTypeNum()) { if (colp->GetColUse(U_VAR)) { diff --git a/storage/connect/connect.h b/storage/connect/connect.h index 3a60cd40160..d1fc2ea592f 100644 --- a/storage/connect/connect.h +++ b/storage/connect/connect.h @@ -46,6 +46,7 @@ int CntIndexRange(PGLOBAL g, PTDB ptdb, const uchar* *key, uint *len, bool *incl, key_part_map *kmap); PGLOBAL CntExit(PGLOBAL g); +#if 0 /***********************************************************************/ /* Definition of classes XKPDEF, DOXDEF, TDBDOX */ /* These classes purpose is chiefly to access protected items! */ @@ -76,3 +77,4 @@ class XKPDEF: public KPARTDEF { public: XKPDEF(const char *name, int n) : KPARTDEF((PSZ)name, n) {} }; // end of class XKPDEF +#endif // 0 diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 79175724158..91fef719ee7 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -2966,12 +2966,10 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) case Item_func::LE_FUNC: vop= OP_LE; break; case Item_func::GE_FUNC: vop= OP_GE; break; case Item_func::GT_FUNC: vop= OP_GT; break; -#if 0 - case Item_func::LIKE_FUNC: - vop= OP_LIKE; - neg= ((Item_func_like*)condf)->negated; - break; -#endif // 0 + //case Item_func::LIKE_FUNC: + // vop = OP_LIKE; + // neg= ((Item_func_like*)condf)->negated; + // break; case Item_func::ISNOTNULL_FUNC: neg= true; // fall through @@ -3789,9 +3787,9 @@ int ha_connect::index_init(uint idx, bool sorted) active_index= MAX_KEY; rc= HA_ERR_INTERNAL_ERROR; } else if (tdbp->GetKindex()) { - if (((PTDBDOX)tdbp)->To_Kindex->GetNum_K()) { + if (((PTDBDOS)tdbp)->GetKindex()->GetNum_K()) { if (tdbp->GetFtype() != RECFM_NAF) - ((PTDBDOX)tdbp)->GetTxfp()->ResetBuffer(g); + ((PTDBDOS)tdbp)->GetTxfp()->ResetBuffer(g); active_index= idx; // } else { // Void table diff --git a/storage/connect/xtable.h b/storage/connect/xtable.h index bc9265e0223..1b499e09047 100644 --- a/storage/connect/xtable.h +++ b/storage/connect/xtable.h @@ -173,9 +173,12 @@ class DllExport TDBASE : public TDB { inline void SetKindex(PKXBASE kxp) {To_Kindex = kxp;} // Properties - virtual PKXBASE GetKindex(void) {return To_Kindex;} + PKXBASE GetKindex(void) {return To_Kindex;} + PXOB *GetLink(void) {return To_Link;} + PIXDEF GetXdp(void) {return To_Xdp;} void ResetKindex(PGLOBAL g, PKXBASE kxp); PCOL Key(int i) {return (To_Key_Col) ? To_Key_Col[i] : NULL;} + PXOB Link(int i) { return (To_Link) ? To_Link[i] : NULL; } // Methods virtual bool IsUsingTemp(PGLOBAL) {return false;} From 38839854b7c097b33f34b2228c4240fe837fda79 Mon Sep 17 00:00:00 2001 From: seppo Date: Tue, 26 Nov 2019 08:49:50 +0200 Subject: [PATCH 14/32] MDEV-19572 async slave node fails to apply MyISAM only writes (#1418) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem happens when MariaDB master replicates writes for only non InnoDB tables (e.g. writes to MyISAM table(s)). Async slave node, in Galera cluster, can apply these writes successfully, but it will, in the end, write gtid position in mysql.gtid_slave_pos table. mysql.gtid_slave_pos table is InnoDB engine, and this write makes innodb handlerton part of the replicated "transaction". Note that wsrep patch identifies that write to gtid_slave_pos should not be replicated and skips appending wsrep keys for these writes. However, as InnoDB was present in the transaction, and there are replication events (for MyISAM table) in transaction cache, but there are no appended keys, wsrep raises an error, and this makes the söave thread to stop. The fix is simply to not treat it as an error if async slave tries to replicate a write set with binlog events, but no keys. We just skip wsrep replication and return successfully. This commit contains also a mtr test which forces mysql.gtid_slave_pos table isto be of InnoDB engine, and executes MyISAM only write through asyn replication. There is additional fix for declaring IO and background slave threads as non wsrep. These threads should not write anything for wsrep replication, and this is just a safeguard to make sure nothing leaks into cluster from these slave threads. --- .../r/galera_as_slave_gtid_myisam.result | 21 ++++++ .../galera/t/galera_as_slave_gtid_myisam.cnf | 6 ++ .../galera/t/galera_as_slave_gtid_myisam.test | 65 +++++++++++++++++++ sql/slave.cc | 7 +- sql/wsrep_hton.cc | 19 +++++- 5 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result create mode 100644 mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.cnf create mode 100644 mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result new file mode 100644 index 00000000000..d9a87571655 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result @@ -0,0 +1,21 @@ +ALTER TABLE mysql.gtid_slave_pos engine = InnoDB; +START SLAVE; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES(1); +SELECT LENGTH(@@global.gtid_binlog_state) > 1; +LENGTH(@@global.gtid_binlog_state) > 1 +1 +gtid_binlog_state_equal +0 +SELECT COUNT(*) = 0 FROM t1; +COUNT(*) = 0 +1 +gtid_binlog_state_equal +0 +#cleanup +DROP TABLE t1; +reset master; +STOP SLAVE; +RESET SLAVE ALL; +reset master; +reset master; diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.cnf b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.cnf new file mode 100644 index 00000000000..01d2eb12630 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.cnf @@ -0,0 +1,6 @@ +!include ../galera_2nodes_as_slave.cnf + +[mysqld] +log-bin=mysqld-bin +log-slave-updates +binlog-format=ROW diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test new file mode 100644 index 00000000000..faa9ddfd5c8 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test @@ -0,0 +1,65 @@ +# +# Test Galera as a slave to a MariaDB master using GTIDs +# +# suite/galera/galera_2nodes_as_slave.cnf describes the setup of the nodes +# suite/galera/t/galera_as_slave_gtid.cnf has the GTID options +# +# This test will replicate writes to MyISAM table and check that slave node is able +# to apply them. +# mysql.gtid_slave_pos table should be defined as innodb engine, original problem +# by writes to mysql.gtid_slave_pos, whereas the replicated transaction contained +# no innodb writes +# + +--source include/have_innodb.inc + +# As node #1 is not a Galera node, we connect to node #2 in order to run include/galera_cluster.inc +--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--source include/galera_cluster.inc + +--connection node_2 +# make sure gtid_slave_pos is of innodb engine, mtr does not currently provide that +ALTER TABLE mysql.gtid_slave_pos engine = InnoDB; + +--disable_query_log +--eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_USER='root', MASTER_PORT=$NODE_MYPORT_1; +--enable_query_log +START SLAVE; + +--connection node_1 +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES(1); + +SELECT LENGTH(@@global.gtid_binlog_state) > 1; +--let $gtid_binlog_state_node1 = `SELECT @@global.gtid_binlog_state;` + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + +--disable_query_log +--eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; +--enable_query_log + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +SELECT COUNT(*) = 0 FROM t1; + +--disable_query_log +--eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; +--enable_query_log + +--echo #cleanup +--connection node_1 +DROP TABLE t1; +reset master; + +--connection node_2 +STOP SLAVE; +RESET SLAVE ALL; +reset master; + +--connection node_3 +reset master; diff --git a/sql/slave.cc b/sql/slave.cc index bf8b8a43b15..c877a1e2c2c 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -306,6 +306,9 @@ handle_slave_background(void *arg __attribute__((unused))) thd->store_globals(); thd->security_ctx->skip_grants(); thd->set_command(COM_DAEMON); +#ifdef WITH_WSREP + thd->variables.wsrep_on= 0; +#endif thd_proc_info(thd, "Loading slave GTID position from table"); if (rpl_load_gtid_slave_state(thd)) @@ -4181,7 +4184,9 @@ pthread_handler_t handle_slave_io(void *arg) goto err; } - +#ifdef WITH_WSREP + thd->variables.wsrep_on= 0; +#endif if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi))) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index 8665a8ae63f..d8f82b13108 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -478,12 +478,29 @@ wsrep_run_wsrep_commit(THD *thd, bool all) if (WSREP_UNDEFINED_TRX_ID == thd->wsrep_ws_handle.trx_id) { - WSREP_WARN("SQL statement was ineffective, THD: %lu, buf: %zu\n" + /* + Async replication slave may have applied some non-innodb workload, + and then has written replication "meta data" into gtid_slave_pos + innodb table. Writes to gtid_slave_pos must not be replicated, + but this activity has caused that innodb hton is registered for this + transaction, but no wsrep keys have been appended. + We enter in this code path, because IO cache has events for non-innodb + tables. + => we should not treat it an error if trx is not introduced for provider + */ + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL) + { + WSREP_DEBUG("skipping wsrep replication for async slave, error not raised"); + DBUG_RETURN(WSREP_TRX_OK); + } + + WSREP_WARN("SQL statement was ineffective thd: %lu buf: %zu\n" "schema: %s \n" "QUERY: %s\n" " => Skipping replication", thd->thread_id, data_len, (thd->db ? thd->db : "(null)"), thd->query()); + rcode = WSREP_TRX_FAIL; } else if (!rcode) From 23664bc7a582f03d1ccb680faec8c65f456e0c72 Mon Sep 17 00:00:00 2001 From: Seth Shelnutt Date: Thu, 21 Nov 2019 10:34:36 -0500 Subject: [PATCH 15/32] Fix incorrect DBUG_ENTER message for join_read_last --- sql/sql_select.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 55d371bdbfd..ca53852fa91 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -19570,7 +19570,7 @@ join_read_last(JOIN_TAB *tab) { TABLE *table=tab->table; int error= 0; - DBUG_ENTER("join_read_first"); + DBUG_ENTER("join_read_last"); if (table->covering_keys.is_set(tab->index) && !table->no_keyread && !table->key_read) From ed2379f98399ee3ecd6147c6acddfa33e35a9879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Mon, 25 Nov 2019 11:08:13 +0200 Subject: [PATCH 16/32] MDEV-13288: Upstream debian patch --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 71329a870c9..fc63c308f75 100644 --- a/debian/control +++ b/debian/control @@ -6,6 +6,7 @@ Uploaders: MariaDB Developers Build-Depends: bison, chrpath, cmake (>= 2.7), + cracklib-runtime, debhelper, dh-apparmor, dpatch, From 427eedd0126de0d9562e843eac911380daa9853c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 26 Nov 2019 19:52:37 +0200 Subject: [PATCH 17/32] MDEV-13288: Proper fix for cracklib-runtime The required dependencies should be added through the autobake script, to also cover distributions that do not support libcrack2. --- debian/autobake-deb.sh | 2 +- debian/control | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 98071ba6d03..a617995724c 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -41,7 +41,7 @@ then MARIADB_OPTIONAL_DEBS="${MARIADB_OPTIONAL_DEBS} cracklib-password-check-10.1" sed -i -e "/\\\${MAYBE_LIBCRACK}/d" debian/control else - MAYBE_LIBCRACK='libcrack2-dev (>= 2.9.0),' + MAYBE_LIBCRACK='libcrack2-dev (>= 2.9.0), cracklib-runtime' sed -i -e "s/\\\${MAYBE_LIBCRACK}/${MAYBE_LIBCRACK}/g" debian/control fi diff --git a/debian/control b/debian/control index fc63c308f75..71329a870c9 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,6 @@ Uploaders: MariaDB Developers Build-Depends: bison, chrpath, cmake (>= 2.7), - cracklib-runtime, debhelper, dh-apparmor, dpatch, From f0da39be7fff39f0ee47abe272e0ef96f229c7e3 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Tue, 26 Nov 2019 19:22:46 +0100 Subject: [PATCH 18/32] - Fix MDEV-13782 Problem with NOT LIKE queries. modified: storage/connect/ha_connect.cc modified: sql/item_cmpfunc.h - Fix MDEV-21084 Misusage of strncat could cause buffer overflow. modified: storage/connect/reldef.cpp modified: storage/connect/tabcmg.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabrest.cpp modified: storage/connect/tabxml.cpp --- sql/item_cmpfunc.h | 2 +- storage/connect/ha_connect.cc | 8 +++---- storage/connect/reldef.cpp | 3 ++- storage/connect/tabcmg.cpp | 23 ++++++++++++-------- storage/connect/tabjson.cpp | 40 ++++++++++++++++++++++------------- storage/connect/tabrest.cpp | 5 +++-- storage/connect/tabxml.cpp | 4 +++- 7 files changed, 52 insertions(+), 33 deletions(-) diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 0a5abfe6273..b4a905896db 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1899,7 +1899,6 @@ class Item_func_like :public Item_bool_func2 bool escape_used_in_parsing; bool use_sampling; - bool negated; DTCollation cmp_collation; String cmp_value1, cmp_value2; @@ -1916,6 +1915,7 @@ protected: Item_func::Functype type, Item *value); public: int escape; + bool negated; Item_func_like(THD *thd, Item *a, Item *b, Item *escape_arg, bool escape_used): Item_bool_func2(thd, a, b), canDoTurboBM(FALSE), pattern(0), pattern_len(0), diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 91fef719ee7..cc9e7d1b55c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -2966,10 +2966,10 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) case Item_func::LE_FUNC: vop= OP_LE; break; case Item_func::GE_FUNC: vop= OP_GE; break; case Item_func::GT_FUNC: vop= OP_GT; break; - //case Item_func::LIKE_FUNC: - // vop = OP_LIKE; - // neg= ((Item_func_like*)condf)->negated; - // break; + case Item_func::LIKE_FUNC: + vop = OP_LIKE; + neg= ((Item_func_like*)condf)->negated; + break; case Item_func::ISNOTNULL_FUNC: neg= true; // fall through diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 88c28757588..ffe5f77661d 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -624,7 +624,8 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) return NULL; } else // PlugSetPath(soname, Module, GetPluginDir()); // Crashes on Fedora - strncat(strcpy(soname, GetPluginDir()), Module, _MAX_PATH); + strncat(strcpy(soname, GetPluginDir()), Module, + sizeof(soname) - strlen(soname) - 1); #if defined(__WIN__) // Is the DLL already loaded? diff --git a/storage/connect/tabcmg.cpp b/storage/connect/tabcmg.cpp index da1cfd34ac7..b9b7f6e4b60 100644 --- a/storage/connect/tabcmg.cpp +++ b/storage/connect/tabcmg.cpp @@ -53,25 +53,30 @@ bool CMGDISC::FindInDoc(PGLOBAL g, bson_iter_t *iter, const bson_t *doc, { if (!doc || bson_iter_init(iter, doc)) { const char *key; - char colname[65]; - char fmt[129]; - bool newcol; + char colname[65]; + char fmt[129]; + bool newcol; + size_t n; while (bson_iter_next(iter)) { key = bson_iter_key(iter); newcol = true; if (pcn) { - strncpy(colname, pcn, 64); - colname[64] = 0; - strncat(strncat(colname, "_", 65), key, 65); + n = sizeof(colname) - 1; + strncpy(colname, pcn, n); + colname[n] = 0; + n -= strlen(colname); + strncat(strncat(colname, "_", n), key, n - 1); } else strcpy(colname, key); if (pfmt) { - strncpy(fmt, pfmt, 128); - fmt[128] = 0; - strncat(strncat(fmt, ".", 129), key, 129); + n = sizeof(fmt) - 1; + strncpy(fmt, pfmt, n); + fmt[n] = 0; + n -= strlen(fmt); + strncat(strncat(fmt, ".", n), key, n - 1); } else strcpy(fmt, key); diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 0b282345c8a..7e8d6c8d9f0 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -394,10 +394,11 @@ err: bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) { - char *p, *pc = colname + strlen(colname); - int ars; - PJOB job; - PJAR jar; + char *p, *pc = colname + strlen(colname); + int ars; + size_t n; + PJOB job; + PJAR jar; if ((valp = jvp ? jvp->GetValue() : NULL)) { jcol.Type = valp->GetType(); @@ -423,8 +424,10 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) PCSZ k = jrp->GetKey(); if (*k != '$') { - strncat(strncat(fmt, sep, 128), k, 128); - strncat(strncat(colname, "_", 64), k, 64); + n = sizeof(fmt) - strlen(fmt) -1; + strncat(strncat(fmt, sep, n), k, n - strlen(sep)); + n = sizeof(colname) - strlen(colname) - 1; + strncat(strncat(colname, "_", n), k, n - 1); } // endif Key if (Find(g, jrp->GetVal(), k, j + 1)) @@ -443,19 +446,26 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) ars = MY_MIN(jar->GetSize(false), 1); for (int k = 0; k < ars; k++) { - if (!tdp->Xcol || stricmp(tdp->Xcol, key)) { + n = sizeof(fmt) - (strlen(fmt) + 1); + + if (!tdp->Xcol || stricmp(tdp->Xcol, key)) { sprintf(buf, "%d", k); - if (tdp->Uri) - strncat(strncat(fmt, sep, 128), buf, 128); - else - strncat(strncat(strncat(fmt, "[", 128), buf, 128), "]", 128); + if (tdp->Uri) { + strncat(strncat(fmt, sep, n), buf, n - strlen(sep)); + } else { + strncat(strncat(fmt, "[", n), buf, n - 1); + strncat(fmt, "]", n - (strlen(buf) + 1)); + } // endif uri - if (all) - strncat(strncat(colname, "_", 64), buf, 64); + if (all) { + n = sizeof(colname) - (strlen(colname) + 1); + strncat(strncat(colname, "_", n), buf, n - 1); + } // endif all - } else - strncat(fmt, (tdp->Uri ? sep : "[*]"), 128); + } else { + strncat(fmt, (tdp->Uri ? sep : "[*]"), n); + } if (Find(g, jar->GetValue(k), "", j)) return true; diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index fe81c0a65be..f9acbc28ddc 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -162,7 +162,7 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) // We used the file name relative to recorded datapath strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); - strncat(filename, fn, _MAX_PATH); + strncat(filename, fn, _MAX_PATH - strlen(filename)); // Retrieve the file from the web and copy it locally if (http && grf(g->Message, trace(515), http, uri, filename)) { @@ -221,7 +221,8 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // We used the file name relative to recorded datapath //PlugSetPath(filename, Fn, GetPath()); - strncat(strcpy(filename, GetPath()), Fn, _MAX_PATH); + strcpy(filename, GetPath()); + strncat(filename, Fn, _MAX_PATH - strlen(filename)); // Retrieve the file from the web and copy it locally rc = grf(g->Message, xt, Http, Uri, filename); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 19490d350e8..717090e9c5a 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -240,7 +240,9 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) more: if (vp->atp) { - strncpy(colname, vp->atp->GetName(g), sizeof(colname)); + size_t z = sizeof(colname) - 1; + strncpy(colname, vp->atp->GetName(g), z); + colname[z] = 0; strncat(xcol->Name, colname, XLEN(xcol->Name)); switch (vp->atp->GetText(g, buf, sizeof(buf))) { From 6218bf1b416219aef37301f054136fcdcc404d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 26 Nov 2019 21:48:22 +0200 Subject: [PATCH 19/32] cracklib-runtime needs a comma after to parse properly --- debian/autobake-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index a617995724c..bfdba96e487 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -41,7 +41,7 @@ then MARIADB_OPTIONAL_DEBS="${MARIADB_OPTIONAL_DEBS} cracklib-password-check-10.1" sed -i -e "/\\\${MAYBE_LIBCRACK}/d" debian/control else - MAYBE_LIBCRACK='libcrack2-dev (>= 2.9.0), cracklib-runtime' + MAYBE_LIBCRACK='libcrack2-dev (>= 2.9.0), cracklib-runtime,' sed -i -e "s/\\\${MAYBE_LIBCRACK}/${MAYBE_LIBCRACK}/g" debian/control fi From 2855edf9ee0344e596ff4b9ee70a23421ae60405 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 23 Nov 2019 23:26:35 +0700 Subject: [PATCH 20/32] try to fix data races in DBUG init_settings.keywords and it's pointee are shared data. Protect them with mutex too. --- dbug/dbug.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index 78605983ccb..21f86ded0a5 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -320,6 +320,10 @@ static void DbugVfprintf(FILE *stream, const char* format, va_list args); */ #include +/* +** Protects writing to all file descriptors, init_settings.keywords +** pointer and it's pointee - a linked list with keywords. +*/ static pthread_mutex_t THR_LOCK_dbug; static void LockMutex(CODE_STATE *cs) @@ -332,6 +336,16 @@ static void UnlockMutex(CODE_STATE *cs) if (!cs->locked) pthread_mutex_unlock(&THR_LOCK_dbug); } +static void LockIfInitSettings(CODE_STATE *cs) +{ + if (cs->stack == &init_settings) + LockMutex(cs); +} +static void UnlockIfInitSettings(CODE_STATE *cs) +{ + if (cs->stack == &init_settings) + UnlockMutex(cs); +} static CODE_STATE *code_state(void) { @@ -476,7 +490,9 @@ static int DbugParse(CODE_STATE *cs, const char *control) stack->sub_level= 0; stack->out_file= sstderr; stack->functions= NULL; + LockIfInitSettings(cs); stack->keywords= NULL; + UnlockIfInitSettings(cs); stack->processes= NULL; } else if (!stack->out_file) @@ -492,7 +508,9 @@ static int DbugParse(CODE_STATE *cs, const char *control) { /* never share with the global parent - it can change under your feet */ stack->functions= ListCopy(init_settings.functions); + LockIfInitSettings(cs); stack->keywords= ListCopy(init_settings.keywords); + UnlockIfInitSettings(cs); stack->processes= ListCopy(init_settings.processes); } else @@ -516,21 +534,31 @@ static int DbugParse(CODE_STATE *cs, const char *control) case 'd': if (sign < 0 && control == end) { + LockIfInitSettings(cs); if (!is_shared(stack, keywords)) FreeList(stack->keywords); stack->keywords=NULL; + UnlockIfInitSettings(cs); stack->flags &= ~DEBUG_ON; break; } + LockIfInitSettings(cs); if (rel && is_shared(stack, keywords)) stack->keywords= ListCopy(stack->keywords); + UnlockIfInitSettings(cs); if (sign < 0) { if (DEBUGGING) + { + LockIfInitSettings(cs); stack->keywords= ListDel(stack->keywords, control, end); + UnlockIfInitSettings(cs); + } break; } + LockIfInitSettings(cs); stack->keywords= ListAdd(stack->keywords, control, end); + UnlockIfInitSettings(cs); stack->flags |= DEBUG_ON; break; case 'D': @@ -997,7 +1025,9 @@ int _db_explain_ (CODE_STATE *cs, char *buf, size_t len) get_code_state_if_not_set_or_return *buf=0; + LockIfInitSettings(cs); op_list_to_buf('d', cs->stack->keywords, DEBUGGING); + UnlockIfInitSettings(cs); op_int_to_buf ('D', cs->stack->delay, 0); op_list_to_buf('f', cs->stack->functions, cs->stack->functions); op_bool_to_buf('F', cs->stack->flags & FILE_ON); @@ -1578,8 +1608,10 @@ static void PushState(CODE_STATE *cs) static void FreeState(CODE_STATE *cs, int free_state) { struct settings *state= cs->stack; + LockIfInitSettings(cs); if (!is_shared(state, keywords)) FreeList(state->keywords); + UnlockIfInitSettings(cs); if (!is_shared(state, functions)) FreeList(state->functions); if (!is_shared(state, processes)) @@ -1702,10 +1734,16 @@ FILE *_db_fp_(void) BOOLEAN _db_keyword_(CODE_STATE *cs, const char *keyword, int strict) { int match= strict ? INCLUDE : INCLUDE|MATCHED; + BOOLEAN result = FALSE; get_code_state_if_not_set_or_return FALSE; - return (DEBUGGING && DoTrace(cs) & DO_TRACE && - InList(cs->stack->keywords, keyword, strict) & match); + if (!DEBUGGING || !(DoTrace(cs) & DO_TRACE)) + return FALSE; + + LockIfInitSettings(cs); + result= (InList(cs->stack->keywords, keyword, strict) & match) ? TRUE : FALSE; + UnlockIfInitSettings(cs); + return result; } /* From 866e5c250e27e32cd295d84988ffdf7ae64503b2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 11 Feb 2018 14:42:11 +1100 Subject: [PATCH 21/32] MDEV-15503: mtr fix --strace $glob_mysql_test_dir was the wrong directory for strace output as it was for in-tree builds only so failed for: * out of tree builds * --parallel; and * --mem strace output wasn't saved. strace-option never replaced existing arguments (so ammended documentation). strace-client didn't accept an argument as described. Replaced specification of client with this with 'stracer' to be consistent with --debugger option. For consistency with debugger options, --client-strace was added to execute the strace on the mysqltest. Example: Running one test $ ./mtr --strace --client-strace funcs_1.is_table_constraints Logging: ./mtr --strace --client-strace funcs_1.is_table_constraints vardir: /home/anel/mariadb/5.5/mysql-test/var Checking leftover processes... Removing old var directory... - WARNING: Using the 'mysql-test/var' symlink Creating var directory '/home/anel/mariadb/5.5/mysql-test/var'... Checking supported features... MariaDB Version 5.5.67-MariaDB-debug Installing system database... - SSL connections supported - binaries are debug compiled Collecting tests... ============================================================================== TEST RESULT TIME (ms) or COMMENT -------------------------------------------------------------------------- worker[1] Using MTR_BUILD_THREAD 300, with reserved ports 16000..16019 funcs_1.is_table_constraints [ pass ] 1270 -------------------------------------------------------------------------- The servers were restarted 0 times Spent 1.270 of 3 seconds executing testcases Completed: All 1 tests were successful $ find -L . -name \*strace -ls 653 56 -rw-r--r-- 1 anel anel 57147 Nov 29 15:08 ./var/log/mysqltest.strace 646 1768 -rw-r--r-- 1 anel anel 1809855 Nov 29 15:08 ./var/log/mysqld.1.strace Example: Running test in parallel $ mysql-test/mtr --strace --client-strace --mem --parallel=3 main.select Logging: /home/dan/software_projects/mariadb-server/mysql-test/mysql-test-run.pl --strace --client-strace --mem --parallel=3 main.select vardir: /home/dan/software_projects/build-mariadb-10.3/mysql-test/var Checking leftover processes... Removing old var directory... Creating var directory '/home/dan/software_projects/build-mariadb-10.3/mysql-test/var'... - symlinking 'var' to '/dev/shm/var_auto_0v2E' Checking supported features... MariaDB Version 5.5.67-MariaDB - SSL connections supported Collecting tests... Installing system database... ============================================================================== TEST WORKER RESULT TIME (ms) or COMMENT -------------------------------------------------------------------------- worker[1] Using MTR_BUILD_THREAD 300, with reserved ports 16000..16019 worker[3] - 'localhost:16040' was not free worker[2] Using MTR_BUILD_THREAD 301, with reserved ports 16020..16039 worker[3] Using MTR_BUILD_THREAD 303, with reserved ports 16060..16079 main.select w1 [ pass ] 7310 -------------------------------------------------------------------------- The servers were restarted 0 times Spent 7.310 of 11 seconds executing testcases Completed: All 1 tests were successful. $ find mysql-test/var/ -name \*strace -ls 5213766 1212 -rw-r--r-- 1 dan dan 1237817 May 20 16:47 mysql-test/var/1/log/mysqltest.strace 5214733 13016 -rw-r--r-- 1 dan dan 13328335 May 20 16:47 mysql-test/var/1/log/mysqld.1.strace $ mysql-test/mtr --strace --client-strace --strace-option='-e' --strace-option='trace=openat' --mem --parallel=3 main.select ... $ find mysql-test/var/ -name \*strace -ls 5220790 8 -rw-r--r-- 1 dan dan 6291 May 20 17:02 mysql-test/var/3/log/mysqltest.strace 5224140 308 -rw-r--r-- 1 dan dan 314356 May 20 17:02 mysql-test/var/3/log/mysqld.1.strace $ more mysql-test/var/3/mysqltest.strace 1692 openat(AT_FDCWD, "/home/dan/software_projects/mariadb-server/libmysql/.libs/tls/x86_64/x86_64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) 1692 openat(AT_FDCWD, "/home/dan/software_projects/mariadb-server/libmysql/.libs/tls/x86_64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOE NT (No such file or directory) Closes #600 --- mysql-test/mysql-test-run.pl | 48 ++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index e3e9460f35a..8b9d91935bf 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -311,7 +311,8 @@ my $opt_valgrind_mysqltest= 0; my @default_valgrind_args= ("--show-reachable=yes"); my @valgrind_args; my $opt_strace= 0; -my $opt_strace_client; +my $opt_stracer; +my $opt_client_strace = 0; my @strace_args; my $opt_valgrind_path; my $valgrind_reports= 0; @@ -1156,9 +1157,10 @@ sub command_line_setup { 'debugger=s' => \$opt_debugger, 'boot-dbx' => \$opt_boot_dbx, 'client-debugger=s' => \$opt_client_debugger, - 'strace' => \$opt_strace, - 'strace-client' => \$opt_strace_client, - 'strace-option=s' => \@strace_args, + 'strace' => \$opt_strace, + 'strace-option=s' => \@strace_args, + 'client-strace' => \$opt_client_strace, + 'stracer=s' => \$opt_stracer, 'max-save-core=i' => \$opt_max_save_core, 'max-save-datadir=i' => \$opt_max_save_datadir, 'max-test-fail=i' => \$opt_max_test_fail, @@ -1750,7 +1752,7 @@ sub command_line_setup { join(" ", @valgrind_args), "\""); } - if (@strace_args) + if (@strace_args || $opt_stracer) { $opt_strace=1; } @@ -5840,14 +5842,6 @@ sub start_mysqltest ($) { mtr_add_arg($args, "--non-blocking-api"); } - if ( $opt_strace_client ) - { - $exe= $opt_strace_client || "strace"; - mtr_add_arg($args, "-o"); - mtr_add_arg($args, "%s/log/mysqltest.strace", $opt_vardir); - mtr_add_arg($args, "$exe_mysqltest"); - } - mtr_add_arg($args, "--timer-file=%s/log/timer", $opt_vardir); if ( $opt_compress ) @@ -5914,6 +5908,17 @@ sub start_mysqltest ($) { mtr_add_arg($args, "%s", $_) for @args_saved; } + # ---------------------------------------------------------------------- + # Prefix the strace options to the argument list. + # ---------------------------------------------------------------------- + if ( $opt_client_strace ) + { + my @args_saved = @$args; + mtr_init_args(\$args); + strace_arguments($args, \$exe, "mysqltest"); + mtr_add_arg($args, "%s", $_) for @args_saved; + } + if ($opt_force > 1) { mtr_add_arg($args, "--continue-on-error"); @@ -6252,16 +6257,17 @@ sub strace_arguments { my $args= shift; my $exe= shift; my $mysqld_name= shift; + my $output= sprintf("%s/log/%s.strace", $path_vardir_trace, $mysqld_name); mtr_add_arg($args, "-f"); - mtr_add_arg($args, "-o%s/var/log/%s.strace", $glob_mysql_test_dir, $mysqld_name); + mtr_add_arg($args, "-o%s", $output); - # Add strace options, can be overriden by user + # Add strace options mtr_add_arg($args, '%s', $_) for (@strace_args); mtr_add_arg($args, $$exe); - $$exe= "strace"; + $$exe= $opt_stracer || "strace"; if ($exe_libtool) { @@ -6520,11 +6526,11 @@ Options for valgrind Options for strace strace Run the "mysqld" executables using strace. Default - options are -f -o var/log/'mysqld-name'.strace - strace-option=ARGS Option to give strace, replaces default option(s), - strace-client=[path] Create strace output for mysqltest client, optionally - specifying name and path to the trace program to use. - Example: $0 --strace-client=ktrace + options are -f -o 'vardir'/log/'mysqld-name'.strace. + client-strace Trace the "mysqltest". + strace-option=ARGS Option to give strace, appends to existing options. + stracer= Specify name and path to the trace program to use. + Example: $0 --strace-client=ktrace. Misc options user=USER User for connecting to mysqld(default: $opt_user) From 3cb60ec2c305f0b0ab9c3680eb787f8dcd78e57b Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 29 Nov 2019 15:50:40 +0100 Subject: [PATCH 22/32] Update `stracer` description in `mtr`. `strace-client` is not used --- mysql-test/mysql-test-run.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 8b9d91935bf..6d32e97d6b4 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -6530,7 +6530,7 @@ Options for strace client-strace Trace the "mysqltest". strace-option=ARGS Option to give strace, appends to existing options. stracer= Specify name and path to the trace program to use. - Example: $0 --strace-client=ktrace. + Default is "strace". Example: $0 --stracer=ktrace. Misc options user=USER User for connecting to mysqld(default: $opt_user) From 6fe2aae3ce8139864dfe45f2f9834f82ef54ff1e Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 30 Nov 2019 12:14:00 +0700 Subject: [PATCH 23/32] InnoDB: log unsuccessful calls to pthread_attr_init() and pthread_create() before crash --- storage/innobase/os/os0thread.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index 792d9cc4e10..ed3f1a13f42 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -143,7 +143,13 @@ os_thread_create_func( pthread_attr_t attr; #ifndef UNIV_HPUX10 - pthread_attr_init(&attr); + ret = pthread_attr_init(&attr); + if (UNIV_UNLIKELY(ret)) { + fprintf(stderr, + "InnoDB: Error: pthread_attr_init() returned %d\n", + ret); + exit(1); + } #endif #ifdef UNIV_AIX @@ -171,7 +177,11 @@ os_thread_create_func( #else ret = pthread_create(&pthread, &attr, func, arg); #endif - ut_a(ret == 0); + if (UNIV_UNLIKELY(ret)) { + fprintf(stderr, + "InnoDB: Error: pthread_create() returned %d\n", ret); + exit(1); + } #ifndef UNIV_HPUX10 pthread_attr_destroy(&attr); From 33cf4da1838f3f13d8e973e7f46cc486f1183338 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sat, 30 Nov 2019 18:19:20 +0700 Subject: [PATCH 24/32] cleanup: replace exit(1) with abort() --- storage/innobase/os/os0thread.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index ed3f1a13f42..1182166f522 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -148,7 +148,7 @@ os_thread_create_func( fprintf(stderr, "InnoDB: Error: pthread_attr_init() returned %d\n", ret); - exit(1); + abort(); } #endif @@ -165,7 +165,7 @@ os_thread_create_func( fprintf(stderr, "InnoDB: Error: pthread_attr_setstacksize" " returned %d\n", ret); - exit(1); + abort(); } #endif os_mutex_enter(os_sync_mutex); @@ -180,7 +180,7 @@ os_thread_create_func( if (UNIV_UNLIKELY(ret)) { fprintf(stderr, "InnoDB: Error: pthread_create() returned %d\n", ret); - exit(1); + abort(); } #ifndef UNIV_HPUX10 From d930422e9e84aa1bc265963225dd29295203e9d3 Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Fri, 29 Nov 2019 13:21:23 +0100 Subject: [PATCH 25/32] Fix the line break warning (groff/lintian). The lintian Debian tool is complaining about: 'W: mariadb-test: manpage-has-errors-from-man usr/share/man/man1/mysql-test-run.pl.1.gz 246: warning [p 2, 6.0i, div '3tbd1,1', 0.3i]: can't break line' See: https://salsa.debian.org/faust-guest/mariadb-10.3/-/jobs/431900 The following command permits to catch the problematic lines: $ groff -man -Tascii ./mysql-test-run.pl.1 | less Closes #1419 --- man/mysql-test-run.pl.1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/man/mysql-test-run.pl.1 b/man/mysql-test-run.pl.1 index 8b8eafd7bdd..abf1d1bc240 100644 --- a/man/mysql-test-run.pl.1 +++ b/man/mysql-test-run.pl.1 @@ -1937,8 +1937,10 @@ Run stress test, providing options to mysql\-stress\-test\&.pl\&. Options are se .\" suite option: mysql-test-run.pl \fB\-\-suite[s]=\fR\fB\fIsuite_name...\fR\fR .sp -Comma separated list of suite names to run. The default is: "main-,archive-,binlog-,csv-,federated-,funcs_1-,funcs_2-,handler-,heap-,innodb-,innodb_fts-, -innodb_zip-,maria-,multi_source-,optimizer_unfixed_bugs-,parts-,percona-,perfschema-, +Comma separated list of suite names to run. The default is: +"main-,archive-,binlog-,csv-,federated-,funcs_1-,funcs_2-, +handler-,heap-,innodb-,innodb_fts-,innodb_zip-,maria-, +multi_source-,optimizer_unfixed_bugs-,parts-,perfschema-, plugins-,roles-,rpl-,sys_vars-,unit-,vcol-"\&. .RE .sp From e3d3bbf59829e93e83ab05d831e92a742ffd7e32 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 28 Nov 2019 15:08:29 +0100 Subject: [PATCH 26/32] Using `variables` instead of `values` in mysqld --help documentation would be more accurate --- mysql-test/r/mysqld--help.result | 2 +- scripts/mysql_install_db.sh | 2 +- sql/mysqld.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 825c7e786a3..bececa14d7a 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -1107,5 +1107,5 @@ userstat FALSE verbose TRUE wait-timeout 28800 -To see what values a running MySQL server is using, type +To see what variables a running MySQL server is using, type 'mysqladmin variables' instead of 'mysqld --verbose --help'. diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index fb1663d9d69..fe7375494fe 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -214,7 +214,7 @@ cannot_find_file() echo "If you compiled from source, you need to either run 'make install' to" echo "copy the software into the correct location ready for operation." echo "If you don't want to do a full install, you can use the --srcdir" - echo "option to only install the mysql database and privilege tables" + echo "option to only install the mysql database and privilege tables." echo echo "If you are using a binary release, you must either be at the top" echo "level of the extracted archive, or pass the --basedir option" diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 96daeab867d..2aedec595c0 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7240,7 +7240,7 @@ because execution stopped before plugins were initialized."); } puts("\n\ -To see what values a running MySQL server is using, type\n\ +To see what variables a running MySQL server is using, type\n\ 'mysqladmin variables' instead of 'mysqld --verbose --help'."); } DBUG_VOID_RETURN; From 3ff8fb5141850aebcf37578c108bfbea99a5d53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Dec 2019 07:34:46 +0200 Subject: [PATCH 27/32] InnoDB: Remove unused get_wkb_of_default_point() The function get_wkb_of_default_point() should never have been added, and the cleanup in commit 56ff6f1b0b248befb80675ba36a3251c7844f410 should have removed it. The unnecessary code was added in mysql/mysql-server@0a27b72171c4449f506ad4e93df8c29fead1ed72 and mostly disabled in mysql/mysql-server@8f11af7734509821b654f2c330dff5374c89073c. In MariaDB, the types DATA_POINT and DATA_VAR_POINT are never used. Instead, DATA_GEOMETRY continues to be used since 10.2.2, introduced by mysql/mysql-server@673bad7c7e17701b69e29d0fbc4e2d68c1a7b613 in MySQL 5.7.1. --- storage/innobase/gis/gis0geo.cc | 32 ------------------------------ storage/innobase/include/gis0geo.h | 13 ------------ 2 files changed, 45 deletions(-) diff --git a/storage/innobase/gis/gis0geo.cc b/storage/innobase/gis/gis0geo.cc index cad3877d3e9..fb55945549e 100644 --- a/storage/innobase/gis/gis0geo.cc +++ b/storage/innobase/gis/gis0geo.cc @@ -801,35 +801,3 @@ rtree_area_overlapping( return(area); } - -/** Get the wkb of default POINT value, which represents POINT(0 0) -if it's of dimension 2, etc. -@param[in] n_dims dimensions -@param[out] wkb wkb buffer for default POINT -@param[in] len length of wkb buffer -@return non-0 indicate the length of wkb of the default POINT, -0 if the buffer is too small */ -uint -get_wkb_of_default_point( - uint n_dims, - uchar* wkb, - uint len) -{ - // JAN: TODO: MYSQL 5.7 GIS - #define GEOM_HEADER_SIZE 16 - if (len < GEOM_HEADER_SIZE + sizeof(double) * n_dims) { - return(0); - } - - /** POINT wkb comprises SRID, wkb header(byte order and type) - and coordinates of the POINT */ - len = GEOM_HEADER_SIZE + sizeof(double) * n_dims; - /** We always use 0 as default coordinate */ - memset(wkb, 0, len); - /** We don't need to write SRID, write 0x01 for Byte Order */ - mach_write_to_n_little_endian(wkb + SRID_SIZE, 1, 0x01); - /** Write wkbType::wkbPoint for the POINT type */ - mach_write_to_n_little_endian(wkb + SRID_SIZE + 1, 4, wkbPoint); - - return(len); -} diff --git a/storage/innobase/include/gis0geo.h b/storage/innobase/include/gis0geo.h index 3b71815d0fe..dea6d63f4e0 100644 --- a/storage/innobase/include/gis0geo.h +++ b/storage/innobase/include/gis0geo.h @@ -54,19 +54,6 @@ enum wkbByteOrder wkbNDR = 1 /* Little Endian */ }; -/** Get the wkb of default POINT value, which represents POINT(0 0) -if it's of dimension 2, etc. -@param[in] n_dims dimensions -@param[out] wkb wkb buffer for default POINT -@param[in] len length of wkb buffer -@return non-0 indicate the length of wkb of the default POINT, -0 if the buffer is too small */ -uint -get_wkb_of_default_point( - uint n_dims, - uchar* wkb, - uint len); - /*************************************************************//** Calculate minimal bounding rectangle (mbr) of the spatial object stored in "well-known binary representation" (wkb) format. From 88073dae79c240eb82e923364aeaed907b25f096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 3 Dec 2019 08:04:46 +0200 Subject: [PATCH 28/32] MDEV-21198 : Galera test failure on galera_var_notify_cmd Add proper wsrep sync wait. --- .../suite/galera/r/galera_var_notify_cmd.result | 17 +++++++++-------- .../suite/galera/t/galera_var_notify_cmd.test | 8 ++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_notify_cmd.result b/mysql-test/suite/galera/r/galera_var_notify_cmd.result index b02124cda70..2b390d0db8d 100644 --- a/mysql-test/suite/galera/r/galera_var_notify_cmd.result +++ b/mysql-test/suite/galera/r/galera_var_notify_cmd.result @@ -1,10 +1,11 @@ connection node_1; -SELECT COUNT(DISTINCT uuid) = 2 FROM mtr_wsrep_notify.membership; -COUNT(DISTINCT uuid) = 2 +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(DISTINCT uuid) FROM mtr_wsrep_notify.membership; +COUNT(DISTINCT uuid) +2 +SELECT MAX(size) FROM mtr_wsrep_notify.status; +MAX(size) +2 +SELECT COUNT(DISTINCT idx) FROM mtr_wsrep_notify.status; +COUNT(DISTINCT idx) 1 -SELECT MAX(size) = 2 FROM mtr_wsrep_notify.status; -MAX(size) = 2 -1 -SELECT COUNT(DISTINCT idx) = 2 FROM mtr_wsrep_notify.status; -COUNT(DISTINCT idx) = 2 -0 diff --git a/mysql-test/suite/galera/t/galera_var_notify_cmd.test b/mysql-test/suite/galera/t/galera_var_notify_cmd.test index b261f878ced..2b0852b4870 100644 --- a/mysql-test/suite/galera/t/galera_var_notify_cmd.test +++ b/mysql-test/suite/galera/t/galera_var_notify_cmd.test @@ -3,10 +3,10 @@ # notifications into a table. # ---source include/have_innodb.inc --source include/galera_cluster.inc --connection node_1 -SELECT COUNT(DISTINCT uuid) = 2 FROM mtr_wsrep_notify.membership; -SELECT MAX(size) = 2 FROM mtr_wsrep_notify.status; -SELECT COUNT(DISTINCT idx) = 2 FROM mtr_wsrep_notify.status; +SET SESSION wsrep_sync_wait=15; +SELECT COUNT(DISTINCT uuid) FROM mtr_wsrep_notify.membership; +SELECT MAX(size) FROM mtr_wsrep_notify.status; +SELECT COUNT(DISTINCT idx) FROM mtr_wsrep_notify.status; From 543f22a2d0633e645fb49a360c55528d8cb008fb Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 27 Nov 2019 11:58:20 +0700 Subject: [PATCH 29/32] check parameters of io_submit() and LinuxAIOHandler::reserve_slot() Ensure that pointer to a buffer which is used for both reads and writes is properly aligned. I've intentially put permanent check right before io_submit() call instead of the time of setting iocb date. I think that'll make check better. LinuxAIOHandler::resever_slot(): check arguments alignment in debug builds --- storage/innobase/os/os0file.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 210ee85798c..b32a2fa77ef 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -1792,6 +1792,9 @@ LinuxAIOHandler::resubmit(Slot* slot) iocb->data = slot; + ut_a(reinterpret_cast(iocb->u.c.buf) % OS_FILE_LOG_BLOCK_SIZE + == 0); + /* Resubmit an I/O request */ int ret = io_submit(m_array->io_ctx(m_segment), 1, &iocb); ut_a(ret != -EINVAL); @@ -2160,6 +2163,9 @@ AIO::linux_dispatch(Slot* slot) io_ctx_index = (slot->pos * m_n_segments) / m_slots.size(); + ut_a(reinterpret_cast(iocb->u.c.buf) % OS_FILE_LOG_BLOCK_SIZE + == 0); + int ret = io_submit(io_ctx(io_ctx_index), 1, &iocb); ut_a(ret != -EINVAL); @@ -2352,6 +2358,8 @@ AIO::is_linux_native_aio_supported() io_prep_pread(p_iocb, fd, ptr, 512, 0); } + ut_a(reinterpret_cast(p_iocb->u.c.buf) % OS_FILE_LOG_BLOCK_SIZE + == 0); int err = io_submit(io_ctx, 1, &p_iocb); ut_a(err != -EINVAL); @@ -6271,6 +6279,10 @@ AIO::reserve_slot( os_offset_t offset, ulint len) { + ut_ad(reinterpret_cast(buf) % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_ad(offset % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_ad(len % OS_FILE_LOG_BLOCK_SIZE == 0); + #ifdef WIN_ASYNC_IO ut_a((len & 0xFFFFFFFFUL) == len); #endif /* WIN_ASYNC_IO */ From ccc5e57730ae480d739adb44fd9a51773ae755ad Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 3 Dec 2019 15:44:41 +0100 Subject: [PATCH 30/32] vew CC v 3.1 --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index f035fc5f7fd..c8833751cf4 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit f035fc5f7fda5ac01d557073ef219c7e8d65fe10 +Subproject commit c8833751cf48d0085d9d7a4285aafdc967a63a4d From f7d35ffc7691bbbf65101fb1a496e441a2b00a83 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 3 Dec 2019 20:39:16 +0100 Subject: [PATCH 31/32] Galera test fix after merge. --- .../suite/galera/r/galera_as_slave_gtid_myisam.result | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result index d9a87571655..0e5c6425d6a 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result @@ -1,21 +1,29 @@ +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2; ALTER TABLE mysql.gtid_slave_pos engine = InnoDB; START SLAVE; +connection node_1; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=MyISAM; INSERT INTO t1 VALUES(1); SELECT LENGTH(@@global.gtid_binlog_state) > 1; LENGTH(@@global.gtid_binlog_state) > 1 1 +connection node_2; gtid_binlog_state_equal 0 +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; SELECT COUNT(*) = 0 FROM t1; COUNT(*) = 0 1 gtid_binlog_state_equal 0 #cleanup +connection node_1; DROP TABLE t1; reset master; +connection node_2; STOP SLAVE; RESET SLAVE ALL; reset master; +connection node_3; reset master; From c9b9eb331570704d020fcc7c7894f19febe7f26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 4 Dec 2019 11:46:37 +0200 Subject: [PATCH 32/32] MDEV-18497 : CTAS async replication from mariadb master crashes galera nodes (#1410) In MariaDB 10.2 master could have been configured so that there is extra annotate events. When we peak next event type for CTAS we need to skip annotate events. --- .../galera/r/galera_as_slave_ctas.result | 14 +++++++++++ sql/slave.cc | 24 ++++++++++++------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_as_slave_ctas.result b/mysql-test/suite/galera/r/galera_as_slave_ctas.result index 79114824008..13cbbc905e3 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_ctas.result +++ b/mysql-test/suite/galera/r/galera_as_slave_ctas.result @@ -1,14 +1,28 @@ +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_2; START SLAVE; +connection node_1; SHOW VARIABLES LIKE 'binlog_format'; Variable_name Value binlog_format ROW +connection node_1; CREATE TABLE source (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE target AS SELECT * FROM source; +connection node_2; +connection node_3; +connection node_1; DROP TABLE target; INSERT INTO source VALUES(1); CREATE TABLE target AS SELECT * FROM source; +connection node_2; +connection node_3; +connection node_1; DROP TABLE source; DROP TABLE target; +connection node_3; +connection node_2; STOP SLAVE; RESET SLAVE ALL; +connection node_1; RESET MASTER; diff --git a/sql/slave.cc b/sql/slave.cc index 17abf8d53f3..83edc52c3fd 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7397,24 +7397,30 @@ err: #ifdef WITH_WSREP enum Log_event_type wsrep_peak_event(rpl_group_info *rgi, ulonglong* event_size) { + enum Log_event_type ev_type; + mysql_mutex_lock(&rgi->rli->data_lock); unsigned long long event_pos= rgi->event_relay_log_pos; + unsigned long long orig_future_pos= rgi->future_event_relay_log_pos; unsigned long long future_pos= rgi->future_event_relay_log_pos; - /* scan the log to read next event */ - my_b_seek(rgi->rli->cur_log, future_pos); - rgi->rli->event_relay_log_pos= future_pos; - rgi->event_relay_log_pos= future_pos; - - Log_event* ev = next_event(rgi, event_size); - enum Log_event_type ev_type= (ev) ? ev->get_type_code() : UNKNOWN_EVENT; - delete ev; + /* scan the log to read next event and we skip + annotate events. */ + do { + my_b_seek(rgi->rli->cur_log, future_pos); + rgi->rli->event_relay_log_pos= future_pos; + rgi->event_relay_log_pos= future_pos; + Log_event* ev= next_event(rgi, event_size); + ev_type= (ev) ? ev->get_type_code() : UNKNOWN_EVENT; + delete ev; + future_pos+= *event_size; + } while (ev_type == ANNOTATE_ROWS_EVENT); /* scan the log back and re-set the positions to original values */ rgi->rli->event_relay_log_pos= event_pos; rgi->event_relay_log_pos= event_pos; - my_b_seek(rgi->rli->cur_log, future_pos); + my_b_seek(rgi->rli->cur_log, orig_future_pos); mysql_mutex_unlock(&rgi->rli->data_lock);