From 671d9b6c611be423246b562878da2a5788cc327d Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 16 Jul 2020 16:30:54 +0200 Subject: [PATCH 01/12] - Fix MDEV-22571 and MDEV-22572. Allow multiple ZIP table and enable using special column in them. modified: storage/connect/tabzip.cpp modified: storage/connect/tabzip.h - Fix some compiler errors modified: storage/connect/tabcmg.cpp --- storage/connect/connect.cc | 1 - storage/connect/filamdbf.cpp | 195 +++++++++++++++++------ storage/connect/filamdbf.h | 2 +- storage/connect/filamzip.cpp | 282 ++++++++++++++++++++++++++++++++-- storage/connect/filamzip.h | 38 ++++- storage/connect/ha_connect.cc | 19 +-- storage/connect/mongo.cpp | 1 - storage/connect/mongo.h | 1 - storage/connect/plgxml.cpp | 2 +- storage/connect/tabcmg.cpp | 2 + storage/connect/tabdos.cpp | 36 +++-- storage/connect/tabdos.h | 1 + storage/connect/tabfix.h | 18 ++- storage/connect/tabjson.cpp | 1 + storage/connect/tabjson.h | 1 - storage/connect/tabzip.cpp | 13 +- storage/connect/tabzip.h | 2 + 17 files changed, 509 insertions(+), 106 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index dfc619cf4af..2a0f2ed037f 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -355,7 +355,6 @@ bool CntOpenTable(PGLOBAL g, PTDB tdbp, MODE mode, char *c1, char *c2, } // endif mode rcop = false; - } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); diff --git a/storage/connect/filamdbf.cpp b/storage/connect/filamdbf.cpp index c8bab2b53a4..542159fd172 100644 --- a/storage/connect/filamdbf.cpp +++ b/storage/connect/filamdbf.cpp @@ -49,6 +49,7 @@ #include "global.h" #include "plgdbsem.h" #include "filamdbf.h" +#include "filamzip.h" #include "tabdos.h" #include "valblk.h" #define NO_FUNC @@ -139,7 +140,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (fread(buf, HEADLEN, 1, file) != 1) { strcpy(g->Message, MSG(NO_READ_32)); return RC_NF; - } // endif fread + } // endif fread // Check first byte to be sure of .dbf type if ((buf->Version & 0x03) != DBFTYPE) { @@ -149,7 +150,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if ((buf->Version & 0x30) == 0x30) { strcpy(g->Message, MSG(FOXPRO_FILE)); dbc = 264; // FoxPro database container - } // endif Version + } // endif Version } else strcpy(g->Message, MSG(DBASE_FILE)); @@ -158,12 +159,12 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (fseek(file, buf->Headlen() - dbc, SEEK_SET) != 0) { sprintf(g->Message, MSG(BAD_HEADER), fn); return RC_FX; - } // endif fseek + } // endif fseek if (fread(&endmark, 2, 1, file) != 1) { strcpy(g->Message, MSG(BAD_HEAD_END)); return RC_FX; - } // endif fread + } // endif fread // Some files have just 1D others have 1D00 following fields if (endmark[0] != EOH && endmark[1] != EOH) { @@ -172,7 +173,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (rc == RC_OK) return RC_FX; - } // endif endmark + } // endif endmark // Calculate here the number of fields while we have the dbc info buf->SetFields((buf->Headlen() - dbc - 1) / 32); @@ -180,13 +181,58 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) return rc; } // end of dbfhead +/****************************************************************************/ +/* dbfields: Analyze a DBF header and set the table fields number. */ +/* Parameters: */ +/* PGLOBAL g -- pointer to the CONNECT Global structure */ +/* DBFHEADER *hdrp -- pointer to _dbfheader structure */ +/* Returns: */ +/* RC_OK, RC_INFO, or RC_FX if error. */ +/****************************************************************************/ +static int dbfields(PGLOBAL g, DBFHEADER* hdrp) +{ + char* endmark; + int dbc = 2, rc = RC_OK; + + *g->Message = '\0'; + + // Check first byte to be sure of .dbf type + if ((hdrp->Version & 0x03) != DBFTYPE) { + strcpy(g->Message, MSG(NOT_A_DBF_FILE)); + rc = RC_INFO; + + if ((hdrp->Version & 0x30) == 0x30) { + strcpy(g->Message, MSG(FOXPRO_FILE)); + dbc = 264; // FoxPro database container + } // endif Version + + } else + strcpy(g->Message, MSG(DBASE_FILE)); + + // Check last byte(s) of header + endmark = (char*)hdrp + hdrp->Headlen() - dbc; + + // Some headers just have 1D others have 1D00 following fields + if (endmark[0] != EOH && endmark[1] != EOH) { + sprintf(g->Message, MSG(NO_0DH_HEAD), dbc); + + if (rc == RC_OK) + return RC_FX; + + } // endif endmark + + // Calculate here the number of fields while we have the dbc info + hdrp->SetFields((hdrp->Headlen() - dbc - 1) / 32); + return rc; +} // end of dbfields + /* -------------------------- Function DBFColumns ------------------------- */ /****************************************************************************/ /* DBFColumns: constructs the result blocks containing the description */ /* of all the columns of a DBF file that will be retrieved by #GetData. */ /****************************************************************************/ -PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) +PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS topt, bool info) { int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; @@ -196,10 +242,12 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) char buf[2], filename[_MAX_PATH]; int ncol = sizeof(buftyp) / sizeof(int); int rc, type, len, field, fields; - bool bad; - DBFHEADER mainhead; - DESCRIPTOR thisfield; - FILE *infile = NULL; + bool bad, mul; + PCSZ target, pwd; + DBFHEADER mainhead, *hp; + DESCRIPTOR thisfield, *tfp; + FILE *infile = NULL; + UNZIPUTL *zutp = NULL; PQRYRES qrp; PCOLRES crp; @@ -217,21 +265,55 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) /************************************************************************/ PlugSetPath(filename, fn, dp); - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "rb"))) - return NULL; + if (topt->zipped) { + target = GetStringTableOption(g, topt, "Entry", NULL); + mul = (target && *target) ? strchr(target, '*') || strchr(target, '?') + : false; + mul = GetBooleanTableOption(g, topt, "Mulentries", mul); - /************************************************************************/ - /* Get the first 32 bytes of the header. */ - /************************************************************************/ - if ((rc = dbfhead(g, infile, filename, &mainhead)) == RC_FX) { - fclose(infile); - return NULL; - } // endif dbfhead + if (mul) { + strcpy(g->Message, "Cannot find column definition for multiple entries"); + return NULL; + } // endif Multiple - /************************************************************************/ - /* Allocate the structures used to refer to the result set. */ - /************************************************************************/ - fields = mainhead.Fields(); + pwd = GetStringTableOption(g, topt, "Password", NULL); + zutp = new(g) UNZIPUTL(target, pwd, mul); + + if (!zutp->OpenTable(g, MODE_READ, filename)) + hp = (DBFHEADER*)zutp->memory; + else + return NULL; + + /**********************************************************************/ + /* Set the table fields number. */ + /**********************************************************************/ + if ((rc = dbfields(g, hp)) == RC_FX) { + zutp->close(); + return NULL; + } // endif dbfields + + tfp = (DESCRIPTOR*)hp; + } else { + if (!(infile = global_fopen(g, MSGID_CANNOT_OPEN, filename, "rb"))) + return NULL; + else + hp = &mainhead; + + /**********************************************************************/ + /* Get the first 32 bytes of the header. */ + /**********************************************************************/ + if ((rc = dbfhead(g, infile, filename, hp)) == RC_FX) { + fclose(infile); + return NULL; + } // endif dbfhead + + tfp = &thisfield; + } // endif zipped + + /************************************************************************/ + /* Get the number of the table fields. */ + /************************************************************************/ + fields = hp->Fields(); } else fields = 0; @@ -241,19 +323,21 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) if (info || !qrp) { if (infile) fclose(infile); + else if (zutp) + zutp->close(); return qrp; - } // endif info + } // endif info if (trace(1)) { htrc("Structure of %s\n", filename); htrc("headlen=%hd reclen=%hd degree=%d\n", - mainhead.Headlen(), mainhead.Reclen(), fields); - htrc("flags(iem)=%d,%d,%d cp=%d\n", mainhead.Incompleteflag, - mainhead.Encryptflag, mainhead.Mdxflag, mainhead.Language); + hp->Headlen(), hp->Reclen(), fields); + htrc("flags(iem)=%d,%d,%d cp=%d\n", hp->Incompleteflag, + hp->Encryptflag, hp->Mdxflag, hp->Language); htrc("%hd records, last changed %02d/%02d/%d\n", - mainhead.Records(), mainhead.Filedate[1], mainhead.Filedate[2], - mainhead.Filedate[0] + (mainhead.Filedate[0] <= 30) ? 2000 : 1900); + hp->Records(), hp->Filedate[1], hp->Filedate[2], + hp->Filedate[0] + (hp->Filedate[0] <= 30) ? 2000 : 1900); htrc("Field Type Offset Len Dec Set Mdx\n"); } // endif trace @@ -265,21 +349,24 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) for (field = 0; field < fields; field++) { bad = FALSE; - if (fread(&thisfield, HEADLEN, 1, infile) != 1) { + if (topt->zipped) { + tfp = (DESCRIPTOR*)((char*)tfp + HEADLEN); + } else if (fread(tfp, HEADLEN, 1, infile) != 1) { sprintf(g->Message, MSG(ERR_READING_REC), field+1, fn); goto err; - } else - len = thisfield.Length; + } // endif fread + + len = tfp->Length; if (trace(1)) htrc("%-11s %c %6ld %3d %2d %3d %3d\n", - thisfield.Name, thisfield.Type, thisfield.Offset, len, - thisfield.Decimals, thisfield.Setfield, thisfield.Mdxfield); + tfp->Name, tfp->Type, tfp->Offset, len, + tfp->Decimals, tfp->Setfield, tfp->Mdxfield); /************************************************************************/ /* Now get the results into blocks. */ /************************************************************************/ - switch (thisfield.Type) { + switch (tfp->Type) { case 'C': // Characters case 'L': // Logical 'T' or 'F' or space type = TYPE_STRING; @@ -294,7 +381,7 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) // type = TYPE_INT; // break; case 'N': - type = (thisfield.Decimals) ? TYPE_DOUBLE + type = (tfp->Decimals) ? TYPE_DOUBLE : (len > 10) ? TYPE_BIGINT : TYPE_INT; break; case 'F': // Float @@ -306,8 +393,8 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) break; default: if (!info) { - sprintf(g->Message, MSG(BAD_DBF_TYPE), thisfield.Type - , thisfield.Name); + sprintf(g->Message, MSG(BAD_DBF_TYPE), tfp->Type + , tfp->Name); goto err; } // endif info @@ -316,27 +403,31 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) } // endswitch Type crp = qrp->Colresp; // Column Name - crp->Kdata->SetValue(thisfield.Name, field); + crp->Kdata->SetValue(tfp->Name, field); crp = crp->Next; // Data Type crp->Kdata->SetValue((int)type, field); crp = crp->Next; // Type Name if (bad) { - buf[0] = thisfield.Type; + buf[0] = tfp->Type; crp->Kdata->SetValue(buf, field); } else crp->Kdata->SetValue(GetTypeName(type), field); crp = crp->Next; // Precision - crp->Kdata->SetValue((int)thisfield.Length, field); + crp->Kdata->SetValue((int)tfp->Length, field); crp = crp->Next; // Length - crp->Kdata->SetValue((int)thisfield.Length, field); + crp->Kdata->SetValue((int)tfp->Length, field); crp = crp->Next; // Scale (precision) - crp->Kdata->SetValue((int)thisfield.Decimals, field); + crp->Kdata->SetValue((int)tfp->Decimals, field); } // endfor field qrp->Nblin = field; - fclose(infile); + + if (infile) + fclose(infile); + else if (zutp) + zutp->close(); #if 0 if (info) { @@ -347,9 +438,9 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) sprintf(buf, "Ver=%02x ncol=%hu nlin=%u lrecl=%hu headlen=%hu date=%02d/%02d/%02d", - mainhead.Version, fields, mainhead.Records, mainhead.Reclen, - mainhead.Headlen, mainhead.Filedate[0], mainhead.Filedate[1], - mainhead.Filedate[2]); + hp->Version, fields, hp->Records, hp->Reclen, + hp->Headlen, hp->Filedate[0], hp->Filedate[1], + hp->Filedate[2]); strcat(g->Message, buf); } // endif info @@ -360,9 +451,13 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) /**************************************************************************/ return qrp; - err: - fclose(infile); - return NULL; +err: + if (infile) + fclose(infile); + else if (zutp) + zutp->close(); + + return NULL; } // end of DBFColumns /* ---------------------------- Class DBFBASE ----------------------------- */ diff --git a/storage/connect/filamdbf.h b/storage/connect/filamdbf.h index 640fc349b4c..dfe5cb5cfc4 100644 --- a/storage/connect/filamdbf.h +++ b/storage/connect/filamdbf.h @@ -19,7 +19,7 @@ typedef class DBMFAM *PDBMFAM; /****************************************************************************/ /* Functions used externally. */ /****************************************************************************/ -PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info); +PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS tiop, bool info); /****************************************************************************/ /* This is the base class for dBASE file access methods. */ diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index e76dc496246..eeb23e1f053 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,11 +1,11 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.3 */ +/* Version 1.4 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2016-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2016-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -45,6 +45,62 @@ #define WRITEBUFFERSIZE (16384) +/****************************************************************************/ +/* Definitions used for DBF tables. */ +/****************************************************************************/ +#define HEADLEN 32 /* sizeof ( mainhead or thisfield ) */ +//efine MEMOLEN 10 /* length of memo field in .dbf */ +#define DBFTYPE 3 /* value of bits 0 and 1 if .dbf */ +#define EOH 0x0D /* end-of-header marker in .dbf file */ + +/****************************************************************************/ +/* First 32 bytes of a DBF table. */ +/* Note: some reserved fields are used here to store info (Fields) */ +/****************************************************************************/ +typedef struct _dbfheader { + uchar Version; /* Version information flags */ + char Filedate[3]; /* date, YYMMDD, binary. YY=year-1900 */ +private: + /* The following four members are stored in little-endian format on disk */ + char m_RecordsBuf[4]; /* records in the file */ + char m_HeadlenBuf[2]; /* bytes in the header */ + char m_ReclenBuf[2]; /* bytes in a record */ + char m_FieldsBuf[2]; /* Reserved but used to store fields */ +public: + char Incompleteflag; /* 01 if incomplete, else 00 */ + char Encryptflag; /* 01 if encrypted, else 00 */ + char Reserved2[12]; /* for LAN use */ + char Mdxflag; /* 01 if production .mdx, else 00 */ + char Language; /* Codepage */ + char Reserved3[2]; + + uint Records(void) const { return uint4korr(m_RecordsBuf); } + ushort Headlen(void) const { return uint2korr(m_HeadlenBuf); } + ushort Reclen(void) const { return uint2korr(m_ReclenBuf); } + ushort Fields(void) const { return uint2korr(m_FieldsBuf); } + + void SetHeadlen(ushort num) { int2store(m_HeadlenBuf, num); } + void SetReclen(ushort num) { int2store(m_ReclenBuf, num); } + void SetFields(ushort num) { int2store(m_FieldsBuf, num); } +} DBFHEADER; + +/****************************************************************************/ +/* Column field descriptor of a .dbf file. */ +/****************************************************************************/ +typedef struct _descriptor { + char Name[11]; /* field name, in capitals, null filled*/ + char Type; /* field type, C, D, F, L, M or N */ + uint Offset; /* used in memvars, not in files. */ + uchar Length; /* field length */ + uchar Decimals; /* number of decimal places */ + short Reserved4; + char Workarea; /* ??? */ + char Reserved5[2]; + char Setfield; /* ??? */ + char Reserved6[7]; + char Mdxfield; /* 01 if tag field in production .mdx */ +} DESCRIPTOR; + bool ZipLoadFile(PGLOBAL g, PCSZ zfn, PCSZ fn, PCSZ entry, bool append, bool mul); /***********************************************************************/ @@ -214,10 +270,21 @@ bool ZipLoadFile(PGLOBAL g, PCSZ zfn, PCSZ fn, PCSZ entry, bool append, bool mul buf = (char*)PlugSubAlloc(g, NULL, WRITEBUFFERSIZE); - if (mul) - err = ZipFiles(g, zutp, fn, buf); - else - err = ZipFile(g, zutp, fn, entry, buf); + if (!mul) { + PCSZ entp; + + if (!entry) { // entry defaults to the file name + char* p = strrchr((char*)fn, '/'); +#if defined(__WIN__) + if (!p) p = strrchr((char*)fn, '\\'); +#endif // __WIN__ + entp = (p) ? p + 1 : entry; + } else + entp = entry; + + err = ZipFile(g, zutp, fn, entp, buf); + } else + err = ZipFiles(g, zutp, fn, buf); zutp->close(); return err; @@ -232,6 +299,7 @@ ZIPUTIL::ZIPUTIL(PCSZ tgt) { zipfile = NULL; target = tgt; + pwd = NULL; fp = NULL; entryopen = false; } // end of ZIPUTIL standard constructor @@ -241,6 +309,7 @@ ZIPUTIL::ZIPUTIL(ZIPUTIL *zutp) { zipfile = zutp->zipfile; target = zutp->target; + pwd = zutp->pwd; fp = zutp->fp; entryopen = zutp->entryopen; } // end of UNZIPUTL copy constructor @@ -385,11 +454,11 @@ void ZIPUTIL::closeEntry() /***********************************************************************/ /* Constructors. */ /***********************************************************************/ -UNZIPUTL::UNZIPUTL(PCSZ tgt, bool mul) +UNZIPUTL::UNZIPUTL(PCSZ tgt, PCSZ pw, bool mul) { zipfile = NULL; target = tgt; - pwd = NULL; + pwd = pw; fp = NULL; memory = NULL; size = 0; @@ -959,7 +1028,7 @@ int UZXFAM::Cardinality(PGLOBAL g) } // end of Cardinality /***********************************************************************/ -/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/* OpenTableFile: Open a FIX/UNIX table file from a ZIP file. */ /***********************************************************************/ bool UZXFAM::OpenTableFile(PGLOBAL g) { @@ -1015,6 +1084,197 @@ int UZXFAM::GetNext(PGLOBAL g) return RC_OK; } // end of GetNext +/* -------------------------- class UZDFAM --------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +UZDFAM::UZDFAM(PDOSDEF tdp) : DBMFAM(tdp) +{ + zutp = NULL; + tdfp = tdp; + //target = tdp->GetEntry(); + //mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of UZXFAM standard constructor + +UZDFAM::UZDFAM(PUZDFAM txfp) : DBMFAM(txfp) +{ + zutp = txfp->zutp; + tdfp = txfp->tdfp; + //target = txfp->target; + //mul = txfp->mul; + //Lrecl = txfp->Lrecl; +} // end of UZXFAM copy constructor + +#if 0 +/****************************************************************************/ +/* dbfhead: Routine to analyze a DBF header. */ +/* Parameters: */ +/* PGLOBAL g -- pointer to the CONNECT Global structure */ +/* DBFHEADER *hdrp -- pointer to _dbfheader structure */ +/* Returns: */ +/* RC_OK, RC_NF, RC_INFO, or RC_FX if error. */ +/* Side effects: */ +/* Set the fields number in the header. */ +/****************************************************************************/ +int UZDFAM::dbfhead(PGLOBAL g, void* buf) +{ + char *endmark; + int dbc = 2, rc = RC_OK; + DBFHEADER* hdrp = (DBFHEADER*)buf; + + *g->Message = '\0'; + + // Check first byte to be sure of .dbf type + if ((hdrp->Version & 0x03) != DBFTYPE) { + strcpy(g->Message, MSG(NOT_A_DBF_FILE)); + rc = RC_INFO; + + if ((hdrp->Version & 0x30) == 0x30) { + strcpy(g->Message, MSG(FOXPRO_FILE)); + dbc = 264; // FoxPro database container + } // endif Version + + } else + strcpy(g->Message, MSG(DBASE_FILE)); + + // Check last byte(s) of header + endmark = (char*)hdrp + hdrp->Headlen() - dbc; + + // Some headers just have 1D others have 1D00 following fields + if (endmark[0] != EOH && endmark[1] != EOH) { + sprintf(g->Message, MSG(NO_0DH_HEAD), dbc); + + if (rc == RC_OK) + return RC_FX; + + } // endif endmark + + // Calculate here the number of fields while we have the dbc info + hdrp->SetFields((hdrp->Headlen() - dbc - 1) / 32); + return rc; +} // end of dbfhead + +/****************************************************************************/ +/* ScanHeader: scan the DBF file header for number of records, record size,*/ +/* and header length. Set Records, check that Reclen is equal to lrecl and */ +/* return the header length or 0 in case of error. */ +/****************************************************************************/ +int UZDFAM::ScanHeader(PGLOBAL g, int* rln) +{ + int rc; + DBFHEADER header; + + /************************************************************************/ + /* Get the first 32 bytes of the header. */ + /************************************************************************/ + rc = dbfhead(g, &header); + + if (rc == RC_FX) + return -1; + + *rln = (int)header.Reclen(); + Records = (int)header.Records(); + return (int)header.Headlen(); +} // end of ScanHeader +#endif // 0 + +/***********************************************************************/ +/* ZIP GetFileLength: returns file size in number of bytes. */ +/***********************************************************************/ +int UZDFAM::GetFileLength(PGLOBAL g) +{ + int len; + + if (!zutp && OpenTableFile(g)) + return 0; + + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength + +/***********************************************************************/ +/* ZIP Cardinality: return the number of rows if possible. */ +/***********************************************************************/ +int UZDFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + card = Records; + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality + +/***********************************************************************/ +/* OpenTableFile: Open a DBF table file from a ZIP file. */ +/***********************************************************************/ +bool UZDFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)UNZIPUTL(tdfp); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + return AllocateBuffer(g); + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile + +/***********************************************************************/ +/* GetNext: go to next entry. */ +/***********************************************************************/ +int UZDFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); + + if (rc != RC_OK) + return rc; + + int len = zutp->size; + +#if 0 + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size +#endif // 0 + + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext + /* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ @@ -1045,7 +1305,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) strcpy(g->Message, "No insert into existing zip file"); return true; } else if (append && len > 0) { - UNZIPUTL *zutp = new(g) UNZIPUTL(target, false); + UNZIPUTL *zutp = new(g) UNZIPUTL(target, NULL, false); if (!zutp->IsInsertOk(g, filename)) { strcpy(g->Message, "No insert into existing entry"); @@ -1129,7 +1389,7 @@ bool ZPXFAM::OpenTableFile(PGLOBAL g) strcpy(g->Message, "No insert into existing zip file"); return true; } else if (append && len > 0) { - UNZIPUTL *zutp = new(g) UNZIPUTL(target, false); + UNZIPUTL *zutp = new(g) UNZIPUTL(target, NULL, false); if (!zutp->IsInsertOk(g, filename)) { strcpy(g->Message, "No insert into existing entry"); diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index be17d954728..7ff1fb0a543 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,7 +1,7 @@ /************** filamzip H Declares Source Code File (.H) **************/ -/* Name: filamzip.h Version 1.2 */ +/* Name: filamzip.h Version 1.3 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2016-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2016-2020 */ /* */ /* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ @@ -11,6 +11,7 @@ #include "block.h" #include "filamap.h" #include "filamfix.h" +#include "filamdbf.h" #include "zip.h" #include "unzip.h" @@ -18,6 +19,7 @@ typedef class UNZFAM *PUNZFAM; typedef class UZXFAM *PUZXFAM; +typedef class UZDFAM* PUZDFAM; typedef class ZIPFAM *PZIPFAM; typedef class ZPXFAM *PZPXFAM; @@ -53,7 +55,7 @@ class DllExport ZIPUTIL : public BLOCK { class DllExport UNZIPUTL : public BLOCK { public: // Constructor - UNZIPUTL(PCSZ tgt, bool mul); + UNZIPUTL(PCSZ tgt, PCSZ pw, bool mul); UNZIPUTL(PDOSDEF tdp); // Implementation @@ -143,6 +145,36 @@ class DllExport UZXFAM : public MPXFAM { PDOSDEF tdfp; }; // end of UZXFAM +/***********************************************************************/ +/* This is the fixed unzip file access method. */ +/***********************************************************************/ +class DllExport UZDFAM : public DBMFAM { + //friend class UNZFAM; +public: + // Constructors + UZDFAM(PDOSDEF tdp); + UZDFAM(PUZDFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)UZDFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); + //virtual int ReadBuffer(PGLOBAL g); + +protected: + int dbfhead(PGLOBAL g, void* buf); + int ScanHeader(PGLOBAL g, int* rln); + + // Members + UNZIPUTL* zutp; + PDOSDEF tdfp; +}; // end of UZDFAM + /***********************************************************************/ /* This is the zip file access method. */ /***********************************************************************/ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index a111082e786..bb56d6cd655 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -4507,12 +4507,12 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_ZIP: case TAB_OEM: - if (table && table->pos_in_table_list) // if SELECT - { - //Switch_to_definer_security_ctx backup_ctx(thd, table->pos_in_table_list); + if (table && table->pos_in_table_list) { // if SELECT +#if MYSQL_VERSION_ID > 100200 + Switch_to_definer_security_ctx backup_ctx(thd, table->pos_in_table_list); +#endif // VERSION_ID > 100200 return check_global_access(thd, FILE_ACL); - } - else + } else return check_global_access(thd, FILE_ACL); case TAB_ODBC: case TAB_JDBC: @@ -4528,7 +4528,7 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_VIR: // This is temporary until a solution is found return false; - } // endswitch type + } // endswitch type my_printf_error(ER_UNKNOWN_ERROR, "check_privileges failed", MYF(0)); return true; @@ -5882,7 +5882,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } else switch (ttp) { case TAB_DBF: - qrp= DBFColumns(g, dpath, fn, fnc == FNC_COL); + qrp= DBFColumns(g, dpath, fn, topt, fnc == FNC_COL); break; #if defined(ODBC_SUPPORT) case TAB_ODBC: @@ -6733,11 +6733,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, PCSZ m= GetListOption(g, "Mulentries", options->oplist, "NO"); bool mul= *m == '1' || *m == 'Y' || *m == 'y' || !stricmp(m, "ON"); - if (!entry && !mul) { - my_message(ER_UNKNOWN_ERROR, "Missing entry name", MYF(0)); - DBUG_RETURN(HA_ERR_INTERNAL_ERROR); - } // endif entry - strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); PlugSetPath(zbuf, options->filename, dbpath); PlugSetPath(buf, fn, dbpath); diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index 53e2bf377c4..bd3d3b893c1 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -380,7 +380,6 @@ MGODEF::MGODEF(void) Uri = NULL; Colist = NULL; Filter = NULL; - Level = 0; Base = 0; Version = 0; Pipe = false; diff --git a/storage/connect/mongo.h b/storage/connect/mongo.h index 97c391a217f..dcefac372c0 100644 --- a/storage/connect/mongo.h +++ b/storage/connect/mongo.h @@ -82,7 +82,6 @@ protected: PSZ Wrapname; /* Java wrapper name */ PCSZ Colist; /* Options list */ PCSZ Filter; /* Filtering query */ - int Level; /* Used for catalog table */ int Base; /* The array index base */ int Version; /* The Java driver version */ bool Pipe; /* True is Colist is a pipeline */ diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index f3d3a010266..8c5cc261899 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -49,7 +49,7 @@ bool XMLDOCUMENT::InitZip(PGLOBAL g, PCSZ entry) { #if defined(ZIP_SUPPORT) bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; - zip = new(g) UNZIPUTL(entry, mul); + zip = new(g) UNZIPUTL(entry, NULL, mul); return zip == NULL; #else // !ZIP_SUPPORT sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); diff --git a/storage/connect/tabcmg.cpp b/storage/connect/tabcmg.cpp index b9b7f6e4b60..f2ff721627c 100644 --- a/storage/connect/tabcmg.cpp +++ b/storage/connect/tabcmg.cpp @@ -26,6 +26,8 @@ #include "tabmul.h" #include "filter.h" +PQRYRES MGOColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt, bool info); + /* -------------------------- Class CMGDISC -------------------------- */ /***********************************************************************/ diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 8efe2aad702..b3147bb7357 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9.4 */ +/* Version 4.9.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2019 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -359,7 +359,26 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Zipped) { + if (Recfm == RECFM_DBF) { + if (Catfunc == FNC_NO) { + if (Zipped) { + if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { + txfp = new(g) UZDFAM(this); + } else { + strcpy(g->Message, "Zipped DBF tables are read only"); + return NULL; + } // endif's mode + + } else if (map) + txfp = new(g) DBMFAM(this); + else + txfp = new(g) DBFFAM(this); + + tdbp = new(g) TDBFIX(this, txfp); + } else + tdbp = new(g) TDBDCL(this); // Catfunc should be 'C' + + } else if (Zipped) { #if defined(ZIP_SUPPORT) if (Recfm == RECFM_VAR) { if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { @@ -389,17 +408,6 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); return NULL; #endif // !ZIP_SUPPORT - } else if (Recfm == RECFM_DBF) { - if (Catfunc == FNC_NO) { - if (map) - txfp = new(g) DBMFAM(this); - else - txfp = new(g) DBFFAM(this); - - tdbp = new(g) TDBFIX(this, txfp); - } else // Catfunc should be 'C' - tdbp = new(g) TDBDCL(this); - } else if (Recfm != RECFM_VAR && Compressed < 2) { if (Huge) txfp = new(g) BGXFAM(this); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 207a1277fce..80dfe63845d 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -30,6 +30,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class DBFBASE; friend class UNZIPUTL; friend class JSONCOL; + friend class TDBDCL; public: // Constructor DOSDEF(void); diff --git a/storage/connect/tabfix.h b/storage/connect/tabfix.h index 53c0af1c422..5f859a2bffe 100644 --- a/storage/connect/tabfix.h +++ b/storage/connect/tabfix.h @@ -98,18 +98,20 @@ class DllExport BINCOL : public DOSCOL { /* This is the class declaration for the DBF columns catalog table. */ /***********************************************************************/ class TDBDCL : public TDBCAT { - public: - // Constructor - TDBDCL(PDOSDEF tdp) : TDBCAT(tdp) {Fn = tdp->GetFn();} +public: + // Constructor + TDBDCL(PDOSDEF tdp) : TDBCAT(tdp) + {Fn = tdp->GetFn(); Topt = tdp->GetTopt();} - protected: +protected: // Specific routines - virtual PQRYRES GetResult(PGLOBAL g) - {return DBFColumns(g, ((PTABDEF)To_Def)->GetPath(), Fn, false);} + virtual PQRYRES GetResult(PGLOBAL g) + {return DBFColumns(g, ((PTABDEF)To_Def)->GetPath(), Fn, Topt, false);} - // Members + // Members PCSZ Fn; // The DBF file (path) name - }; // end of class TDBOCL + PTOS Topt; +}; // end of class TDBOCL #endif // __TABFIX__ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 7e8d6c8d9f0..3b0d458a7a6 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -739,6 +739,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) /***********************************************************************/ TDBJSN::TDBJSN(PJDEF tdp, PTXF txfp) : TDBDOS(tdp, txfp) { + G = NULL; Top = NULL; Row = NULL; Val = NULL; diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 8721a2a5ab7..8c3f1013919 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -104,7 +104,6 @@ public: PCSZ Xcol; /* Name of expandable column */ int Limit; /* Limit of multiple values */ int Pretty; /* Depends on file structure */ - int Level; /* Used for catalog table */ int Base; /* The array index base */ bool Strict; /* Strict syntax checking */ char Sep; /* The Jpath separator */ diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp index c026744dba8..d9c13e2a58a 100644 --- a/storage/connect/tabzip.cpp +++ b/storage/connect/tabzip.cpp @@ -23,6 +23,7 @@ #include "filamzip.h" #include "resource.h" // for IDS_COLUMNS #include "tabdos.h" +#include "tabmul.h" #include "tabzip.h" /* -------------------------- Class ZIPDEF --------------------------- */ @@ -41,7 +42,14 @@ bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) /***********************************************************************/ PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) { - return new(g) TDBZIP(this); + PTDB tdbp = NULL; + + tdbp = new(g) TDBZIP(this); + + if (Multiple) + tdbp = new(g) TDBMUL(tdbp); + + return tdbp; } // end of GetTable /* ------------------------------------------------------------------- */ @@ -108,7 +116,7 @@ int TDBZIP::Cardinality(PGLOBAL g) Cardinal = (err == UNZ_OK) ? (int)ginfo.number_entry : 0; } else - Cardinal = 0; + Cardinal = 10; // Dummy for multiple tables } // endif Cardinal @@ -187,6 +195,7 @@ int TDBZIP::DeleteDB(PGLOBAL g, int irc) void TDBZIP::CloseDB(PGLOBAL g) { close(); + nexterr = UNZ_OK; // For multiple tables Use = USE_READY; // Just to be clean } // end of CloseDB diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h index 32b15281f81..d36e4dc01d0 100644 --- a/storage/connect/tabzip.h +++ b/storage/connect/tabzip.h @@ -48,6 +48,8 @@ public: // Implementation virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + virtual PCSZ GetFile(PGLOBAL) {return zfn;} + virtual void SetFile(PGLOBAL, PCSZ fn) {zfn = fn;} // Methods virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); From 43ec9370b328fc9bf82e318bf992953a71925cd3 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Thu, 22 Oct 2020 07:16:29 +0530 Subject: [PATCH 02/12] MDEV-10149: sys_vars.rpl_init_slave_func fails sporadically in buildbot problem: ======== mysqltest: In included file "./include/assert.inc": included from mysql-test/suite/sys_vars/t/rpl_init_slave_func.test at line 69: Assertion text: '@@global.max_connections = @start_max_connections' Assertion result: '0' mysqltest: In included file "./include/assert.inc": included from mysql-test/suite/sys_vars/t/rpl_init_slave_func.test at line 86: Assertion text: '@@global.max_connections = @start_max_connections + 1' Assertion result: '0' Analysis: ========= A slave SQL thread sets its Running state to Yes very early in its initialisation, before the majority of initialisation actions, including executing the init_slave command, are done. Thus the testcase has a race condition where the initial replication setup might finish executing later than the testcase SET GLOBAL init_slave, making the testcase see its effect where it checks for its absence. Fix: === Include 'sync_slave_sql_with_master.inc' at the beginning of the test to ensure that slave applier has completed the execution of 'init_slave' command and proceeded to event application. Replace the apparently needless RESET MASTER / RESET SLAVE etc. Patch is based on: https://github.com/percona/percona-server/pull/1464/commits/b91e2e6f90611aa299c302929fb8b068e8ac0dee Author: laurynas-biveinis --- .../sys_vars/r/rpl_init_slave_func.result | 15 +++------ .../suite/sys_vars/t/rpl_init_slave_func.test | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result b/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result index 691f6f10e02..d63a540fe15 100644 --- a/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result +++ b/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -connection slave +include/sync_slave_sql_with_master.inc SET @start_max_connections= @@global.max_connections; SET @start_init_slave= @@global.init_slave; SET NAMES utf8; @@ -19,18 +19,11 @@ SELECT @@global.init_slave = 'SET @@global.max_connections = @@global.max_connec 1 Expect 1 include/assert.inc [@@global.max_connections = @start_max_connections] -STOP SLAVE; -RESET MASTER; -RESET SLAVE; -START SLAVE; -include/wait_for_slave_to_start.inc +include/restart_slave.inc +include/sync_slave_sql_with_master.inc include/assert.inc [@@global.max_connections = @start_max_connections + 1] SET @@global.init_slave = "SET @a=5"; -STOP SLAVE; -RESET MASTER; -RESET SLAVE; -START SLAVE; -include/wait_for_slave_to_start.inc +include/restart_slave.inc SHOW VARIABLES LIKE 'init_slave'; Variable_name Value init_slave SET @a=5 diff --git a/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test b/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test index 1d57bfeddc5..0e5ff7a3f39 100644 --- a/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test +++ b/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test @@ -29,8 +29,13 @@ ############################################################################### source include/master-slave.inc; ---echo connection slave -connection slave; + +# Since a part of slave SQL thread initialisation happens after Slave_SQL_Running +# has been set to Yes, there is a race condition between initialisation above and +# init_slave setting given below. Synchronise slave applier with master to ensure +# init_slave is complete and applier had processed few events like FD. +--source include/sync_slave_sql_with_master.inc + --disable_query_log call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); --enable_query_log @@ -67,14 +72,15 @@ let $wait_condition= SELECT @@global.max_connections = @start_max_connections; --let $assert_text= @@global.max_connections = @start_max_connections --let $assert_cond= @@global.max_connections = @start_max_connections --source include/assert.inc -# -# reset of the server -STOP SLAVE; ---wait_for_slave_to_stop -RESET MASTER; -RESET SLAVE; -START SLAVE; -source include/wait_for_slave_to_start.inc; + +--source include/restart_slave_sql.inc + +# Upon slave start, sync the applier with master, to ensure slave has +# completed init_slave command execution and processed FD event from the +# master. +--connection master +--source include/sync_slave_sql_with_master.inc + # # wait for the slave threads have set the global variable. let $wait_timeout= 90; @@ -88,12 +94,7 @@ let $wait_condition= SELECT @@global.max_connections = @start_max_connections + # Setting a variable(which is local to a session) and must not be visible SET @@global.init_slave = "SET @a=5"; # -STOP SLAVE; ---wait_for_slave_to_stop -RESET MASTER; -RESET SLAVE; -START SLAVE; -source include/wait_for_slave_to_start.inc; +--source include/restart_slave_sql.inc # SHOW VARIABLES LIKE 'init_slave'; # expect NULL From 897ea21e57034e45bf3bc5997ba1b560eabc8221 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Aug 2020 13:08:33 +0200 Subject: [PATCH 03/12] MDEV-23358 main.upgrade_MDEV-19650 fails with result difference When including a generated file, always use <...>. We need the compiler to find it in the BINDIR, not in the SRCDIR. But when including as "..." SRCDIR is always searched first. The bug can only happen in out-of-source builds, if there was an in-source build before. --- client/mysql_upgrade.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index d18dc97c9b8..8da7a4b01f2 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -18,7 +18,7 @@ #include "client_priv.h" #include -#include "../scripts/mysql_fix_privilege_tables_sql.c" +#include <../scripts/mysql_fix_privilege_tables_sql.c> #include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ From 94b493571a8ba8f30c97a5c30fc641171ca48e8a Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Fri, 23 Oct 2020 12:20:17 +0400 Subject: [PATCH 04/12] MDEV-20744 SET GLOBAL `replicate_do_db` = DEFAULT causes crash. DEFAULT for the replicate_do_db is the "" as our documentation states. --- mysql-test/suite/sys_vars/r/replicate_do_db_basic.result | 4 ++++ mysql-test/suite/sys_vars/t/replicate_do_db_basic.test | 3 +++ sql/sys_vars.ic | 6 +++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result index a05b85a9bfd..54adf835962 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result @@ -37,5 +37,9 @@ SET @@GLOBAL.replicate_do_db=null; SELECT @@GLOBAL.replicate_do_db; @@GLOBAL.replicate_do_db +SET @@GLOBAL.replicate_do_db=DEFAULT; +SELECT @@GLOBAL.replicate_do_db; +@@GLOBAL.replicate_do_db + # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test index 59d0176add2..b7004d1938b 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test @@ -38,5 +38,8 @@ SELECT @@GLOBAL.replicate_do_db; SET @@GLOBAL.replicate_do_db=null; SELECT @@GLOBAL.replicate_do_db; +SET @@GLOBAL.replicate_do_db=DEFAULT; +SELECT @@GLOBAL.replicate_do_db; + --echo # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index ae4568866b7..737f9bb545c 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -614,7 +614,11 @@ public: { DBUG_ASSERT(FALSE); } void global_save_default(THD *thd, set_var *var) - { DBUG_ASSERT(FALSE); } + { + char *ptr= (char*)(intptr)option.def_value; + var->save_result.string_value.str= ptr; + var->save_result.string_value.length= ptr ? strlen(ptr) : 0; + } bool session_update(THD *thd, set_var *var) { From f679d72679376d04c863e80bc68d084eb56795a5 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Thu, 30 Nov 2017 00:41:43 +0300 Subject: [PATCH 05/12] MDEV-24017: Blackhole : Specified key was too long; max key length is 1000 bytes The maximum innodb key length is 3500 what is hardcoded in ha_innobase::max_supported_key_length()). The maximum number of innodb indexes is configured with MAX_INDEXES macro (see also MAX_KEY definition). The same is currently implemented for blackhole storage engine. Cherry picked from percona-server 0d90d81c3c507a6b1476246a405504f6e4ef9d4d Original lp bug 1733049 Reviewed-by: daniel@mariadb.org --- mysql-test/r/blackhole.result | 6 ++++++ mysql-test/t/blackhole.test | 16 ++++++++++++++++ storage/blackhole/ha_blackhole.h | 5 +++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/blackhole.result b/mysql-test/r/blackhole.result index 36f5459ff85..a7281c42ca7 100644 --- a/mysql-test/r/blackhole.result +++ b/mysql-test/r/blackhole.result @@ -24,3 +24,9 @@ SELECT 0 FROM t1 FORCE INDEX FOR GROUP BY(a) WHERE a = 0 OR b = 0 AND c = 0; 0 DROP TABLE t1; End of 5.6 tests +CREATE TABLE `t` ( +`a` varchar(3000) NOT NULL default '', +PRIMARY KEY (`a`) +) ENGINE=BLACKHOLE; +DROP TABLE `t`; +End of 10.1 tests diff --git a/mysql-test/t/blackhole.test b/mysql-test/t/blackhole.test index 7f394e0f846..c80ceffef4c 100644 --- a/mysql-test/t/blackhole.test +++ b/mysql-test/t/blackhole.test @@ -38,3 +38,19 @@ SELECT 0 FROM t1 FORCE INDEX FOR GROUP BY(a) WHERE a = 0 OR b = 0 AND c = 0; DROP TABLE t1; --echo End of 5.6 tests + +# +# MDEV-24017 / bug 53588 test case. +# +# Create long enough index (between 1000 and 3500). 1000 is the old value, +# 3500 is innodb value (see ha_innobase::max_supported_key_length()). Without +# the fix the test will fail with "Specified key was too long" error. +# +CREATE TABLE `t` ( + `a` varchar(3000) NOT NULL default '', + PRIMARY KEY (`a`) +) ENGINE=BLACKHOLE; + +DROP TABLE `t`; + +--echo End of 10.1 tests diff --git a/storage/blackhole/ha_blackhole.h b/storage/blackhole/ha_blackhole.h index e34386ddf33..07275b8eec1 100644 --- a/storage/blackhole/ha_blackhole.h +++ b/storage/blackhole/ha_blackhole.h @@ -20,6 +20,7 @@ #include "thr_lock.h" /* THR_LOCK */ #include "handler.h" /* handler */ #include "table.h" /* TABLE_SHARE */ +#include "sql_const.h" /* MAX_KEY */ /* Shared structure for correct LOCK operation @@ -65,9 +66,9 @@ public: HA_READ_ORDER | HA_KEYREAD_ONLY); } /* The following defines can be increased if necessary */ -#define BLACKHOLE_MAX_KEY 64 /* Max allowed keys */ +#define BLACKHOLE_MAX_KEY MAX_KEY /* Max allowed keys */ #define BLACKHOLE_MAX_KEY_SEG 16 /* Max segments for key */ -#define BLACKHOLE_MAX_KEY_LENGTH 1000 +#define BLACKHOLE_MAX_KEY_LENGTH 3500 /* Like in InnoDB */ uint max_supported_keys() const { return BLACKHOLE_MAX_KEY; } uint max_supported_key_length() const { return BLACKHOLE_MAX_KEY_LENGTH; } uint max_supported_key_part_length() const { return BLACKHOLE_MAX_KEY_LENGTH; } From 44c958dd7b454ebbdbf7ac8b066592c82dd3409f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 24 Oct 2020 14:57:16 +0300 Subject: [PATCH 06/12] Fix test failure on wsrep/variables test case. --- mysql-test/include/galera_have_debug_sync.inc | 9 +++++++++ mysql-test/suite/wsrep/t/variables.test | 1 + 2 files changed, 10 insertions(+) create mode 100644 mysql-test/include/galera_have_debug_sync.inc diff --git a/mysql-test/include/galera_have_debug_sync.inc b/mysql-test/include/galera_have_debug_sync.inc new file mode 100644 index 00000000000..21e7b3c88c3 --- /dev/null +++ b/mysql-test/include/galera_have_debug_sync.inc @@ -0,0 +1,9 @@ +--disable_query_log + +--let $galera_have_debug_sync = `SELECT 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'` + +--if (!$galera_have_debug_sync) { + --skip Test requires Galera debug library with debug_sync functionality +} + +--enable_query_log diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 1a2ab2579a5..15b69f8b7f6 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -1,4 +1,5 @@ --source include/have_wsrep.inc +--source include/galera_have_debug_sync.inc SET @wsrep_provider_options_saved= @@global.wsrep_provider_options; SET @wsrep_cluster_address_saved= @@global.wsrep_cluster_address; From 858434910835117215890f8368b91b2905e74815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 23 Oct 2020 18:09:01 +0300 Subject: [PATCH 07/12] MDEV-14945 possible buffer overflow in stack resolver According to https://stackoverflow.com/questions/22827510/how-to-avoid-bad-fd-set-buffer-overflow-crash it seems that using select instead of poll can cause additional memory allocations. As we are in a crashed state, we must prevent allocating any memory (if possible). Thus, switch select call to poll. Also move some bigger datastructures to global space. The code is not run in a multithreaded context so best we don't use up stack space if it's not needed. --- mysys/my_addr_resolve.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index ecb512572b1..e0ebbe699be 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -147,10 +147,18 @@ err: #include +#if defined(HAVE_POLL_H) +#include +#elif defined(HAVE_SYS_POLL_H) +#include +#endif /* defined(HAVE_POLL_H) */ + static int in[2], out[2]; static pid_t pid; static char addr2line_binary[1024]; static char output[1024]; +static struct pollfd poll_fds; +Dl_info info; int start_addr2line_fork(const char *binary_path) { @@ -200,15 +208,16 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) ssize_t extra_bytes_read = 0; ssize_t parsed = 0; - fd_set set; - struct timeval timeout; + int ret; int filename_start = -1; int line_number_start = -1; - Dl_info info; void *offset; + poll_fds.fd = out[0]; + poll_fds.events = POLLIN | POLLRDBAND; + if (!dladdr(ptr, &info)) return 1; @@ -230,16 +239,16 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) if (write(in[1], input, len) <= 0) return 1; - FD_ZERO(&set); - FD_SET(out[0], &set); - /* 100 ms should be plenty of time for addr2line to issue a response. */ - timeout.tv_sec = 0; - timeout.tv_usec = 100000; + /* 500 ms should be plenty of time for addr2line to issue a response. */ /* Read in a loop till all the output from addr2line is complete. */ while (parsed == total_bytes_read && - select(out[0] + 1, &set, NULL, NULL, &timeout) > 0) + (ret= poll(&poll_fds, 1, 500))) { + /* error during poll */ + if (ret < 0) + return 1; + extra_bytes_read= read(out[0], output + total_bytes_read, sizeof(output) - total_bytes_read); if (extra_bytes_read < 0) From 320a73f6a29c6d6adf8576651263812d42796235 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 25 Oct 2020 18:16:24 +0100 Subject: [PATCH 08/12] cleanup: PRIV_LOCK_TABLES (10.5 style) --- sql/sql_parse.cc | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 14ac657862f..fb7552e5929 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -100,6 +100,8 @@ #include "my_json_writer.h" +#define PRIV_LOCK_TABLES (SELECT_ACL | LOCK_TABLES_ACL) + #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") #ifdef WITH_ARIA_STORAGE_ENGINE @@ -4560,7 +4562,7 @@ mysql_execute_command(THD *thd) if (first_table && lex->type & (REFRESH_READ_LOCK|REFRESH_FOR_EXPORT)) { /* Check table-level privileges. */ - if (check_table_access(thd, LOCK_TABLES_ACL | SELECT_ACL, all_tables, + if (check_table_access(thd, PRIV_LOCK_TABLES, all_tables, FALSE, UINT_MAX, FALSE)) goto error; @@ -6065,7 +6067,7 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, @param thd Thread handler @param privilege requested privilege - @param all_tables global table list of query + @param tables global table list of query @param no_errors FALSE/TRUE - report/don't report error to the client (using my_error() call). @@ -6075,32 +6077,29 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, 1 access denied, error is sent to client */ -bool check_single_table_access(THD *thd, ulong privilege, - TABLE_LIST *all_tables, bool no_errors) +bool check_single_table_access(THD *thd, ulong privilege, TABLE_LIST *tables, + bool no_errors) { Security_context * backup_ctx= thd->security_ctx; /* we need to switch to the saved context (if any) */ - if (all_tables->security_ctx) - thd->security_ctx= all_tables->security_ctx; + if (tables->security_ctx) + thd->security_ctx= tables->security_ctx; const char *db_name; - if ((all_tables->view || all_tables->field_translation) && - !all_tables->schema_table) - db_name= all_tables->view_db.str; + if ((tables->view || tables->field_translation) && !tables->schema_table) + db_name= tables->view_db.str; else - db_name= all_tables->db; + db_name= tables->db; - if (check_access(thd, privilege, db_name, - &all_tables->grant.privilege, - &all_tables->grant.m_internal, - 0, no_errors)) + if (check_access(thd, privilege, db_name, &tables->grant.privilege, + &tables->grant.m_internal, 0, no_errors)) goto deny; /* Show only 1 table for check_grant */ - if (!(all_tables->belong_to_view && - (thd->lex->sql_command == SQLCOM_SHOW_FIELDS)) && - check_grant(thd, privilege, all_tables, FALSE, 1, no_errors)) + if (!(tables->belong_to_view && + (thd->lex->sql_command == SQLCOM_SHOW_FIELDS)) && + check_grant(thd, privilege, tables, FALSE, 1, no_errors)) goto deny; thd->security_ctx= backup_ctx; @@ -9060,7 +9059,7 @@ static bool lock_tables_precheck(THD *thd, TABLE_LIST *tables) if (is_temporary_table(table)) continue; - if (check_table_access(thd, LOCK_TABLES_ACL | SELECT_ACL, table, + if (check_table_access(thd, PRIV_LOCK_TABLES, table, FALSE, 1, FALSE)) return TRUE; } From 0c3723e1d50e61303b63f6264c2c193397ee4475 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 25 Oct 2020 18:17:34 +0100 Subject: [PATCH 09/12] Bug#31304432 "INSUFFICIENT PRIVILEGE CHECK BY LOCK TABLES" `LOCK TABLES view_name` should require * invoker to have SELECT and LOCK TABLES privileges on the view * either invoker or definer (only if sql security definer) to have SELECT and LOCK TABLES privileges on the used tables/views. --- mysql-test/r/lock_view.result | 216 ++++++++++++++++++++++++++++++++++ mysql-test/t/lock_view.test | 76 ++++++++++++ sql/sql_parse.cc | 30 +++++ 3 files changed, 322 insertions(+) create mode 100644 mysql-test/r/lock_view.result create mode 100644 mysql-test/t/lock_view.test diff --git a/mysql-test/r/lock_view.result b/mysql-test/r/lock_view.result new file mode 100644 index 00000000000..aa76dd73609 --- /dev/null +++ b/mysql-test/r/lock_view.result @@ -0,0 +1,216 @@ +create database mysqltest1; +create database mysqltest2; +create database mysqltest3; +create user invoker@localhost; +create user definer@localhost; +grant select,show view on mysqltest1.* to invoker@localhost; +grant select,show view on mysqltest1.* to definer@localhost; +grant select,show view on mysqltest2.* to invoker@localhost; +grant select,show view on mysqltest2.* to definer@localhost; +grant select,show view on mysqltest3.* to invoker@localhost; +grant select on performance_schema.* to definer@localhost; +create table mysqltest1.t1 (a int); +create definer=definer@localhost view mysqltest2.v2 as select * from mysqltest1.t1; +create definer=definer@localhost view mysqltest3.v3 as select * from mysqltest2.v2; +create definer=definer@localhost view mysqltest3.v3is as select schema_name from information_schema.schemata order by schema_name; +create definer=definer@localhost view mysqltest3.v3ps as select user from performance_schema.users where current_connections>0 order by user; +create definer=definer@localhost view mysqltest3.v3nt as select 1; +create definer=definer@localhost sql security invoker view mysqltest3.v3i as select * from mysqltest1.t1; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest1` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest1`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest2` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest2`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v2` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest3` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest3`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3i` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3is` ( + `schema_name` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3nt` ( + `1` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3ps` ( + `user` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; + +USE `mysqltest1`; + +USE `mysqltest2`; +/*!50001 DROP TABLE IF EXISTS `v2`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v2` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; + +USE `mysqltest3`; +/*!50001 DROP TABLE IF EXISTS `v3`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3` AS select `v2`.`a` AS `a` from `mysqltest2`.`v2` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3i`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY INVOKER */ +/*!50001 VIEW `v3i` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3is`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3is` AS select `information_schema`.`schemata`.`SCHEMA_NAME` AS `schema_name` from `information_schema`.`schemata` order by `information_schema`.`schemata`.`SCHEMA_NAME` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3nt`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3nt` AS select 1 AS `1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3ps`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3ps` AS select `performance_schema`.`users`.`USER` AS `user` from `performance_schema`.`users` where (`performance_schema`.`users`.`CURRENT_CONNECTIONS` > 0) order by `performance_schema`.`users`.`USER` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +lock table mysqltest3.v3 write; +ERROR 42000: Access denied for user 'invoker'@'localhost' to database 'mysqltest3' +grant lock tables on mysqltest3.* to invoker@localhost; +show create view mysqltest3.v3; +View Create View character_set_client collation_connection +v3 CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3` AS select `v2`.`a` AS `a` from `mysqltest2`.`v2` latin1 latin1_swedish_ci +show create view mysqltest3.v3is; +View Create View character_set_client collation_connection +v3is CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3is` AS select `information_schema`.`schemata`.`SCHEMA_NAME` AS `schema_name` from `information_schema`.`schemata` order by `information_schema`.`schemata`.`SCHEMA_NAME` latin1 latin1_swedish_ci +show create view mysqltest3.v3ps; +View Create View character_set_client collation_connection +v3ps CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3ps` AS select `performance_schema`.`users`.`USER` AS `user` from `performance_schema`.`users` where (`performance_schema`.`users`.`CURRENT_CONNECTIONS` > 0) order by `performance_schema`.`users`.`USER` latin1 latin1_swedish_ci +show create view mysqltest3.v3nt; +View Create View character_set_client collation_connection +v3nt CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3nt` AS select 1 AS `1` latin1 latin1_swedish_ci +show create view mysqltest3.v3i; +View Create View character_set_client collation_connection +v3i CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY INVOKER VIEW `mysqltest3`.`v3i` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` latin1 latin1_swedish_ci +lock table mysqltest3.v3 write; +ERROR HY000: View 'mysqltest3.v3' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3is write; +select * from mysqltest3.v3is; +schema_name +information_schema +mysqltest1 +mysqltest2 +mysqltest3 +test +lock table mysqltest3.v3ps write; +select * from mysqltest3.v3ps; +user +NULL +invoker +root +lock table mysqltest3.v3nt write; +select * from mysqltest3.v3nt; +1 +1 +grant lock tables on mysqltest2.* to invoker@localhost; +lock table mysqltest3.v3 write; +ERROR HY000: View 'mysqltest3.v3' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +grant lock tables on mysqltest1.* to definer@localhost; +lock table mysqltest3.v3 write; +select * from mysqltest3.v3; +a +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +grant lock tables on mysqltest1.* to invoker@localhost; +lock table mysqltest3.v3i write; +select * from mysqltest3.v3i; +a +drop user invoker@localhost; +drop user definer@localhost; +drop database mysqltest1; +drop database mysqltest2; +drop database mysqltest3; diff --git a/mysql-test/t/lock_view.test b/mysql-test/t/lock_view.test new file mode 100644 index 00000000000..dd8809ab89d --- /dev/null +++ b/mysql-test/t/lock_view.test @@ -0,0 +1,76 @@ +source include/not_embedded.inc; +# +# LOCK TABLES and privileges on views +# +create database mysqltest1; +create database mysqltest2; +create database mysqltest3; +create user invoker@localhost; +create user definer@localhost; +grant select,show view on mysqltest1.* to invoker@localhost; +grant select,show view on mysqltest1.* to definer@localhost; +grant select,show view on mysqltest2.* to invoker@localhost; +grant select,show view on mysqltest2.* to definer@localhost; +grant select,show view on mysqltest3.* to invoker@localhost; +grant select on performance_schema.* to definer@localhost; +create table mysqltest1.t1 (a int); +create definer=definer@localhost view mysqltest2.v2 as select * from mysqltest1.t1; +create definer=definer@localhost view mysqltest3.v3 as select * from mysqltest2.v2; +create definer=definer@localhost view mysqltest3.v3is as select schema_name from information_schema.schemata order by schema_name; +create definer=definer@localhost view mysqltest3.v3ps as select user from performance_schema.users where current_connections>0 order by user; +create definer=definer@localhost view mysqltest3.v3nt as select 1; +create definer=definer@localhost sql security invoker view mysqltest3.v3i as select * from mysqltest1.t1; + +exec $MYSQL_DUMP --compact -B mysqltest1 mysqltest2 mysqltest3; + +connect inv,localhost,invoker; +error ER_DBACCESS_DENIED_ERROR; +lock table mysqltest3.v3 write; +disconnect inv; +connection default; + +grant lock tables on mysqltest3.* to invoker@localhost; +connect inv,localhost,invoker; +show create view mysqltest3.v3; +show create view mysqltest3.v3is; +show create view mysqltest3.v3ps; +show create view mysqltest3.v3nt; +show create view mysqltest3.v3i; +error ER_VIEW_INVALID; +lock table mysqltest3.v3 write; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +lock table mysqltest3.v3is write; select * from mysqltest3.v3is; +lock table mysqltest3.v3ps write; select * from mysqltest3.v3ps; +lock table mysqltest3.v3nt write; select * from mysqltest3.v3nt; +disconnect inv; +connection default; + +grant lock tables on mysqltest2.* to invoker@localhost; +connect inv,localhost,invoker; +error ER_VIEW_INVALID; +lock table mysqltest3.v3 write; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +disconnect inv; +connection default; + +grant lock tables on mysqltest1.* to definer@localhost; +connect inv,localhost,invoker; +lock table mysqltest3.v3 write; select * from mysqltest3.v3; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +disconnect inv; +connection default; + +grant lock tables on mysqltest1.* to invoker@localhost; +connect inv,localhost,invoker; +lock table mysqltest3.v3i write; select * from mysqltest3.v3i; +disconnect inv; +connection default; + +drop user invoker@localhost; +drop user definer@localhost; +drop database mysqltest1; +drop database mysqltest2; +drop database mysqltest3; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index fb7552e5929..bf1dad804e3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2392,10 +2392,40 @@ static bool lock_tables_open_and_lock_tables(THD *thd, TABLE_LIST *tables) We don't set TABLE_LIST::lock_type in this case as this might result in extra warnings from THD::decide_logging_format() even though binary logging is totally irrelevant for LOCK TABLES. + + Check privileges of view tables here, after views were opened. + Either definer or invoker has to have PRIV_LOCK_TABLES to be able to + lock view and its tables. For mysqldump (that locks views before dumping + their structures) compatibility we allow locking views that select + from I_S or P_S tables, but downrade the lock to TL_READ */ for (table= tables; table; table= table->next_global) + { if (!table->placeholder() && table->table->s->tmp_table) table->table->reginfo.lock_type= TL_WRITE; + else if (table->belong_to_view && + check_single_table_access(thd, PRIV_LOCK_TABLES, table, 1)) + { + if (table->grant.m_internal.m_schema_access) + table->lock_type= TL_READ; + else + { + bool error= true; + if (Security_context *sctx= table->security_ctx) + { + table->security_ctx= 0; + error= check_single_table_access(thd, PRIV_LOCK_TABLES, table, 1); + table->security_ctx= sctx; + } + if (error) + { + my_error(ER_VIEW_INVALID, MYF(0), table->belong_to_view->view_db.str, + table->belong_to_view->view_name.str); + goto err; + } + } + } + } if (lock_tables(thd, tables, counter, 0) || thd->locked_tables_list.init_locked_tables(thd)) From 1269fd420dbcac2cedb345944a35aebe3562ddc1 Mon Sep 17 00:00:00 2001 From: Karthik Kamath Date: Mon, 31 Aug 2020 12:21:07 +0530 Subject: [PATCH 10/12] BUG#31650096: MYSQL SERVER HEAP-USE-AFTER-FREE IN TRANS_SAVEPOINT ANALYSIS: ========= During Bootstrap, while executing the statements from sql file passed to the init-file server option, transaction mem_root was being freed for every statement. This creates an issue with multi statement transactions especially when a statement in the transaction has to access the memory used by the previous statement in the transaction. FIX: ==== Transaction mem_root is freed whenever a transaction is committed or rolled-back. Hence explicitly freeing it is not necessary in the bootstrap implementation. Change-Id: I40f71d49781bf7ad32d474bb176bd6060c9377dc --- sql/sql_parse.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index bf1dad804e3..64b9f35d664 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -811,7 +811,6 @@ static void handle_bootstrap_impl(THD *thd) thd->reset_kill_query(); /* Ensure that killed_errmsg is released */ free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC)); - free_root(&thd->transaction.mem_root,MYF(MY_KEEP_PREALLOC)); thd->lex->restore_set_statement_var(); } From a7d5e85c4935080458fea21c718f56c7bf6f02bd Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Oct 2020 17:20:59 +0100 Subject: [PATCH 11/12] cleanup: have_static_innodb.inc and remove unused files --- mysql-test/include/have_not_innodb_plugin.inc | 4 ---- mysql-test/include/have_static_innodb.inc | 7 +++++++ mysql-test/r/not_true.require | 2 -- mysql-test/t/plugin_innodb.test | 9 +-------- 4 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 mysql-test/include/have_not_innodb_plugin.inc create mode 100644 mysql-test/include/have_static_innodb.inc delete mode 100644 mysql-test/r/not_true.require diff --git a/mysql-test/include/have_not_innodb_plugin.inc b/mysql-test/include/have_not_innodb_plugin.inc deleted file mode 100644 index e40fd811021..00000000000 --- a/mysql-test/include/have_not_innodb_plugin.inc +++ /dev/null @@ -1,4 +0,0 @@ -disable_query_log; ---require r/not_true.require -select (PLUGIN_LIBRARY LIKE 'ha_innodb_plugin%' OR PLUGIN_DESCRIPTION LIKE '%xtradb%') as `TRUE` from information_schema.plugins where PLUGIN_NAME='InnoDB'; -enable_query_log; diff --git a/mysql-test/include/have_static_innodb.inc b/mysql-test/include/have_static_innodb.inc new file mode 100644 index 00000000000..0d7bb856f4f --- /dev/null +++ b/mysql-test/include/have_static_innodb.inc @@ -0,0 +1,7 @@ +source include/have_innodb.inc; + +if (!`select count(*) from information_schema.plugins + where plugin_name = 'innodb' and plugin_status = 'active' and + plugin_library is null`) { + skip Need compiled-in InnoDB; +} diff --git a/mysql-test/r/not_true.require b/mysql-test/r/not_true.require deleted file mode 100644 index 0032832f3d1..00000000000 --- a/mysql-test/r/not_true.require +++ /dev/null @@ -1,2 +0,0 @@ -TRUE -NULL diff --git a/mysql-test/t/plugin_innodb.test b/mysql-test/t/plugin_innodb.test index fb5dd84b997..03afa190400 100644 --- a/mysql-test/t/plugin_innodb.test +++ b/mysql-test/t/plugin_innodb.test @@ -1,13 +1,6 @@ --source include/not_embedded.inc --source include/have_example_plugin.inc ---source include/have_innodb.inc - -if (!`select count(*) from information_schema.plugins - where plugin_name = 'innodb' and plugin_status = 'active' and - plugin_library is null`) { - skip Need compiled-in InnoDB; -} - +--source include/have_static_innodb.inc --replace_regex /\.dll/.so/ eval install plugin example soname '$HA_EXAMPLE_SO'; From d03ea82759aef7734f162c3137b10aa2476b35e9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Oct 2020 17:21:26 +0100 Subject: [PATCH 12/12] test case for BUG#31650096 --- mysql-test/r/bootstrap_innodb.result | 7 +++++++ mysql-test/t/bootstrap_innodb.test | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 mysql-test/r/bootstrap_innodb.result create mode 100644 mysql-test/t/bootstrap_innodb.test diff --git a/mysql-test/r/bootstrap_innodb.result b/mysql-test/r/bootstrap_innodb.result new file mode 100644 index 00000000000..2fc7800843b --- /dev/null +++ b/mysql-test/r/bootstrap_innodb.result @@ -0,0 +1,7 @@ +create table t1(a int) engine=innodb; +select * from t1; +a +1 +2 +5 +drop table t1; diff --git a/mysql-test/t/bootstrap_innodb.test b/mysql-test/t/bootstrap_innodb.test new file mode 100644 index 00000000000..ddaefb32155 --- /dev/null +++ b/mysql-test/t/bootstrap_innodb.test @@ -0,0 +1,27 @@ +source include/have_static_innodb.inc; +source include/not_embedded.inc; + +let $datadir= `select @@datadir`; + +create table t1(a int) engine=innodb; +source include/shutdown_mysqld.inc; + +write_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql; +use test; +insert t1 values (1); +start transaction; +insert t1 values (2); +savepoint s1; +insert t1 values (3); +savepoint s2; +insert t1 values (4); +rollback to savepoint s1; +insert t1 values (5); +commit; +EOF +exec $MYSQLD_BOOTSTRAP_CMD --datadir=$datadir --innodb < $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1; +remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql; + +source include/start_mysqld.inc; +select * from t1; +drop table t1;