1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BUG#19580 - FULLTEXT search produces wrong results on UTF-8 columns

Browse Source 
The problem was that MySQL hadn't true ctype implementation. As a
result many multibyte punctuation/whitespace characters were
treated as word characters.

This fix uses recently added CTYPE table for unicode character sets
(WL1386) to detect unicode punctuation/whitespace characters
correctly.

Note: this is incompatible change since it changes parser behavior.
One will have to use REPAIR TABLE statement to rebuild fulltext
indexes.


mysql-test/r/fulltext2.result:
  Testcase for BUG#19580.
mysql-test/t/fulltext2.test:
  Testcase for BUG#19580.
storage/myisam/ft_parser.c:
  Use WL1386 "CTYPE table for unicode character sets" functionality.
storage/myisam/ft_update.c:
  Use WL1386 "CTYPE table for unicode character sets" functionality.
  
  Reverse fix for BUG#16489 "utf8 + fulltext leads to corrupt index
  file.". It is not needed anymore, since we have true ctype
  implementation.
storage/myisam/ftdefs.h:
  Use WL1386 "CTYPE table for unicode character sets" functionality.
  
  Rework true_word_char macro so it accepts ctype instead of charset
  as first param. It doesn't use my_isalnum anymore, but instead
  directly checks ctype.
  Obsolete word_char macro removed.
This commit is contained in:
unknown 2006-05-29 16:46:46 +05:00
parent 52078846fc
commit 528e85a4c0
5 changed files with 43 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mysql-test/r/fulltext2.result
View File

@ -241,3 +241,11 @@ select * from t1 where match a against('ab c' in boolean mode);
a
drop table t1;
set names latin1;
SET NAMES utf8;
CREATE TABLE t1(a VARCHAR(255), FULLTEXT(a)) ENGINE=MyISAM DEFAULT CHARSET=utf8;
INSERT INTO t1 VALUES('„MySQL“');
SELECT a FROM t1 WHERE MATCH a AGAINST('“MySQL„' IN BOOLEAN MODE);
a
„MySQL“
DROP TABLE t1;
SET NAMES latin1;

10
mysql-test/t/fulltext2.test
View File

@ -221,3 +221,13 @@ drop table t1;
set names latin1;
# End of 4.1 tests
#
# BUG#19580 - FULLTEXT search produces wrong results on UTF-8 columns
#
SET NAMES utf8;
CREATE TABLE t1(a VARCHAR(255), FULLTEXT(a)) ENGINE=MyISAM DEFAULT CHARSET=utf8;
INSERT INTO t1 VALUES('„MySQL“');
SELECT a FROM t1 WHERE MATCH a AGAINST('“MySQL„' IN BOOLEAN MODE);
DROP TABLE t1;
SET NAMES latin1;

32
storage/myisam/ft_parser.c
View File

@ -114,6 +114,7 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
{
byte *doc=*start;
int ctype;
uint mwc, length, mbl;
param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
@ -122,9 +123,11 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
while (doc<end)
{
for (;doc<end;doc++)
for (; doc < end; doc+= (mbl > 0 ? mbl : 1))
{
if (true_word_char(cs,*doc)) break;
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
if (true_word_char(ctype, *doc))
break;
if (*doc == FTB_RQUOT && param->quot)
{
param->quot=doc;
@ -158,14 +161,16 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
}
mwc=length=0;
for (word->pos=doc; doc<end; length++, mbl=my_mbcharlen(cs, *(uchar *)doc), doc+=(mbl ? mbl : 1))
if (true_word_char(cs,*doc))
for (word->pos= doc; doc < end; length++, doc+= (mbl > 0 ? mbl : 1))
{
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
if (true_word_char(ctype, *doc))
mwc=0;
else if (!misc_word_char(*doc) || mwc)
break;
else
mwc++;
}
param->prev='A'; /* be sure *prev is true_word_char */
word->len= (uint)(doc-word->pos) - mwc;
if ((param->trunc=(doc<end && *doc == FTB_TRUNC)))
@ -200,24 +205,31 @@ byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end,
{
byte *doc= *start;
uint mwc, length, mbl;
int ctype;
DBUG_ENTER("ft_simple_get_word");
do
{
for (;; doc++)
for (;; doc+= (mbl > 0 ? mbl : 1))
{
if (doc >= end) DBUG_RETURN(0);
if (true_word_char(cs, *doc)) break;
if (doc >= end)
DBUG_RETURN(0);
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
if (true_word_char(ctype, *doc))
break;
}
mwc= length= 0;
for (word->pos=doc; doc<end; length++, mbl=my_mbcharlen(cs, *(uchar *)doc), doc+=(mbl ? mbl : 1))
if (true_word_char(cs,*doc))
for (word->pos= doc; doc < end; length++, doc+= (mbl > 0 ? mbl : 1))
{
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
if (true_word_char(ctype, *doc))
mwc= 0;
else if (!misc_word_char(*doc) || mwc)
break;
else
mwc++;
}
word->len= (uint)(doc-word->pos) - mwc;

5
storage/myisam/ft_update.c
View File

@ -174,11 +174,6 @@ int _mi_ft_cmp(MI_INFO *info, uint keynr, const byte *rec1, const byte *rec2)
FT_SEG_ITERATOR ftsi1, ftsi2;
CHARSET_INFO *cs=info->s->keyinfo[keynr].seg->charset;
DBUG_ENTER("_mi_ft_cmp");
#ifndef MYSQL_HAS_TRUE_CTYPE_IMPLEMENTATION
if (cs->mbmaxlen > 1)
DBUG_RETURN(THOSE_TWO_DAMN_KEYS_ARE_REALLY_DIFFERENT);
#endif
_mi_ft_segiterator_init(info, keynr, rec1, &ftsi1);
_mi_ft_segiterator_init(info, keynr, rec2, &ftsi2);

5
storage/myisam/ftdefs.h
View File

@ -24,9 +24,10 @@
#include <queues.h>
#include <mysql/plugin.h>
#define true_word_char(s,X) (my_isalnum(s,X) || (X)=='_')
#define true_word_char(ctype, character) \
((ctype) & (_MY_U | _MY_L | _MY_NMR) || \
(character) == '_')
#define misc_word_char(X) 0
#define word_char(s,X) (true_word_char(s,X) || misc_word_char(X))
#define FT_MAX_WORD_LEN_FOR_SORT 31