diff --git a/ChangeLog b/ChangeLog index 4f8312e80a..9b2727538c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Wed Oct 24 17:09:39 2007 Nobuyoshi Nakada + + * parse.y (parser_tokspace): make space in token buffer. + + * parse.y (parser_yylex): fix encoding of single character literal. + Tue Oct 23 13:44:14 2007 Nobuyoshi Nakada * parse.y (call_args2): nd_head of NODE_BLOCK_PASS should be a list. diff --git a/parse.y b/parse.y index 8a46f2d71d..3182ceee0e 100644 --- a/parse.y +++ b/parse.y @@ -4494,6 +4494,7 @@ static int parser_here_document(struct parser_params*,NODE*); # define nextc() parser_nextc(parser) # define pushback(c) parser_pushback(parser, c) # define newtok() parser_newtok(parser) +# define tokspace(n) parser_tokspace(parser, n) # define tokadd(c) parser_tokadd(parser, c) # define read_escape(m) parser_read_escape(parser, m) # define tokadd_escape(t,m) parser_tokadd_escape(parser, t, m) @@ -4913,6 +4914,18 @@ parser_newtok(struct parser_params *parser) return tokenbuf; } +static char * +parser_tokspace(struct parser_params *parser, int n) +{ + int idx = tokidx + n; + + if (idx >= toksiz) { + do {toksiz *= 2;} while (toksiz < idx); + REALLOC_N(tokenbuf, char, toksiz); + } + return &tokenbuf[tokidx]; +} + static void parser_tokadd(struct parser_params *parser, int c) { @@ -6086,21 +6099,24 @@ parser_yylex(struct parser_params *parser) } newtok(); if (parser_ismbchar()) { + mb = ENC_CODERANGE_MULTI; tokadd_mbchar(c); } else if ((rb_enc_isalnum(c, parser->enc) || c == '_') && lex_p < lex_pend && is_identchar(lex_p, lex_pend, parser->enc)) { goto ternary; } - else if (c == '\\') { - c = read_escape(0); - tokadd(c); + else if (c == '\\' && (c = read_escape(0)) >= 0x80) { + rb_encoding *enc = parser->enc; + mb = ENC_CODERANGE_UNKNOWN; + rb_enc_mbcput(c, tokspace(rb_enc_codelen(c, enc)), enc); } else { + mb = ENC_CODERANGE_SINGLE; tokadd(c); } tokfix(); - set_yylval_str(STR_NEW(tok(), toklen())); + set_yylval_str(STR_NEW3(tok(), toklen(), mb)); lex_state = EXPR_ENDARG; return tCHAR; @@ -6865,9 +6881,9 @@ parser_yylex(struct parser_params *parser) break; } - mb = 0; + mb = ENC_CODERANGE_SINGLE; do { - if (!ISASCII(c)) mb = 1; + if (!ISASCII(c)) mb = ENC_CODERANGE_UNKNOWN; tokadd_mbchar(c); c = nextc(); } while (parser_is_identchar()); @@ -6920,7 +6936,7 @@ parser_yylex(struct parser_params *parser) } } - if (!mb && lex_state != EXPR_DOT) { + if (mb == ENC_CODERANGE_SINGLE && lex_state != EXPR_DOT) { const struct kwtable *kw; /* See if it is a reserved word. */ diff --git a/version.h b/version.h index 3843b67e39..c693516162 100644 --- a/version.h +++ b/version.h @@ -1,7 +1,7 @@ #define RUBY_VERSION "1.9.0" -#define RUBY_RELEASE_DATE "2007-10-23" +#define RUBY_RELEASE_DATE "2007-10-24" #define RUBY_VERSION_CODE 190 -#define RUBY_RELEASE_CODE 20071023 +#define RUBY_RELEASE_CODE 20071024 #define RUBY_PATCHLEVEL 0 #define RUBY_VERSION_MAJOR 1 @@ -9,7 +9,7 @@ #define RUBY_VERSION_TEENY 0 #define RUBY_RELEASE_YEAR 2007 #define RUBY_RELEASE_MONTH 10 -#define RUBY_RELEASE_DAY 23 +#define RUBY_RELEASE_DAY 24 #ifdef RUBY_EXTERN RUBY_EXTERN const char ruby_version[];