MDEV-30662 SQL/PL package body does not appear in I_S.ROUTINES.ROUTINE_DEFINITION

- Moving the code from a public function trim_whitespaces()
  to the class Lex_cstring as methods. This code may
  be useful in other contexts, and also this code becomes
  visible inside sql_class.h

- Adding a helper method THD::strmake_lex_cstring_trim_whitespaces()

- Unifying the way how CREATE PROCEDURE/CREATE FUNCTION and
  CREATE PACKAGE/CREATE PACKAGE BODY work:

  a) Now CREATE PACKAGE/CREATE PACKAGE BODY also calls
  Lex->sphead->set_body_start() to remember the cpp body start inside
  an sp_head member.

  b) adding a "const char *cpp_body_end" parameter to
  sp_head::set_stmt_end().

  These changes made it possible to reuse sp_head::set_stmt_end() inside
  LEX::create_package_finalize() and remove the duplucate code.

- Renaming sp_head::m_body_begin to m_cpp_body_begin and adding a comment
  to make it clear that this member is used only during parsing, and
  points to a fragment inside the cpp buffer.

- Changed sp_head::set_body_start() and sp_head::set_stmt_end()
  to skip the calls related to "body_utf8" in cases when m_parent is not NULL.
  A non-NULL m_parent means that we're inside a package routine.
  "body_utf8" in such case belongs not to the current sphead itself,
  but to parent (the package) sphead.
  So an sphead instance of a package routine should neither initialize,
  nor finalize, nor change in any other ways the "body_utf8" related
  members of Lex_input_stream, and should not take over or copy "body_utf8"
  data from Lex_input_stream to "this".

mysql-test/suite/compat/oracle/r/rpl_sp_package.result

@@ -43,7 +43,10 @@ comment
 character_set_client	latin1
 collation_connection	latin1_swedish_ci
 db_collation	latin1_swedish_ci
-body_utf8	
+body_utf8	AS
+FUNCTION f1 RETURN INT;
+PROCEDURE p1;
+END
 aggregate	NONE
 db	test
 name	pack
@@ -73,7 +76,16 @@ comment
 character_set_client	latin1
 collation_connection	latin1_swedish_ci
 db_collation	latin1_swedish_ci
-body_utf8	
+body_utf8	AS
+FUNCTION f1 RETURN INT AS
+BEGIN
+RETURN 10;
+END;
+PROCEDURE p1 AS
+BEGIN
+SELECT f1();
+END;
+END
 aggregate	NONE
 SELECT * FROM mysql.proc WHERE db='test' AND name LIKE 'pack.%';
 SET @@sql_mode=ORACLE;

mysql-test/suite/compat/oracle/r/sp-package-i_s.result

@@ -0,0 +1,75 @@
+#
+# Start of 10.5 tests
+#
+#
+#  MDEV-30662 SQL/PL package body does not appear in I_S.ROUTINES.ROUTINE_DEFINITION
+#
+SET sql_mode=ORACLE;
+CREATE OR REPLACE PACKAGE pkg1 AS
+FUNCTION f1() RETURN INT;
+END;
+$$
+CREATE PACKAGE BODY pkg1 AS
+FUNCTION f1() RETURN INT AS
+BEGIN
+RETURN 1;
+END;
+END;
+$$
+SELECT routine_name, routine_type, routine_definition
+FROM information_schema.routines
+WHERE routine_type LIKE 'PACKAGE%'
+ORDER BY routine_type;
+routine_name	pkg1
+routine_type	PACKAGE
+routine_definition	AS
+FUNCTION f1() RETURN INT;
+END
+routine_name	pkg1
+routine_type	PACKAGE BODY
+routine_definition	AS
+FUNCTION f1() RETURN INT AS
+BEGIN
+RETURN 1;
+END;
+END
+DROP PACKAGE pkg1;
+SET sql_mode=ORACLE;
+CREATE OR REPLACE PACKAGE pkg1 AS
+FUNCTION f1() RETURN INT;
+END;
+$$
+CREATE PACKAGE BODY pkg1 AS
+FUNCTION f1() RETURN INT AS
+BEGIN
+RETURN 1;
+END;
+BEGIN
+SET @a=10;
+SET @a=f1();
+END;
+$$
+SELECT routine_name, routine_type, routine_definition
+FROM information_schema.routines
+WHERE routine_type LIKE 'PACKAGE%'
+ORDER BY routine_type;
+routine_name	pkg1
+routine_type	PACKAGE
+routine_definition	AS
+FUNCTION f1() RETURN INT;
+END
+routine_name	pkg1
+routine_type	PACKAGE BODY
+routine_definition	AS
+FUNCTION f1() RETURN INT AS
+BEGIN
+RETURN 1;
+END;
+BEGIN
+SET @a=10;
+SET @a=f1();
+END
+DROP PACKAGE pkg1;
+#
+# End of 10.5 tests
+#

mysql-test/suite/compat/oracle/r/sp-package.result

@@ -537,7 +537,13 @@ comment	package-test2-comment
 character_set_client	latin1
 collation_connection	latin1_swedish_ci
 db_collation	latin1_swedish_ci
-body_utf8	
+body_utf8	AS
+FUNCTION f1 RETURN INT DETERMINISTIC;
+FUNCTION f2(a INT) RETURN INT;
+FUNCTION concat RETURN INT;
+PROCEDURE p1;
+PROCEDURE p2(a INT);
+END
 aggregate	NONE
 SELECT * FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA='test' AND ROUTINE_NAME='test2';
 SPECIFIC_NAME	test2
@@ -555,7 +561,13 @@ CHARACTER_SET_NAME	NULL
 COLLATION_NAME	NULL
 DTD_IDENTIFIER	NULL
 ROUTINE_BODY	SQL
-ROUTINE_DEFINITION	
+ROUTINE_DEFINITION	AS
+FUNCTION f1 RETURN INT DETERMINISTIC;
+FUNCTION f2(a INT) RETURN INT;
+FUNCTION concat RETURN INT;
+PROCEDURE p1;
+PROCEDURE p2(a INT);
+END
 EXTERNAL_NAME	NULL
 EXTERNAL_LANGUAGE	NULL
 PARAMETER_STYLE	SQL
@@ -641,7 +653,13 @@ CHARACTER_SET_NAME	NULL
 COLLATION_NAME	NULL
 DTD_IDENTIFIER	NULL
 ROUTINE_BODY	SQL
-ROUTINE_DEFINITION	
+ROUTINE_DEFINITION	AS
+FUNCTION f1 RETURN INT DETERMINISTIC;
+FUNCTION f2(a INT) RETURN INT;
+FUNCTION concat RETURN INT;
+PROCEDURE p1;
+PROCEDURE p2(a INT);
+END
 EXTERNAL_NAME	NULL
 EXTERNAL_LANGUAGE	NULL
 PARAMETER_STYLE	SQL
@@ -672,7 +690,19 @@ CHARACTER_SET_NAME	NULL
 COLLATION_NAME	NULL
 DTD_IDENTIFIER	NULL
 ROUTINE_BODY	SQL
-ROUTINE_DEFINITION	
+ROUTINE_DEFINITION	AS
+FUNCTION f1 RETURN INT AS BEGIN RETURN 10; END;
+FUNCTION f2(a INT) RETURN INT AS BEGIN RETURN f1()+a; END;
+FUNCTION concat RETURN INT AS BEGIN RETURN 1; END;
+PROCEDURE p1 AS
+BEGIN
+SELECT f2(0);
+END;
+PROCEDURE p2(a INT) AS
+BEGIN
+SELECT f2(a);
+END;
+END
 EXTERNAL_NAME	NULL
 EXTERNAL_LANGUAGE	NULL
 PARAMETER_STYLE	SQL

mysql-test/suite/compat/oracle/t/sp-package-i_s.test

@@ -0,0 +1,69 @@
+--source include/default_charset.inc
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+
+--echo #
+--echo #  MDEV-30662 SQL/PL package body does not appear in I_S.ROUTINES.ROUTINE_DEFINITION
+--echo #
+
+# Testing a package without the executable section
+
+SET sql_mode=ORACLE;
+DELIMITER $$;
+CREATE OR REPLACE PACKAGE pkg1 AS
+  FUNCTION f1() RETURN INT;
+END;
+$$
+CREATE PACKAGE BODY pkg1 AS
+  FUNCTION f1() RETURN INT AS
+  BEGIN
+    RETURN 1;
+  END;
+END;
+$$
+DELIMITER ;$$
+
+--vertical_results
+SELECT routine_name, routine_type, routine_definition
+FROM information_schema.routines
+WHERE routine_type LIKE 'PACKAGE%'
+ORDER BY routine_type;
+--horizontal_results
+
+DROP PACKAGE pkg1;
+
+# Testing a package with the executable section
+
+SET sql_mode=ORACLE;
+DELIMITER $$;
+CREATE OR REPLACE PACKAGE pkg1 AS
+  FUNCTION f1() RETURN INT;
+END;
+$$
+CREATE PACKAGE BODY pkg1 AS
+  FUNCTION f1() RETURN INT AS
+  BEGIN
+    RETURN 1;
+  END;
+BEGIN
+  SET @a=10;
+  SET @a=f1();
+END;
+$$
+DELIMITER ;$$
+
+--vertical_results
+SELECT routine_name, routine_type, routine_definition
+FROM information_schema.routines
+WHERE routine_type LIKE 'PACKAGE%'
+ORDER BY routine_type;
+--horizontal_results
+
+DROP PACKAGE pkg1;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

sql/lex_string.h

@@ -50,6 +50,60 @@ class Lex_cstring : public LEX_CSTRING
     str= _str;
     length= _len;
   }
+
+  /*
+    Trim left white spaces.
+    Assumes that there are no multi-bytes characters
+    that can be considered white-space.
+  */
+  Lex_cstring ltrim_whitespace(CHARSET_INFO *cs) const
+  {
+    DBUG_ASSERT(cs->mbminlen == 1);
+    Lex_cstring str= *this;
+    while (str.length > 0 && my_isspace(cs, str.str[0]))
+    {
+      str.length--;
+      str.str++;
+    }
+    return str;
+  }
+
+  /*
+    Trim right white spaces.
+    Assumes that there are no multi-bytes characters
+    that can be considered white-space.
+    Also, assumes that the character set supports backward space parsing.
+  */
+  Lex_cstring rtrim_whitespace(CHARSET_INFO *cs) const
+  {
+    DBUG_ASSERT(cs->mbminlen == 1);
+    Lex_cstring str= *this;
+    while (str.length > 0 && my_isspace(cs, str.str[str.length - 1]))
+    {
+      str.length --;
+    }
+    return str;
+  }
+
+  /*
+    Trim all spaces.
+  */
+  Lex_cstring trim_whitespace(CHARSET_INFO *cs) const
+  {
+    return ltrim_whitespace(cs).rtrim_whitespace(cs);
+  }
+
+  /*
+    Trim all spaces and return the length of the leading space sequence.
+  */
+  Lex_cstring trim_whitespace(CHARSET_INFO *cs, size_t *prefix_length) const
+  {
+    Lex_cstring tmp= Lex_cstring(*this).ltrim_whitespace(cs);
+    if (prefix_length)
+      *prefix_length= tmp.str - str;
+    return tmp.rtrim_whitespace(cs);
+  }
+
 };
 
 

sql/sp_head.cc

@@ -558,7 +558,7 @@ sp_head::sp_head(MEM_ROOT *mem_root_arg, sp_package *parent,
    m_next_cached_sp(0),
    m_param_begin(NULL),
    m_param_end(NULL),
-   m_body_begin(NULL),
+   m_cpp_body_begin(NULL),
    m_thd_root(NULL),
    m_thd(NULL),
    m_pcont(new (&main_mem_root) sp_pcontext()),
@@ -819,18 +819,18 @@ sp_head::init_psi_share()
 
 
 void
-sp_head::set_body_start(THD *thd, const char *begin_ptr)
+sp_head::set_body_start(THD *thd, const char *cpp_body_start)
 {
-  m_body_begin= begin_ptr;
+  m_cpp_body_begin= cpp_body_start;
-  thd->m_parser_state->m_lip.body_utf8_start(thd, begin_ptr);
+  if (!m_parent)
+    thd->m_parser_state->m_lip.body_utf8_start(thd, cpp_body_start);
 }
 
 
 void
-sp_head::set_stmt_end(THD *thd)
+sp_head::set_stmt_end(THD *thd, const char *cpp_body_end)
 {
   Lex_input_stream *lip= & thd->m_parser_state->m_lip; /* shortcut */
-  const char *end_ptr= lip->get_cpp_tok_start(); /* shortcut */
 
   /* Make the string of parameters. */
 
@@ -842,30 +842,27 @@ sp_head::set_stmt_end(THD *thd)
 
   /* Remember end pointer for further dumping of whole statement. */
 
-  thd->lex->stmt_definition_end= end_ptr;
+  thd->lex->stmt_definition_end= cpp_body_end;
 
   /* Make the string of body (in the original character set). */
 
-  m_body.length= end_ptr - m_body_begin;
+  m_body= thd->strmake_lex_cstring_trim_whitespace(
-  m_body.str= thd->strmake(m_body_begin, m_body.length);
+                 Lex_cstring(m_cpp_body_begin, cpp_body_end));
-  trim_whitespace(thd->charset(), &m_body);
 
   /* Make the string of UTF-body. */
 
-  lip->body_utf8_append(end_ptr);
+  lip->body_utf8_append(cpp_body_end);
 
-  m_body_utf8.length= lip->get_body_utf8_length();
+  if (!m_parent)
-  m_body_utf8.str= thd->strmake(lip->get_body_utf8_str(), m_body_utf8.length);
+    m_body_utf8= thd->strmake_lex_cstring_trim_whitespace(lip->body_utf8());
-  trim_whitespace(thd->charset(), &m_body_utf8);
 
   /*
     Make the string of whole stored-program-definition query (in the
     original character set).
   */
 
-  m_defstr.length= end_ptr - lip->get_cpp_buf();
+  m_defstr= thd->strmake_lex_cstring_trim_whitespace(
-  m_defstr.str= thd->strmake(lip->get_cpp_buf(), m_defstr.length);
+                   Lex_cstring(lip->get_cpp_buf(), cpp_body_end));
-  trim_whitespace(thd->charset(), &m_defstr);
 }
 
 

sql/sp_head.h

@@ -314,7 +314,13 @@ public:
   const char *m_param_end;
 
 private:
-  const char *m_body_begin;
+  /*
+    A pointer to the body start inside the cpp buffer.
+    Used only during parsing. Should be removed eventually.
+    The affected functions/methods should be fixed to get the cpp body start
+    as a parameter, rather than through this member.
+  */
+  const char *m_cpp_body_begin;
 
 public:
   /*
@@ -351,12 +357,11 @@ public:
 
   /** Set the body-definition start position. */
   void
-  set_body_start(THD *thd, const char *begin_ptr);
+  set_body_start(THD *thd, const char *cpp_body_start);
 
   /** Set the statement-definition (body-definition) end position. */
   void
-  set_stmt_end(THD *thd);
+  set_stmt_end(THD *thd, const char *cpp_body_end);
-
 
   bool
   execute_trigger(THD *thd,

sql/sql_class.h

@@ -3913,6 +3913,10 @@ public:
   {
     return strmake_lex_cstring(from.str, from.length);
   }
+  LEX_CSTRING strmake_lex_cstring_trim_whitespace(const LEX_CSTRING &from)
+  {
+    return strmake_lex_cstring(Lex_cstring(from).trim_whitespace(charset()));
+  }
 
   LEX_STRING *make_lex_string(LEX_STRING *lex_str, const char* str, size_t length)
   {

sql/sql_lex.cc

@@ -2860,34 +2860,6 @@ int Lex_input_stream::scan_ident_delimited(THD *thd,
 }
 
 
-void trim_whitespace(CHARSET_INFO *cs, LEX_CSTRING *str, size_t * prefix_length)
-{
-  /*
-    TODO:
-    This code assumes that there are no multi-bytes characters
-    that can be considered white-space.
-  */
-
-  size_t plen= 0;
-  while ((str->length > 0) && (my_isspace(cs, str->str[0])))
-  {
-    plen++;
-    str->length --;
-    str->str ++;
-  }
-  if (prefix_length)
-    *prefix_length= plen;
-  /*
-    FIXME:
-    Also, parsing backward is not safe with multi bytes characters
-  */
-  while ((str->length > 0) && (my_isspace(cs, str->str[str->length-1])))
-  {
-    str->length --;
-  }
-}
-
-
 /*
   st_select_lex structures initialisations
 */
@@ -7336,7 +7308,7 @@ bool LEX::sp_body_finalize_routine(THD *thd)
 {
   if (sphead->check_unresolved_goto())
     return true;
-  sphead->set_stmt_end(thd);
+  sphead->set_stmt_end(thd, thd->m_parser_state->m_lip.get_cpp_tok_start());
   sphead->restore_thd_mem_root(thd);
   return false;
 }
@@ -9268,8 +9240,7 @@ sp_package *LEX::create_package_start(THD *thd,
 bool LEX::create_package_finalize(THD *thd,
                                   const sp_name *name,
                                   const sp_name *name2,
-                                  const char *body_start,
+                                  const char *cpp_body_end)
-                                  const char *body_end)
 {
   if (name2 &&
       (name2->m_explicit_name != name->m_explicit_name ||
@@ -9282,18 +9253,8 @@ bool LEX::create_package_finalize(THD *thd,
              exp ? ErrConvDQName(name).ptr() : name->m_name.str);
     return true;
   }
-  // TODO: reuse code in LEX::create_package_finalize and sp_head::set_stmt_end
-  sphead->m_body.length= body_end - body_start;
-  if (unlikely(!(sphead->m_body.str= thd->strmake(body_start,
-                                                  sphead->m_body.length))))
-    return true;
-
-  size_t not_used;
-  Lex_input_stream *lip= & thd->m_parser_state->m_lip;
-  sphead->m_defstr.length= lip->get_cpp_ptr() - lip->get_cpp_buf();
-  sphead->m_defstr.str= thd->strmake(lip->get_cpp_buf(), sphead->m_defstr.length);
-  trim_whitespace(thd->charset(), &sphead->m_defstr, &not_used);
 
+  sphead->set_stmt_end(thd, cpp_body_end);
   sphead->restore_thd_mem_root(thd);
   sp_package *pkg= sphead->get_package();
   DBUG_ASSERT(pkg);

sql/sql_lex.h

@@ -2716,15 +2716,9 @@ public:
     return p;
   }
   /** Get the utf8-body string. */
-  const char *get_body_utf8_str() const
+  LEX_CSTRING body_utf8() const
   {
-    return m_body_utf8;
+    return LEX_CSTRING({m_body_utf8, (size_t) (m_body_utf8_ptr - m_body_utf8)});
-  }
-
-  /** Get the utf8-body length. */
-  size_t get_body_utf8_length() const
-  {
-    return (size_t) (m_body_utf8_ptr - m_body_utf8);
   }
 
   void body_utf8_start(THD *thd, const char *begin_ptr);
@@ -3855,8 +3849,7 @@ public:
   bool create_package_finalize(THD *thd,
                                const sp_name *name,
                                const sp_name *name2,
-                               const char *body_start,
+                               const char *cpp_body_end);
-                               const char *body_end);
   bool call_statement_start(THD *thd, sp_name *name);
   bool call_statement_start(THD *thd, const Lex_ident_sys_st *name);
   bool call_statement_start(THD *thd, const Lex_ident_sys_st *name1,
@@ -5096,7 +5089,12 @@ int init_lex_with_single_table(THD *thd, TABLE *table, LEX *lex);
 extern int MYSQLlex(union YYSTYPE *yylval, THD *thd);
 extern int ORAlex(union YYSTYPE *yylval, THD *thd);
 
-extern void trim_whitespace(CHARSET_INFO *cs, LEX_CSTRING *str, size_t * prefix_length = 0);
+inline void trim_whitespace(CHARSET_INFO *cs, LEX_CSTRING *str,
+                            size_t * prefix_length = 0)
+{
+  *str= Lex_cstring(*str).trim_whitespace(cs, prefix_length);
+}
+
 
 extern bool is_lex_native_function(const LEX_CSTRING *name); 
 extern bool is_native_function(THD *thd, const LEX_CSTRING *name);

sql/sql_yacc.yy

@@ -19118,8 +19118,6 @@ create_routine:
           }
         | create_or_replace definer_opt PACKAGE_ORACLE_SYM
           opt_if_not_exists sp_name opt_create_package_chistics_init
-          sp_tail_is
-          remember_name
           {
             sp_package *pkg;
             if (unlikely(!(pkg= Lex->
@@ -19129,17 +19127,17 @@ create_routine:
                                                 $5, $1 | $4))))
               MYSQL_YYABORT;
             pkg->set_c_chistics(Lex->sp_chistics);
+            Lex->sphead->set_body_start(thd, YYLIP->get_cpp_tok_start());
           }
+          sp_tail_is
           opt_package_specification_element_list END
           remember_end_opt opt_sp_name
           {
-            if (unlikely(Lex->create_package_finalize(thd, $5, $13, $8, $12)))
+            if (unlikely(Lex->create_package_finalize(thd, $5, $12, $11)))
               MYSQL_YYABORT;
           }
         | create_or_replace definer_opt PACKAGE_ORACLE_SYM BODY_ORACLE_SYM
           opt_if_not_exists sp_name opt_create_package_chistics_init
-          sp_tail_is
-          remember_name
           {
             sp_package *pkg;
             if (unlikely(!(pkg= Lex->
@@ -19149,8 +19147,10 @@ create_routine:
                                                 $6, $1 | $5))))
               MYSQL_YYABORT;
             pkg->set_c_chistics(Lex->sp_chistics);
+            Lex->sphead->set_body_start(thd, YYLIP->get_cpp_tok_start());
             Lex->sp_block_init(thd);
           }
+          sp_tail_is
           package_implementation_declare_section
           {
             if (unlikely(Lex->sp_block_with_exceptions_finalize_declarations(thd)))
@@ -19158,13 +19158,13 @@ create_routine:
           }
           package_implementation_executable_section
           {
-            $11.hndlrs+= $13.hndlrs;
+            $10.hndlrs+= $12.hndlrs;
-            if (unlikely(Lex->sp_block_finalize(thd, $11)))
+            if (unlikely(Lex->sp_block_finalize(thd, $10)))
               MYSQL_YYABORT;
           }
           remember_end_opt opt_sp_name
           {
-            if (unlikely(Lex->create_package_finalize(thd, $6, $16, $9, $15)))
+            if (unlikely(Lex->create_package_finalize(thd, $6, $15, $14)))
               MYSQL_YYABORT;
           }
         ;