From 284593413f73cbaf4723f1e230bc7eb178774ba3 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 25 Oct 2024 15:33:57 +0300 Subject: [PATCH] MDEV-35253: xa_prepare_unlock_unmodified fails: shift exponent 32 is too large The code in best_access_path() uses PREV_BITS(uint, N) to compute a bitmap of all keyparts: {keypart0, ... keypart{N-1}). The problem is that PREV_BITS($type, N) macro code can't handle the case when N=key == key && keyuse->table == table); - if (bound_parts == PREV_BITS(uint, keyinfo->user_defined_key_parts)) + if (bound_parts == PREV_BITS(key_part_map, + keyinfo->user_defined_key_parts)) return TRUE; } else diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 37ec152141c..a9852b81063 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -7797,7 +7797,7 @@ best_access_path(JOIN *join, loose_scan_opt.check_ref_access_part1(s, key, start_key, found_part); /* Check if we found full key */ - const key_part_map all_key_parts= PREV_BITS(uint, key_parts); + const key_part_map all_key_parts= PREV_BITS(key_part_map, key_parts); if (found_part == all_key_parts && !ref_or_null_part) { /* use eq key */ max_key_part= (uint) ~0; @@ -7935,7 +7935,8 @@ best_access_path(JOIN *join, */ if ((found_part & 1) && (!(table->file->index_flags(key, 0, 0) & HA_ONLY_WHOLE_INDEX) || - found_part == PREV_BITS(uint,keyinfo->user_defined_key_parts))) + found_part == PREV_BITS(key_part_map, + keyinfo->user_defined_key_parts))) { max_key_part= max_part_bit(found_part); /* @@ -23636,7 +23637,7 @@ static int test_if_order_by_key(JOIN *join, if (have_pk_suffix && reverse == 0 && // all were =const so far key_parts == table->key_info[idx].ext_key_parts && - table->const_key_parts[pk] == PREV_BITS(uint, + table->const_key_parts[pk] == PREV_BITS(key_part_map, table->key_info[pk]. user_defined_key_parts)) { diff --git a/sql/sql_select.h b/sql/sql_select.h index 86947fc0bc6..0214c290ff7 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2598,4 +2598,15 @@ void propagate_new_equalities(THD *thd, Item *cond, COND_EQUAL *inherited, bool *is_simplifiable_cond); +template T prev_bits(T n_bits) +{ + if (!n_bits) + return 0; + T tmp= ((T)1 << (n_bits - 1)); + return (tmp - 1) | tmp; +} +// A wrapper for the above function: +#define PREV_BITS(type, A) prev_bits(A) + + #endif /* SQL_SELECT_INCLUDED */