MDEV-16523 update RocksDB to 6.29fb

This commit is contained in:
Sergei Petrunia 2025-03-21 11:43:31 +02:00 committed by Sergei Golubchik
parent 75ad1e9f00
commit 1a044437a3
10 changed files with 110 additions and 67 deletions

View File

@ -176,35 +176,53 @@ set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
# - *_test.cc
# - *_bench.cc
set(ROCKSDB_SOURCES
cache/cache.cc
cache/cache_entry_roles.cc
cache/cache_key.cc
cache/cache_reservation_manager.cc
cache/clock_cache.cc
cache/lru_cache.cc
cache/sharded_cache.cc
db/arena_wrapped_db_iter.cc
db/blob/blob_fetcher.cc
db/blob/blob_file_addition.cc
db/blob/blob_file_builder.cc
db/blob/blob_file_builder.cc
db/blob/blob_file_cache.cc
db/blob/blob_file_garbage.cc
db/blob/blob_file_meta.cc
db/blob/blob_file_reader.cc
db/blob/blob_garbage_meter.cc
db/blob/blob_log_format.cc
db/blob/blob_log_sequential_reader.cc
db/blob/blob_log_writer.cc
db/blob/prefetch_buffer_collection.cc
db/builder.cc
db/c.cc
db/column_family.cc
db/compacted_db_impl.cc
db/compaction/compaction.cc
db/compaction/compaction_iterator.cc
db/compaction/compaction_picker.cc
db/compaction/compaction_job.cc
db/compaction/compaction_picker.cc
db/compaction/compaction_picker_fifo.cc
db/compaction/compaction_picker_level.cc
db/compaction/compaction_picker_universal.cc
db/compaction/sst_partitioner.cc
db/convenience.cc
db/db_filesnapshot.cc
db/dbformat.cc
db/db_impl/compacted_db_impl.cc
db/db_impl/db_impl.cc
db/db_impl/db_impl_write.cc
db/db_impl/db_impl_compaction_flush.cc
db/db_impl/db_impl_files.cc
db/db_impl/db_impl_open.cc
db/db_impl/db_impl_debug.cc
db/db_impl/db_impl_experimental.cc
db/db_impl/db_impl_files.cc
db/db_impl/db_impl_open.cc
db/db_impl/db_impl_readonly.cc
db/db_impl/db_impl_secondary.cc
db/db_impl/db_impl_write.cc
db/db_info_dumper.cc
db/db_iter.cc
db/dbformat.cc
db/error_handler.cc
db/event_helpers.cc
db/experimental.cc
@ -215,14 +233,16 @@ set(ROCKSDB_SOURCES
db/forward_iterator.cc
db/import_column_family_job.cc
db/internal_stats.cc
db/logs_with_prep_tracker.cc
db/log_reader.cc
db/logs_with_prep_tracker.cc
db/log_writer.cc
db/malloc_stats.cc
db/memtable.cc
db/memtable_list.cc
db/merge_helper.cc
db/merge_operator.cc
db/output_validator.cc
db/periodic_work_scheduler.cc
db/range_del_aggregator.cc
db/range_tombstone_fragmenter.cc
db/repair.cc
@ -233,25 +253,32 @@ set(ROCKSDB_SOURCES
db/trim_history_scheduler.cc
db/version_builder.cc
db/version_edit.cc
db/version_edit_handler.cc
db/version_set.cc
db/wal_edit.cc
db/wal_manager.cc
db/write_batch.cc
db/write_batch_base.cc
db/write_batch.cc
db/write_controller.cc
db/write_thread.cc
env/composite_env.cc
env/env.cc
env/env_chroot.cc
env/env_encryption.cc
env/env_hdfs.cc
env/file_system.cc
env/file_system_tracer.cc
env/fs_remap.cc
env/mock_env.cc
env/unique_id_gen.cc
file/delete_scheduler.cc
file/filename.cc
file/file_prefetch_buffer.cc
file/file_util.cc
file/filename.cc
file/line_file_reader.cc
file/random_access_file_reader.cc
file/read_write_util.cc
file/readahead_raf.cc
file/read_write_util.cc
file/sequence_file_reader.cc
file/sst_file_manager_impl.cc
file/writable_file_writer.cc
@ -281,29 +308,38 @@ set(ROCKSDB_SOURCES
monitoring/thread_status_util.cc
monitoring/thread_status_util_debug.cc
options/cf_options.cc
options/configurable.cc
options/customizable.cc
options/db_options.cc
options/options.cc
options/options_helper.cc
options/options_parser.cc
options/options_sanity_check.cc
port/stack_trace.cc
table/adaptive/adaptive_table_factory.cc
table/block_based/block.cc
table/block_based/binary_search_index_reader.cc
table/block_based/block_based_filter_block.cc
table/block_based/block_based_table_builder.cc
table/block_based/block_based_table_factory.cc
table/block_based/block_based_table_iterator.cc
table/block_based/block_based_table_reader.cc
table/block_based/block_builder.cc
table/block_based/block.cc
table/block_based/block_prefetcher.cc
table/block_based/block_prefix_index.cc
table/block_based/data_block_hash_index.cc
table/block_based/data_block_footer.cc
table/block_based/data_block_hash_index.cc
table/block_based/filter_block_reader_common.cc
table/block_based/filter_policy.cc
table/block_based/flush_block_policy.cc
table/block_based/full_filter_block.cc
table/block_based/hash_index_reader.cc
table/block_based/index_builder.cc
table/block_based/index_reader_common.cc
table/block_based/parsed_full_filter_block.cc
table/block_based/partitioned_filter_block.cc
table/block_based/partitioned_index_iterator.cc
table/block_based/partitioned_index_reader.cc
table/block_based/reader_common.cc
table/block_based/uncompression_dict_reader.cc
table/block_fetcher.cc
table/cuckoo/cuckoo_table_builder.cc
@ -321,10 +357,13 @@ set(ROCKSDB_SOURCES
table/plain/plain_table_index.cc
table/plain/plain_table_key_coding.cc
table/plain/plain_table_reader.cc
table/sst_file_dumper.cc
table/sst_file_reader.cc
table/sst_file_writer.cc
table/table_factory.cc
table/table_properties.cc
table/two_level_iterator.cc
table/unique_id.cc
test_util/sync_point.cc
test_util/sync_point_impl.cc
test_util/testutil.cc
@ -335,8 +374,12 @@ set(ROCKSDB_SOURCES
tools/ldb_tool.cc
tools/sst_dump_tool.cc
tools/trace_analyzer_tool.cc
trace_replay/trace_replay.cc
trace_replay/block_cache_tracer.cc
trace_replay/io_tracer.cc
trace_replay/trace_record.cc
trace_replay/trace_record_handler.cc
trace_replay/trace_record_result.cc
trace_replay/trace_replay.cc
util/coding.cc
util/compaction_job_stats_impl.cc
util/comparator.cc
@ -344,17 +387,8 @@ set(ROCKSDB_SOURCES
util/concurrent_task_limiter_impl.cc
util/crc32c.cc
util/dynamic_bloom.cc
util/hash.cc
util/murmurhash.cc
util/random.cc
util/rate_limiter.cc
util/slice.cc
util/file_checksum_helper.cc
util/status.cc
util/string_util.cc
util/thread_local.cc
util/threadpool_imp.cc
util/xxhash.cc
util/hash.cc
utilities/backupable/backupable_db.cc
utilities/blob_db/blob_compaction_filter.cc
utilities/blob_db/blob_db.cc
@ -362,10 +396,8 @@ set(ROCKSDB_SOURCES
utilities/blob_db/blob_db_impl_filesnapshot.cc
utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc
utilities/blob_db/blob_log_reader.cc
utilities/blob_db/blob_log_writer.cc
utilities/blob_db/blob_log_format.cc
utilities/checkpoint/checkpoint_impl.cc
utilities/compaction_filters.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/debug.cc
utilities/env_mirror.cc
@ -373,11 +405,12 @@ set(ROCKSDB_SOURCES
utilities/leveldb_options/leveldb_options.cc
utilities/memory/memory_util.cc
utilities/merge_operators/bytesxor.cc
utilities/merge_operators.cc
utilities/merge_operators/max.cc
utilities/merge_operators/put.cc
utilities/merge_operators/sortlist.cc
utilities/merge_operators/string_append/stringappend.cc
utilities/merge_operators/string_append/stringappend2.cc
utilities/merge_operators/string_append/stringappend.cc
utilities/merge_operators/uint64add.cc
utilities/object_registry.cc
utilities/option_change_migration/option_change_migration.cc
@ -391,22 +424,37 @@ set(ROCKSDB_SOURCES
utilities/simulator_cache/sim_cache.cc
utilities/table_properties_collectors/compact_on_deletion_collector.cc
utilities/trace/file_trace_reader_writer.cc
utilities/transactions/optimistic_transaction_db_impl.cc
utilities/trace/replayer_impl.cc
utilities/transactions/lock/lock_manager.cc
utilities/transactions/lock/point/point_lock_manager.cc
utilities/transactions/lock/point/point_lock_tracker.cc
utilities/transactions/optimistic_transaction.cc
utilities/transactions/optimistic_transaction_db_impl.cc
utilities/transactions/pessimistic_transaction.cc
utilities/transactions/pessimistic_transaction_db.cc
utilities/transactions/snapshot_checker.cc
utilities/transactions/transaction_base.cc
utilities/transactions/transaction_db_mutex_impl.cc
utilities/transactions/transaction_lock_mgr.cc
utilities/transactions/transaction_util.cc
utilities/transactions/write_prepared_txn.cc
utilities/transactions/write_prepared_txn_db.cc
utilities/transactions/write_unprepared_txn.cc
utilities/transactions/write_unprepared_txn_db.cc
utilities/ttl/db_ttl_impl.cc
utilities/wal_filter.cc
utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc
util/murmurhash.cc
util/random.cc
util/rate_limiter.cc
util/regex.cc
util/ribbon_config.cc
util/slice.cc
util/status.cc
util/string_util.cc
util/thread_local.cc
util/threadpool_imp.cc
util/xxhash.cc
)
@ -484,8 +532,10 @@ IF(CMAKE_VERSION VERSION_GREATER "2.8.10")
STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S")
ENDIF()
# psergey-added:
SET(GIT_MOD 0)
CONFIGURE_FILE(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in build_version.cc @ONLY)
INCLUDE_DIRECTORIES(${ROCKSDB_SOURCE_DIR}/util)
list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc)
ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES})

View File

@ -1596,7 +1596,7 @@ static MYSQL_SYSVAR_BOOL(
"BlockBasedTableOptions::no_block_cache for RocksDB", nullptr, nullptr,
rocksdb_tbl_options->no_block_cache);
static MYSQL_SYSVAR_SIZE_T(block_size, rocksdb_tbl_options->block_size,
static MYSQL_SYSVAR_UINT64_T(block_size, rocksdb_tbl_options->block_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"BlockBasedTableOptions::block_size for RocksDB",
nullptr, nullptr, rocksdb_tbl_options->block_size,
@ -3992,7 +3992,7 @@ static int rocksdb_commit_by_xid(handlerton *const hton, XID *const xid) {
DBUG_ASSERT(xid != nullptr);
DBUG_ASSERT(commit_latency_stats != nullptr);
rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true);
rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true);
const auto name = rdb_xid_to_string(*xid);
DBUG_ASSERT(!name.empty());
@ -4187,7 +4187,7 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx)
DBUG_ASSERT(thd != nullptr);
DBUG_ASSERT(commit_latency_stats != nullptr);
rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true);
rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true);
/* note: h->external_lock(F_UNLCK) is called after this function is called) */
Rdb_transaction *tx = get_tx_from_thd(thd);
@ -4732,8 +4732,7 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd,
if (tf_name.find("BlockBasedTable") != std::string::npos) {
const rocksdb::BlockBasedTableOptions *const bbt_opt =
reinterpret_cast<rocksdb::BlockBasedTableOptions *>(
table_factory->GetOptions());
table_factory->GetOptions<rocksdb::BlockBasedTableOptions>();
if (bbt_opt != nullptr) {
if (bbt_opt->block_cache.get() != nullptr) {

View File

@ -20,7 +20,7 @@ set @tmp1=@@rocksdb_verify_row_debug_checksums;
set rocksdb_verify_row_debug_checksums=1;
set session debug_dbug= "+d,myrocks_simulate_bad_row_read1";
select * from t1 where pk=1;
ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
set session debug_dbug= "-d,myrocks_simulate_bad_row_read1";
set rocksdb_verify_row_debug_checksums=@tmp1;
select * from t1 where pk=1;
@ -28,11 +28,11 @@ pk col1
1 1
set session debug_dbug= "+d,myrocks_simulate_bad_row_read2";
select * from t1 where pk=1;
ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
set session debug_dbug= "-d,myrocks_simulate_bad_row_read2";
set session debug_dbug= "+d,myrocks_simulate_bad_row_read3";
select * from t1 where pk=1;
ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
set session debug_dbug= "-d,myrocks_simulate_bad_row_read3";
insert into t1 values(4,'0123456789');
select * from t1;
@ -56,7 +56,7 @@ pk col1
ABCD 1
set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1";
select * from t2;
ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1";
drop table t2;
create table t2 (
@ -69,6 +69,6 @@ pk col1
ABCD 1
set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1";
select * from t2;
ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB
ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB
set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1";
drop table t2;

View File

@ -1,12 +1,6 @@
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
DROP TABLE IF EXISTS t1;
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
# restart
CREATE TABLE t1 (
a int not null,
b int not null,
@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1',
key (b) comment 'rev:cf2'
) ENGINE=RocksDB;
DELETE FROM t1;
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
# restart
select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
drop table t1;
select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';

View File

@ -982,7 +982,7 @@ rocksdb_skip_fill_cache OFF
rocksdb_skip_unique_check_tables .*
rocksdb_sst_mgr_rate_bytes_per_sec 0
rocksdb_stats_dump_period_sec 600
rocksdb_stats_level 0
rocksdb_stats_level 1
rocksdb_stats_recalc_rate 0
rocksdb_store_row_debug_checksums OFF
rocksdb_strict_collation_check OFF

View File

@ -1,2 +1,2 @@
Check for MANIFEST files
MANIFEST-000006
MANIFEST-000004

View File

@ -1,12 +1,6 @@
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
DROP TABLE IF EXISTS t1;
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
# restart
CREATE TABLE t1 (
a int not null,
b int not null,
@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1',
key (b) comment 'rev:cf2'
) ENGINE=RocksDB;
DELETE FROM t1;
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
# restart
select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
truncate table t1;
select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';

View File

@ -1,8 +1,5 @@
--source include/have_rocksdb.inc
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
--disable_warnings
DROP TABLE IF EXISTS t1;
--enable_warnings
@ -10,11 +7,6 @@ DROP TABLE IF EXISTS t1;
call mtr.add_suppression("Column family 'cf1' not found");
call mtr.add_suppression("Column family 'rev:cf2' not found");
# Start from clean slate
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
--source include/restart_mysqld.inc
CREATE TABLE t1 (
a int not null,
@ -29,6 +21,12 @@ let $max = 50000;
let $table = t1;
--source drop_table3_repopulate_table.inc
# Start from clean slate
set global rocksdb_compact_cf = 'cf1';
set global rocksdb_compact_cf = 'rev:cf2';
set global rocksdb_signal_drop_index_thread = 1;
--source include/restart_mysqld.inc
--disable_cursor_protocol
select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
--enable_cursor_protocol
@ -49,6 +47,7 @@ let $wait_condition = select count(*) = 0
--source include/wait_condition.inc
select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
#select variable_value-@a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes';
# Cleanup
DROP TABLE IF EXISTS t1;

View File

@ -587,8 +587,7 @@ static int rdb_i_s_cfoptions_fill_table(
cf_option_types.push_back(
{"PREFIX_EXTRACTOR", opts.prefix_extractor == nullptr
? "NULL"
: std::string(opts.prefix_extractor->Name())});
: std::string(opts.prefix_extractor->AsString())});
// get COMPACTION_STYLE option
switch (opts.compaction_style) {
case rocksdb::kCompactionStyleLevel:
@ -646,7 +645,7 @@ static int rdb_i_s_cfoptions_fill_table(
// get table related options
std::vector<std::string> table_options =
split_into_vector(opts.table_factory->GetPrintableTableOptions(), '\n');
split_into_vector(opts.table_factory->GetPrintableOptions(), '\n');
for (auto option : table_options) {
option.erase(std::remove(option.begin(), option.end(), ' '),

@ -1 +1 @@
Subproject commit bba5e7bc21093d7cfa765e1280a7c4fdcd284288
Subproject commit 79f08d7ffa6d34d9ca3357777bcb335884a56cfb