From 2cff674c1d11e07d2110972154d4b5ce9327035e Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Sat, 17 Feb 2007 23:52:17 +0100 Subject: [PATCH 1/5] ndb - bug#26457 master failure during master take over --- ndb/src/kernel/blocks/ERROR_codes.txt | 4 +- ndb/src/kernel/blocks/dbdih/DbdihMain.cpp | 27 +++++++++- ndb/test/include/NdbRestarter.hpp | 2 + ndb/test/ndbapi/testNodeRestart.cpp | 42 +++++++++++++++ ndb/test/run-test/daily-basic-tests.txt | 4 ++ ndb/test/src/NdbRestarter.cpp | 62 +++++++++++++++++++++++ 6 files changed, 139 insertions(+), 2 deletions(-) diff --git a/ndb/src/kernel/blocks/ERROR_codes.txt b/ndb/src/kernel/blocks/ERROR_codes.txt index 0bcc99a6334..f7cb49014cb 100644 --- a/ndb/src/kernel/blocks/ERROR_codes.txt +++ b/ndb/src/kernel/blocks/ERROR_codes.txt @@ -5,7 +5,7 @@ Next DBACC 3002 Next DBTUP 4014 Next DBLQH 5043 Next DBDICT 6007 -Next DBDIH 7178 +Next DBDIH 7181 Next DBTC 8039 Next CMVMI 9000 Next BACKUP 10022 @@ -71,6 +71,8 @@ Delay GCP_SAVEREQ by 10 secs 7177: Delay copying of sysfileData in execCOPY_GCIREQ +7180: Crash master during master-take-over in execMASTER_LCPCONF + ERROR CODES FOR TESTING NODE FAILURE, LOCAL CHECKPOINT HANDLING: ----------------------------------------------------------------- diff --git a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp index 5f573d40dfe..0e6fe4714b6 100644 --- a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp +++ b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp @@ -4612,6 +4612,8 @@ void Dbdih::startLcpMasterTakeOver(Signal* signal, Uint32 nodeId){ jam(); + Uint32 oldNode = c_lcpMasterTakeOverState.failedNodeId; + c_lcpMasterTakeOverState.minTableId = ~0; c_lcpMasterTakeOverState.minFragId = ~0; c_lcpMasterTakeOverState.failedNodeId = nodeId; @@ -4630,7 +4632,20 @@ Dbdih::startLcpMasterTakeOver(Signal* signal, Uint32 nodeId){ /** * Node failure during master take over... */ - ndbout_c("Nodefail during master take over"); + ndbout_c("Nodefail during master take over (old: %d)", oldNode); + } + + NodeRecordPtr nodePtr; + nodePtr.i = oldNode; + if (oldNode > 0 && oldNode < MAX_NDB_NODES) + { + jam(); + ptrCheckGuard(nodePtr, MAX_NDB_NODES, nodeRecord); + if (nodePtr.p->m_nodefailSteps.get(NF_LCP_TAKE_OVER)) + { + jam(); + checkLocalNodefailComplete(signal, oldNode, NF_LCP_TAKE_OVER); + } } setLocalNodefailHandling(signal, nodeId, NF_LCP_TAKE_OVER); @@ -5646,6 +5661,14 @@ void Dbdih::execMASTER_LCPREQ(Signal* signal) jamEntry(); const BlockReference newMasterBlockref = req->masterRef; + if (newMasterBlockref != cmasterdihref) + { + jam(); + ndbout_c("resending GSN_MASTER_LCPREQ"); + sendSignalWithDelay(reference(), GSN_MASTER_LCPREQ, signal, + signal->getLength(), 50); + return; + } Uint32 failedNodeId = req->failedNodeId; /** @@ -5946,6 +5969,8 @@ void Dbdih::execMASTER_LCPCONF(Signal* signal) ptrCheckGuard(nodePtr, MAX_NDB_NODES, nodeRecord); nodePtr.p->lcpStateAtTakeOver = lcpState; + CRASH_INSERTION(7180); + #ifdef VM_TRACE ndbout_c("MASTER_LCPCONF"); printMASTER_LCP_CONF(stdout, &signal->theData[0], 0, 0); diff --git a/ndb/test/include/NdbRestarter.hpp b/ndb/test/include/NdbRestarter.hpp index 3ec92ae786e..a2e6a4b3248 100644 --- a/ndb/test/include/NdbRestarter.hpp +++ b/ndb/test/include/NdbRestarter.hpp @@ -62,6 +62,8 @@ public: int dumpStateAllNodes(int * _args, int _num_args); int getMasterNodeId(); + int getNextMasterNodeId(int nodeId); + int getNodeGroup(int nodeId); int getRandomNodeSameNodeGroup(int nodeId, int randomNumber); int getRandomNodeOtherNodeGroup(int nodeId, int randomNumber); int getRandomNotMasterNodeId(int randomNumber); diff --git a/ndb/test/ndbapi/testNodeRestart.cpp b/ndb/test/ndbapi/testNodeRestart.cpp index 43fb77342b5..e729e8179b5 100644 --- a/ndb/test/ndbapi/testNodeRestart.cpp +++ b/ndb/test/ndbapi/testNodeRestart.cpp @@ -1045,6 +1045,45 @@ int runBug25554(NDBT_Context* ctx, NDBT_Step* step){ return NDBT_OK; } +int +runBug26457(NDBT_Context* ctx, NDBT_Step* step) +{ + NdbRestarter res; + if (res.getNumDbNodes() < 4) + return NDBT_OK; + + int loops = ctx->getNumLoops(); + while (loops --) + { +retry: + int master = res.getMasterNodeId(); + int next = res.getNextMasterNodeId(master); + + ndbout_c("master: %d next: %d", master, next); + + if (res.getNodeGroup(master) == res.getNodeGroup(next)) + { + res.restartOneDbNode(next, false, false, true); + if (res.waitClusterStarted()) + return NDBT_FAILED; + goto retry; + } + + int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 }; + + if (res.dumpStateOneNode(next, val2, 2)) + return NDBT_FAILED; + + if (res.insertErrorInNode(next, 7180)) + return NDBT_FAILED; + + res.restartOneDbNode(master, false, false, true); + if (res.waitClusterStarted()) + return NDBT_FAILED; + } + + return NDBT_OK; +} NDBT_TESTSUITE(testNodeRestart); TESTCASE("NoLoad", @@ -1367,6 +1406,9 @@ TESTCASE("Bug25364", ""){ TESTCASE("Bug25554", ""){ INITIALIZER(runBug25554); } +TESTCASE("Bug26457", ""){ + INITIALIZER(runBug26457); +} NDBT_TESTSUITE_END(testNodeRestart); int main(int argc, const char** argv){ diff --git a/ndb/test/run-test/daily-basic-tests.txt b/ndb/test/run-test/daily-basic-tests.txt index 00fa14eea2c..9074ff145d8 100644 --- a/ndb/test/run-test/daily-basic-tests.txt +++ b/ndb/test/run-test/daily-basic-tests.txt @@ -477,6 +477,10 @@ max-time: 1000 cmd: testNodeRestart args: -n Bug25554 T1 +max-time: 1000 +cmd: testNodeRestart +args: -n Bug26457 T1 + # OLD FLEX max-time: 500 cmd: flexBench diff --git a/ndb/test/src/NdbRestarter.cpp b/ndb/test/src/NdbRestarter.cpp index 2c16a05240d..4c7f52a8622 100644 --- a/ndb/test/src/NdbRestarter.cpp +++ b/ndb/test/src/NdbRestarter.cpp @@ -128,6 +128,68 @@ NdbRestarter::getMasterNodeId(){ return node; } +int +NdbRestarter::getNodeGroup(int nodeId){ + if (!isConnected()) + return -1; + + if (getStatus() != 0) + return -1; + + for(size_t i = 0; i < ndbNodes.size(); i++) + { + if(ndbNodes[i].node_id == nodeId) + { + return ndbNodes[i].node_group; + } + } + + return -1; +} + +int +NdbRestarter::getNextMasterNodeId(int nodeId){ + if (!isConnected()) + return -1; + + if (getStatus() != 0) + return -1; + + size_t i; + for(i = 0; i < ndbNodes.size(); i++) + { + if(ndbNodes[i].node_id == nodeId) + { + break; + } + } + assert(i < ndbNodes.size()); + if (i == ndbNodes.size()) + return -1; + + int dynid = ndbNodes[i].dynamic_id; + int minid = dynid; + for (i = 0; i minid) + minid = ndbNodes[i].dynamic_id; + + for (i = 0; i dynid && + ndbNodes[i].dynamic_id < minid) + { + minid = ndbNodes[i].dynamic_id; + } + + if (minid != ~0) + { + for (i = 0; i Date: Mon, 19 Feb 2007 20:13:08 +0100 Subject: [PATCH 2/5] ndb - bug#26481 fix for killed node during initial node restart --- ndb/src/kernel/blocks/dbdih/Dbdih.hpp | 1 + ndb/src/kernel/blocks/dbdih/DbdihInit.cpp | 1 + ndb/src/kernel/blocks/dbdih/DbdihMain.cpp | 22 ++++++++++++ ndb/src/kernel/blocks/dblqh/DblqhMain.cpp | 3 +- ndb/test/ndbapi/testNodeRestart.cpp | 43 +++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) diff --git a/ndb/src/kernel/blocks/dbdih/Dbdih.hpp b/ndb/src/kernel/blocks/dbdih/Dbdih.hpp index f1b3897c76f..41240a2d620 100644 --- a/ndb/src/kernel/blocks/dbdih/Dbdih.hpp +++ b/ndb/src/kernel/blocks/dbdih/Dbdih.hpp @@ -1368,6 +1368,7 @@ private: Uint32 csystemnodes; Uint32 currentgcp; Uint32 c_newest_restorable_gci; + Uint32 c_set_initial_start_flag; enum GcpMasterTakeOverState { GMTOS_IDLE = 0, diff --git a/ndb/src/kernel/blocks/dbdih/DbdihInit.cpp b/ndb/src/kernel/blocks/dbdih/DbdihInit.cpp index bb96c4b8831..4e71ecd5fe2 100644 --- a/ndb/src/kernel/blocks/dbdih/DbdihInit.cpp +++ b/ndb/src/kernel/blocks/dbdih/DbdihInit.cpp @@ -75,6 +75,7 @@ void Dbdih::initData() c_blockCommit = false; c_blockCommitNo = 1; cntrlblockref = RNIL; + c_set_initial_start_flag = FALSE; }//Dbdih::initData() void Dbdih::initRecords() diff --git a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp index 0e6fe4714b6..bf71bc56723 100644 --- a/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp +++ b/ndb/src/kernel/blocks/dbdih/DbdihMain.cpp @@ -667,6 +667,12 @@ done: { jam(); memcpy(sysfileData, cdata, sizeof(sysfileData)); + + if (c_set_initial_start_flag) + { + jam(); + Sysfile::setInitialStartOngoing(SYSFILE->systemRestartBits); + } } c_copyGCISlave.m_copyReason = reason; @@ -1260,6 +1266,11 @@ void Dbdih::execNDB_STTOR(Signal* signal) // The permission is given by the master node in the alive set. /*-----------------------------------------------------------------------*/ createMutexes(signal, 0); + if (cstarttype == NodeState::ST_INITIAL_NODE_RESTART) + { + jam(); + c_set_initial_start_flag = TRUE; // In sysfile... + } break; case ZNDB_SPH3: @@ -10277,6 +10288,17 @@ Dbdih::sendLCP_COMPLETE_REP(Signal* signal){ sendSignal(c_lcpState.m_masterLcpDihRef, GSN_LCP_COMPLETE_REP, signal, LcpCompleteRep::SignalLength, JBB); + + /** + * Say that an initial node restart does not need to be redone + * once node has been part of first LCP + */ + if (c_set_initial_start_flag && + c_lcpState.m_participatingLQH.get(getOwnNodeId())) + { + jam(); + c_set_initial_start_flag = FALSE; + } } /*-------------------------------------------------------------------------- */ diff --git a/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp b/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp index 3d93ce3fc31..ab5deca27b5 100644 --- a/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp +++ b/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp @@ -11673,7 +11673,8 @@ void Dblqh::execGCP_SAVEREQ(Signal* signal) return; } - if(getNodeState().getNodeRestartInProgress()){ + if(getNodeState().getNodeRestartInProgress() && cstartRecReq == ZFALSE) + { GCPSaveRef * const saveRef = (GCPSaveRef*)&signal->theData[0]; saveRef->dihPtr = dihPtr; saveRef->nodeId = getOwnNodeId(); diff --git a/ndb/test/ndbapi/testNodeRestart.cpp b/ndb/test/ndbapi/testNodeRestart.cpp index e729e8179b5..c8c8ddd88d6 100644 --- a/ndb/test/ndbapi/testNodeRestart.cpp +++ b/ndb/test/ndbapi/testNodeRestart.cpp @@ -1085,6 +1085,46 @@ retry: return NDBT_OK; } +int +runBug26481(NDBT_Context* ctx, NDBT_Step* step) +{ + + int result = NDBT_OK; + int loops = ctx->getNumLoops(); + int records = ctx->getNumRecords(); + NdbRestarter res; + + int node = res.getRandomNotMasterNodeId(rand()); + ndbout_c("node: %d", node); + if (res.restartOneDbNode(node, true, true, true)) + return NDBT_FAILED; + + if (res.waitNodesNoStart(&node, 1)) + return NDBT_FAILED; + + int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 }; + if (res.dumpStateOneNode(node, val2, 2)) + return NDBT_FAILED; + + if (res.insertErrorInNode(node, 7018)) + return NDBT_FAILED; + + if (res.startNodes(&node, 1)) + return NDBT_FAILED; + + res.waitNodesStartPhase(&node, 1, 3); + + if (res.waitNodesNoStart(&node, 1)) + return NDBT_FAILED; + + res.startNodes(&node, 1); + + if (res.waitClusterStarted()) + return NDBT_FAILED; + + return NDBT_OK; +} + NDBT_TESTSUITE(testNodeRestart); TESTCASE("NoLoad", "Test that one node at a time can be stopped and then restarted "\ @@ -1409,6 +1449,9 @@ TESTCASE("Bug25554", ""){ TESTCASE("Bug26457", ""){ INITIALIZER(runBug26457); } +TESTCASE("Bug26481", ""){ + INITIALIZER(runBug26481); +} NDBT_TESTSUITE_END(testNodeRestart); int main(int argc, const char** argv){ From f0253235b80d9d66100b976b89a415fd202baeee Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Mon, 19 Feb 2007 22:30:54 +0100 Subject: [PATCH 3/5] ndb - forgott to add testcase for bug#26481 --- ndb/test/run-test/daily-basic-tests.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ndb/test/run-test/daily-basic-tests.txt b/ndb/test/run-test/daily-basic-tests.txt index 9074ff145d8..baad45013dd 100644 --- a/ndb/test/run-test/daily-basic-tests.txt +++ b/ndb/test/run-test/daily-basic-tests.txt @@ -481,6 +481,10 @@ max-time: 1000 cmd: testNodeRestart args: -n Bug26457 T1 +max-time: 1000 +cmd: testNodeRestart +args: -n Bug26481 T1 + # OLD FLEX max-time: 500 cmd: flexBench From a51d58e6412a009dc45d7bd4f0caaa8d1ba87a4a Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Mon, 19 Feb 2007 22:51:04 +0100 Subject: [PATCH 4/5] ndb - fix correct argv/argc handling in testBitfield --- ndb/test/ndbapi/testBitfield.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ndb/test/ndbapi/testBitfield.cpp b/ndb/test/ndbapi/testBitfield.cpp index e26f495f5a4..1d1e203d185 100644 --- a/ndb/test/ndbapi/testBitfield.cpp +++ b/ndb/test/ndbapi/testBitfield.cpp @@ -36,9 +36,10 @@ main(int argc, char** argv){ load_defaults("my",load_default_groups,&argc,&argv); int ho_error; - argc--; - argv++; - + if ((ho_error=handle_options(&argc, &argv, my_long_options, + ndb_std_get_one_option))) + return NDBT_ProgramExit(NDBT_WRONGARGS); + Ndb_cluster_connection con(opt_connect_str); if(con.connect(12, 5, 1)) { From 8801e8e7af8c34f72761a38224a4457f3467c033 Mon Sep 17 00:00:00 2001 From: "jonas@perch.ndb.mysql.com" <> Date: Tue, 20 Feb 2007 00:55:33 +0100 Subject: [PATCH 5/5] fix compile error in testBitfield --- ndb/test/ndbapi/testBitfield.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ndb/test/ndbapi/testBitfield.cpp b/ndb/test/ndbapi/testBitfield.cpp index 1d1e203d185..8ba8f3d92ef 100644 --- a/ndb/test/ndbapi/testBitfield.cpp +++ b/ndb/test/ndbapi/testBitfield.cpp @@ -8,6 +8,15 @@ static const char* _dbname = "TEST_DB"; static int g_loops = 7; + +NDB_STD_OPTS_VARS; + +static struct my_option my_long_options[] = +{ + NDB_STD_OPTS("ndb_desc"), + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + static void usage() { ndb_std_print_version();