From eed6b3c6f189fcb318a38ef70c66d1501123f342 Mon Sep 17 00:00:00 2001 From: vogelor Date: Mon, 28 Mar 2011 19:53:05 +0000 Subject: [PATCH] the new rescue-module is now able to rescue (restart) mysql --- server/lib/classes/db_mysql.inc.php | 10 +- server/lib/classes/monitor_tools.inc.php | 81 ++++++- .../mods-available/rescue_core_module.inc.php | 94 +++++++- server/server.php | 200 ++++++++++-------- 4 files changed, 283 insertions(+), 102 deletions(-) diff --git a/server/lib/classes/db_mysql.inc.php b/server/lib/classes/db_mysql.inc.php index 445563ee50..2c113b57d0 100644 --- a/server/lib/classes/db_mysql.inc.php +++ b/server/lib/classes/db_mysql.inc.php @@ -61,8 +61,8 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. function updateError($location) { global $app; - $this->errorNumber = mysql_errno($this->linkId); - $this->errorMessage = mysql_error($this->linkId); + $this->errorNumber = @mysql_errno($this->linkId); + $this->errorMessage = @mysql_error($this->linkId); $this->errorLocation = $location; if($this->errorNumber && $this->show_error_messages && method_exists($app,'log')) { @@ -76,7 +76,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. { if($this->linkId == 0) { - $this->linkId = mysql_connect($this->dbHost, $this->dbUser, $this->dbPass); + $this->linkId = @mysql_connect($this->dbHost, $this->dbUser, $this->dbPass); if(!$this->linkId) { $this->updateError('DB::connect()-> mysql_connect'); @@ -463,7 +463,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. return $this->query($sql); } - // gibt Array mit Tabellennamen zurück + // gibt Array mit Tabellennamen zur�ck function getTables($database_name = '') { if($database_name == '') $database_name = $this->dbName; @@ -474,7 +474,7 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. return $tb_names; } - // gibt Feldinformationen zur Tabelle zurück + // gibt Feldinformationen zur Tabelle zur�ck /* $columns = array(action => add | alter | drop name => Spaltenname diff --git a/server/lib/classes/monitor_tools.inc.php b/server/lib/classes/monitor_tools.inc.php index 2fbd4020d5..3ac85932e5 100644 --- a/server/lib/classes/monitor_tools.inc.php +++ b/server/lib/classes/monitor_tools.inc.php @@ -518,8 +518,16 @@ class monitor_tools { /** the id of the server as int */ $server_id = intval($conf['server_id']); - /** get the "active" Services of the server from the DB */ + /** get the "active" Services of the server from the DB */ $services = $app->dbmaster->queryOneRecord('SELECT * FROM server WHERE server_id = ' . $server_id); + /* + * If the DB is down, we have to set the db to "yes". + * If we don't do this, then the monitor will NOT monitor, that the db is down and so the + * rescue-module can not try to rescue the db + */ + if ($services == null) { + $services['db_server'] = 1; + } /* The type of the Monitor-data */ $type = 'services'; @@ -1539,16 +1547,16 @@ class monitor_tools { * We got a connection, but maybe apache is not able to send data over this * connection? */ - fwrite($fp, "GET / HTTP/1.0\r\n\r\n"); + fwrite($fp, "GET / HTTP/1.0\r\n\r\n"); stream_set_timeout($fp, 2); $res = fread($fp, 10); - $info = stream_get_meta_data($fp); + $info = stream_get_meta_data($fp); fclose($fp); - if ($info['timed_out']) { + if ($info['timed_out']) { return false; // Apache was not able to send data over this connection - } else { + } else { return true; // Apache was able to send data over this connection - } + } } else { return false; // Apache was not able to establish a connection } @@ -1577,6 +1585,67 @@ class monitor_tools { return false; } } + + /* + * Set the state to the given level (or higher, but not lesser). + * * If the actual state is critical and you call the method with ok, + * then the state is critical. + * + * * If the actual state is critical and you call the method with error, + * then the state is error. + */ + private function _setState($oldState, $newState) + { + /* + * Calculate the weight of the old state + */ + switch ($oldState) { + case 'no_state': $oldInt = 0; + break; + case 'ok': $oldInt = 1; + break; + case 'unknown': $oldInt = 2; + break; + case 'info': $oldInt = 3; + break; + case 'warning': $oldInt = 4; + break; + case 'critical': $oldInt = 5; + break; + case 'error': $oldInt = 6; + break; + } + /* + * Calculate the weight of the new state + */ + switch ($newState) { + case 'no_state': $newInt = 0 ; + break; + case 'ok': $newInt = 1 ; + break; + case 'unknown': $newInt = 2 ; + break; + case 'info': $newInt = 3 ; + break; + case 'warning': $newInt = 4 ; + break; + case 'critical': $newInt = 5 ; + break; + case 'error': $newInt = 6 ; + break; + } + + /* + * Set to the higher level + */ + if ($newInt > $oldInt){ + return $newState; + } + else + { + return $oldState; + } + } private function _getIntArray($line) { /** The array of float found */ diff --git a/server/mods-available/rescue_core_module.inc.php b/server/mods-available/rescue_core_module.inc.php index 7ce828d259..4fcd60c780 100644 --- a/server/mods-available/rescue_core_module.inc.php +++ b/server/mods-available/rescue_core_module.inc.php @@ -93,6 +93,11 @@ class rescue_core_module { */ $this->_rescueApache(); + /* + * rescue mysql if needed + */ + $this->_rescueMySql(); + /* * The last step is to save the rescue-data */ @@ -263,16 +268,95 @@ class rescue_core_module { $app->log('Apache is down! Try rescue apache (try:' . $tryCount . ')...', LOGLEVEL_WARN); // echo 'Apache is down! Try rescue apache (try:' . $tryCount . ')...'; - /* - * First we stop the running service "normally" - */ - $daemon = ''; if(is_file($conf['init_scripts'] . '/' . 'httpd')) { $daemon = 'httpd'; } else { $daemon = 'apache2'; } + $this->_rescueDaemon($daemon); + } + + /** + * restarts mysql, if needed + */ + private function _rescueMySql(){ + global $app, $conf; + + /* + * do nothing, if it is not allowed to rescue mysql + */ + if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) == 'y')){ + return; + } + + /* + * if the service is up and running, or the service is not installed there is nothing to do... + */ + if ($this->_monitoringData[0][0]['data']['mysqlserver'] != 0){ + /* Clear the try counter, because we do not have to try to rescue the service */ + $this->_rescueData['mysqlserver']['try_counter'] = 0; + return; + } + + /* + * OK, the service is installed and down. + * Maybe this is because of a restart of the service by the admin. + * This means, we check the data 1 minute ago + */ + if ((!isset($this->_monitoringData[1][0]['data']['mysqlserver'])) || + ((isset($this->_monitoringData[1][0]['data']['mysqlserver'])) && ($this->_monitoringData[1][0]['data']['mysqlserver'] != 0))){ + /* + * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago. + * This means, it could be, that the admin is restarting the server. + * We wait one more minute... + */ + return; + } + + /*##### + * The service is down and it was down 1 minute ago. + * We try to rescue it + *#####*/ + + /* Get the try counter */ + $tryCount = (!isset($this->_rescueData['mysqlserver']['try_counter']))? 1 : $this->_rescueData['mysqlserver']['try_counter'] + 1; + + /* Set the new try counter */ + $this->_rescueData['mysqlserver']['try_counter'] = $tryCount; + + /* if 5 times will not work, we have to give up... */ + if ($tryCount > 5){ + $app->log('MySQL is down! Rescue will not help!', LOGLEVEL_ERROR); + return; + } + + + $app->log('MySQL is down! Try rescue mysql (try:' . $tryCount . ')...', LOGLEVEL_WARN); +// echo 'MySQL is down! Try rescue mysql (try:' . $tryCount . ')...'; + + if(is_file($conf['init_scripts'] . '/' . 'mysqld')) { + $daemon = 'mysqld'; + } else { + $daemon = 'mysql'; + } + + $this->_rescueDaemon($daemon); + } + + /** + * Tries to stop and then restart the daemon + * + * @param type $daemon the name of the daemon + */ + private function _rescueDaemon($daemon){ + global $conf; + + // if you need to find all restarts search for "['init_scripts']" + /* + * First we stop the running service "normally" + */ + /* * ATTENTION! * The service hangs. this means it could be, that "stop" will hang also. @@ -292,7 +376,5 @@ class rescue_core_module { */ exec($conf['init_scripts'] . '/' . $daemon . ' start'); } - -// if you need to find all restarts search for "['init_scripts']" } ?> diff --git a/server/server.php b/server/server.php index 537176c6f4..6a3499fd5b 100644 --- a/server/server.php +++ b/server/server.php @@ -1,110 +1,132 @@ db->queryOneRecord("SELECT * FROM server WHERE update = 1 AND server_id = ".$conf['server_id']); -if($server_db_record == false) { - $app->log('Nothing to update for server_id '.$conf['server_id']); - die(); -} else { - // Set update status to 0, so we dont start the update process twice - $app->db->query("UPDATE server SET update = 0 WHERE server_id = ".$conf['server_id']); - $app->log('Begin update.'); -} -*/ + // Get server record, if updates where available for this server + $server_db_record = $app->db->queryOneRecord("SELECT * FROM server WHERE update = 1 AND server_id = ".$conf['server_id']); + if($server_db_record == false) { + $app->log('Nothing to update for server_id '.$conf['server_id']); + die(); + } else { + // Set update status to 0, so we dont start the update process twice + $app->db->query("UPDATE server SET update = 0 WHERE server_id = ".$conf['server_id']); + $app->log('Begin update.'); + } + */ -//* Load the server configuration -if($app->dbmaster->connect()) { +/* + * Try to Load the server configuration from the master-db + */ +if ($app->dbmaster->connect()) { // get the dalaog_id of the last performed record - $server_db_record = $app->dbmaster->queryOneRecord("SELECT * FROM server WHERE server_id = ".$conf['server_id']); - $conf['last_datalog_id'] = (int)$server_db_record['updated']; - $conf['mirror_server_id'] = (int)$server_db_record['mirror_server_id']; + $server_db_record = $app->dbmaster->queryOneRecord("SELECT * FROM server WHERE server_id = " . $conf['server_id']); + $conf['last_datalog_id'] = (int) $server_db_record['updated']; + $conf['mirror_server_id'] = (int) $server_db_record['mirror_server_id']; // Load the ini_parser $app->uses('ini_parser'); // Get server configuration $conf['serverconfig'] = $app->ini_parser->parse_ini_string(stripslashes($server_db_record['config'])); // Set the loglevel $conf['log_priority'] = intval($conf['serverconfig']['server']['loglevel']); - + unset($server_db_record); +} else { + /* + * The master-db is not available. + * Problem: because we need to start the rescue-module (to rescue the DB if this IS the + * server, the master-db is running at) we have to initialize some config... + */ + $conf['last_datalog_id'] = intval('9223372036854775807'); // maxint at 32 and 64 bit systems + $conf['mirror_server_id'] = 0; // no mirror + // Set the loglevel to warning + $conf['log_priority'] = LOGLEVEL_WARN; + /* + * Set the configuration to rescue the database + */ + $conf['serverconfig']['rescue']['try_rescue'] = 'y'; + $conf['serverconfig']['rescue']['do_not_try_rescue_mysql'] = 'n'; } // Check whether another instance of this script is already running -if(is_file($conf['temppath'].$conf['fs_div'].'.ispconfig_lock')){ - clearstatcache(); - for($i=0;$i<120;$i++){ // Wait max. 1200 sec, then retry - if(is_file($conf['temppath'].$conf['fs_div'].'.ispconfig_lock')){ - exec("ps aux | grep '/usr/local/ispconfig/server/[s]erver.php' | wc -l", $check); - if(intval($check[0]) > 1) { // 1 because this is 2nd instance! - $app->log('There is already an instance of server.php running. Exiting.', LOGLEVEL_DEBUG); - exit; - } - $app->log('There is already a lockfile set. Waiting another 10 seconds...', LOGLEVEL_DEBUG); - sleep(10); - clearstatcache(); - } - } +if (is_file($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock')) { + clearstatcache(); + for ($i = 0; $i < 120; $i++) { // Wait max. 1200 sec, then retry + if (is_file($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock')) { + exec("ps aux | grep '/usr/local/ispconfig/server/[s]erver.php' | wc -l", $check); + if (intval($check[0]) > 1) { // 1 because this is 2nd instance! + $app->log('There is already an instance of server.php running. Exiting.', LOGLEVEL_DEBUG); + exit; + } + $app->log('There is already a lockfile set. Waiting another 10 seconds...', LOGLEVEL_DEBUG); + sleep(10); + clearstatcache(); + } + } } // Set Lockfile -@touch($conf['temppath'].$conf['fs_div'].'.ispconfig_lock'); -$app->log('Set Lock: '.$conf['temppath'].$conf['fs_div'].'.ispconfig_lock', LOGLEVEL_DEBUG); +@touch($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock'); +$app->log('Set Lock: ' . $conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock', LOGLEVEL_DEBUG); +/** Do we need to start the core-modules */ +$needStartCore = true; -if($app->db->connect() && $app->dbmaster->connect()) { +/* + * Next we try to process the datalog + */ +if ($app->db->connect() && $app->dbmaster->connect()) { // Check if there is anything to update - if($conf['mirror_server_id'] > 0) { - $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > ".$conf['last_datalog_id']." AND (server_id = ".$conf['server_id']." OR server_id = ".$conf['mirror_server_id']." OR server_id = 0)"); + if ($conf['mirror_server_id'] > 0) { + $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > " . $conf['last_datalog_id'] . " AND (server_id = " . $conf['server_id'] . " OR server_id = " . $conf['mirror_server_id'] . " OR server_id = 0)"); } else { - $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > ".$conf['last_datalog_id']." AND (server_id = ".$conf['server_id']." OR server_id = 0)"); + $tmp_rec = $app->dbmaster->queryOneRecord("SELECT count(server_id) as number from sys_datalog WHERE datalog_id > " . $conf['last_datalog_id'] . " AND (server_id = " . $conf['server_id'] . " OR server_id = 0)"); } - + $tmp_num_records = $tmp_rec['number']; unset($tmp_rec); - if($tmp_num_records > 0) { + if ($tmp_num_records > 0) { /* - There is something to do, triggert by the database -> do it! - */ + There is something to do, triggert by the database -> do it! + */ // Write the Log $app->log("Found $tmp_num_records changes, starting update process.", LOGLEVEL_DEBUG); // Load required base-classes @@ -118,30 +140,38 @@ if($app->db->connect() && $app->dbmaster->connect()) { $app->modules->processDatalog(); // Restart services that need to after configuration $app->services->processDelayedActions(); - } else { - /* - There is no trigger inside the database -> load only the core, maybe they have to do something - */ - // Write the log - $app->log('No Updated records found, starting only the core.', LOGLEVEL_DEBUG); - // Load required base-classes - $app->uses('modules,plugins,file,services'); - // Load the modules that are im the mods-core folder - $app->modules->loadModules('core'); - // Load the plugins that are in the plugins-core folder - $app->plugins->loadPlugins('core'); + // All modules are already loaded and processed, so there is NO NEED to load the core once again... + $needStartCore = false; } } else { - if(!$app->db->connect()) { - $app->log('Unable to connect to local server.'.$app->db->errorMessage,LOGLEVEL_WARN); + if (!$app->db->connect()) { + $app->log('Unable to connect to local server.' . $app->db->errorMessage, LOGLEVEL_WARN); } else { - $app->log('Unable to connect to master server.'.$app->dbmaster->errorMessage,LOGLEVEL_WARN); + $app->log('Unable to connect to master server.' . $app->dbmaster->errorMessage, LOGLEVEL_WARN); } } +/* + * Under normal circumstances the system was loaded and all updates are done. + * but if we do not have to update anything or if the database is not accessible, then we + * have to start the core-system (if the database is accessible, we need the core because of the + * monitoring. If the databse is NOT accessible, we need the core because of rescue the db... + */ +if ($needStartCore) { + // Write the log + $app->log('No Updated records found, starting only the core.', LOGLEVEL_DEBUG); + // Load required base-classes + $app->uses('modules,plugins,file,services'); + // Load the modules that are im the mods-core folder + $app->modules->loadModules('core'); + // Load the plugins that are in the plugins-core folder + $app->plugins->loadPlugins('core'); +} + + // Remove lock -@unlink($conf['temppath'].$conf['fs_div'].'.ispconfig_lock'); -$app->log('Remove Lock: '.$conf['temppath'].$conf['fs_div'].'.ispconfig_lock',LOGLEVEL_DEBUG); +@unlink($conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock'); +$app->log('Remove Lock: ' . $conf['temppath'] . $conf['fs_div'] . '.ispconfig_lock', LOGLEVEL_DEBUG); die("finished.\n"); -- GitLab