_doRescue(); } /** * This function is called when a change in one of the registered tables is detected. * The function then raises the events for the plugins. */ function process($tablename, $action, $data) { // not needed } /** * This Method tries to rescue the server if a service is down. */ private function _doRescue() { /* * do nothing, if the rescue-system is not enabled */ global $conf; if ((!isset($conf['serverconfig']['rescue']['try_rescue'])) || ((isset($conf['serverconfig']['rescue']['try_rescue'])) && ($conf['serverconfig']['rescue']['try_rescue'] !='y'))){ return; } /* * First we get the monitoring data needed for rescuing the system */ $this->_monitoringData = $this->_getMonitoringData(); /* * Next we get the rescue data needed for rescuing the system */ $this->_rescueData = $this->_getRescueData(); /* * rescue mysql if needed (maybe apache depends on mysql, so try this first!) */ $this->_rescueMySql(); /* * rescue apache if needed */ $this->_rescueApache(); /* * The last step is to save the rescue-data */ $this->_saveRescueData(); } /** * This gets the Monitoring-Data, needed for rescuing the system. * Because we can not be 100% sure, that the mysql-DB is up and running, so we use the * file-system (this is no problem, because this module is the only one using this data, * so we do not have parallel access. */ private function _getMonitoringData() { global $app; $dataFilename = dirname(__FILE__) . "/../temp/rescue_module_monitoringdata.ser.txt"; /* * If the file containing the data is too old (older than 5 minutes) it is better to * delete it, because it could be, that the server was down for some times and so the data * are outdated */ if (file_exists($dataFilename) && (filemtime($dataFilename) < (time() - 5 * 60))) { unlink($dataFilename); } /* * Get the monitoring-data */ if (file_exists($dataFilename)) { $data = unserialize(file_get_contents($dataFilename)); } else { $data = array(); } /* * $temp[0] was the data of the last monitoring (means 1 minute ago), $temp[1] is the data * 2 minutes ago and so on. Now we have make place for the newest data... */ $max = sizeof($data); if ($max > 10){ $max = 10; // not more than 10 histories } for ($i = $max; $i > 0; $i--){ $data[$i] = $data[$i -1]; } /* * we need the monitoring tools */ $app->load('monitor_tools'); $tools = new monitor_tools(); /* * Call the needed Monitoring-step and get the data */ $tmp[0] = $tools->monitorServices(); /* Add the data at the FIRST position of the history */ $data[0] = $tmp; /* * We have the newest monitoring data. Save it! * (and protect it, because there may be sensible data in it) */ file_put_contents($dataFilename, serialize($data)); chmod($dataFilename, 0600); /* Thats it */ return $data; } /** * This gets the rescue-Data, needed for rescuing the system. * Because we can not be 100% sure, that the mysql-DB is up and running, so we use the * file-system (this is no problem, because this module is the only one using this data, * so we do not have parallel access. */ private function _getRescueData() { $dataFilename = dirname(__FILE__) . "/../temp/rescue_module_rescuedata.ser.txt"; /* * If the file containing the data is too old (older than 5 minutes) it is better to * delete it, because it could be, that the server was down for some times and so the data * are outdated */ if (file_exists($dataFilename) && (filemtime($dataFilename) < (time() - 5 * 60))) { unlink($dataFilename); } /* * Get the rescue-data */ if (file_exists($dataFilename)) { $data = unserialize(file_get_contents($dataFilename)); } else { $data = array(); } /* Thats it */ return $data; } /** * Writes the rescue data to disk. * Because we can not be 100% sure, that the mysql-DB is up and running, so we use the * file-system (this is no problem, because this module is the only one using this data, * so we do not have parallel access. */ private function _saveRescueData() { $dataFilename = dirname(__FILE__) . "/../temp/rescue_module_rescuedata.ser.txt"; /* * We have the newest data. Save it! * (and protect it, because there may be sensible data in it) */ file_put_contents($dataFilename, serialize($this->_rescueData)); chmod($dataFilename, 0600); } /** * restarts apache, if needed */ private function _rescueApache(){ global $app, $conf; /* * do nothing, if it is not allowed to rescue apache */ if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_apache']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_apache']) == 'y')){ return; } /* * if the service is up and running, or the service is not installed there is nothing to do... */ if ($this->_monitoringData[0][0]['data']['webserver'] != 0){ /* Clear the try counter, because we do not have to try to rescue the service */ $this->_rescueData['webserver']['try_counter'] = 0; return; } /* * OK, the service is installed and down. * Maybe this is because of a restart of the service by the admin. * This means, we check the data 1 minute ago */ if ((!isset($this->_monitoringData[1][0]['data']['webserver'])) || ((isset($this->_monitoringData[1][0]['data']['webserver'])) && ($this->_monitoringData[1][0]['data']['webserver'] != 0))){ /* * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago. * This means, it could be, that the admin is restarting the server. * We wait one more minute... */ return; } /*##### * The service is down and it was down 1 minute ago. * We try to rescue it *#####*/ /* Get the try counter */ $tryCount = (!isset($this->_rescueData['webserver']['try_counter']))? 1 : $this->_rescueData['webserver']['try_counter'] + 1; /* Set the new try counter */ $this->_rescueData['webserver']['try_counter'] = $tryCount; /* if 5 times will not work, we have to give up... */ if ($tryCount > 5){ $app->log('Apache is down! Rescue will not help!', LOGLEVEL_ERROR); return; } $app->log('Apache is down! Try rescue apache (try:' . $tryCount . ')...', LOGLEVEL_WARN); if(is_file($conf['init_scripts'] . '/' . 'httpd')) { $daemon = 'httpd'; } else { $daemon = 'apache2'; } $this->_rescueDaemon($daemon); } /** * restarts mysql, if needed */ private function _rescueMySql(){ global $app, $conf; /* * do nothing, if it is not allowed to rescue mysql */ if ((isset($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) && ($conf['serverconfig']['rescue']['do_not_try_rescue_mysql']) == 'y')){ return; } /* * if the service is up and running, or the service is not installed there is nothing to do... */ if ($this->_monitoringData[0][0]['data']['mysqlserver'] != 0){ /* Clear the try counter, because we do not have to try to rescue the service */ $this->_rescueData['mysqlserver']['try_counter'] = 0; return; } /* * OK, the service is installed and down. * Maybe this is because of a restart of the service by the admin. * This means, we check the data 1 minute ago */ if ((!isset($this->_monitoringData[1][0]['data']['mysqlserver'])) || ((isset($this->_monitoringData[1][0]['data']['mysqlserver'])) && ($this->_monitoringData[1][0]['data']['mysqlserver'] != 0))){ /* * We do NOT have this data or we have this data, but the webserver was not down 1 minute ago. * This means, it could be, that the admin is restarting the server. * We wait one more minute... */ return; } /*##### * The service is down and it was down 1 minute ago. * We try to rescue it *#####*/ /* Get the try counter */ $tryCount = (!isset($this->_rescueData['mysqlserver']['try_counter']))? 1 : $this->_rescueData['mysqlserver']['try_counter'] + 1; /* Set the new try counter */ $this->_rescueData['mysqlserver']['try_counter'] = $tryCount; /* if 5 times will not work, we have to give up... */ if ($tryCount > 5){ $app->log('MySQL is down! Rescue will not help!', LOGLEVEL_ERROR); return; } $app->log('MySQL is down! Try rescue mysql (try:' . $tryCount . ')...', LOGLEVEL_WARN); if(is_file($conf['init_scripts'] . '/' . 'mysqld')) { $daemon = 'mysqld'; } else { $daemon = 'mysql'; } $this->_rescueDaemon($daemon); } /** * Tries to stop and then restart the daemon * * @param type $daemon the name of the daemon */ private function _rescueDaemon($daemon){ global $conf; // if you need to find all restarts search for "['init_scripts']" /* * First we stop the running service "normally" */ /* * ATTENTION! * The service hangs. this means it could be, that "stop" will hang also. * So we have to try to stop but if this will not work, we have to kill the stopping * of the service */ exec($conf['init_scripts'] . '/' . $daemon . ' stop && (sleep 3; kill $!; sleep 2; kill -9 $!) &> /dev/null'); /* * OK, we tryed to stop it normally, maybe this worked maybe not. So we have to look * if the service is already running or not. If so, we have to kill them hard */ exec("kill -9 `ps -A | grep " . $daemon . "| grep -v grep | awk '{print $1}'` &> /dev/null"); /* * There are no more zombies left. Lets start the service.. */ exec($conf['init_scripts'] . '/' . $daemon . ' start'); } } ?>