Commit 4ebc7d18 authored by vogelor's avatar vogelor
Browse files

The very first version of the new rescue-module.

This module tries to rescue (restart) a service if it is down...
At the moment only apache is rescued (more coming soon...)
parent 8bbcc1db
......@@ -34,13 +34,19 @@ class rescue_core_module {
var $class_name = 'rescue_core_module';
/* No actions at this time. maybe later... */
var $actions_available = array();
/**
* The monitoring-Data of this module.
* [0] are the actual data, [1] are the data 1 minnute ago [2] are teh data 2 minuntes...
*/
private $_monitoringData = array();
/** The rescue-Data of this module. */
private $_rescueData = array();
/**
* This function is called during ispconfig installation to determine
* if a symlink shall be created for this plugin.
*/
function onInstall() {
global $conf;
return true;
}
......@@ -48,1875 +54,245 @@ class rescue_core_module {
* This function is called when the module is loaded
*/
function onLoad() {
global $app;
/*
* Do the monitor every n minutes and write the result to the db
*/
$min = @date('i');
if (($min % $this->interval) == 0) {
$this->doMonitor();
}
$this->_doRescue();
}
/*
This function is called when a change in one of the registered tables is detected.
The function then raises the events for the plugins.
/**
* This function is called when a change in one of the registered tables is detected.
* The function then raises the events for the plugins.
*/
function process($tablename, $action, $data) {
// not needed
}
// end function
//** Get distribution identifier
//** IMPORTANT!
// This is the same code as in install/lib/install.lib.php
// So if you change it here, you also have to change it in there!
// Please do not forget to remove the swriteln(); - lines here at this file
function get_distname() {
$distname = '';
$distver = '';
$distid = '';
$distbaseid = '';
//** Debian or Ubuntu
if (file_exists('/etc/debian_version')) {
if (trim(file_get_contents('/etc/debian_version')) == '4.0') {
$distname = 'Debian';
$distver = '4.0';
$distid = 'debian40';
$distbaseid = 'debian';
} elseif (strstr(trim(file_get_contents('/etc/debian_version')), '5.0')) {
$distname = 'Debian';
$distver = 'Lenny';
$distid = 'debian40';
$distbaseid = 'debian';
} elseif (strstr(trim(file_get_contents('/etc/debian_version')), '6.0') || trim(file_get_contents('/etc/debian_version')) == 'squeeze/sid') {
$distname = 'Debian';
$distver = 'Squeeze/Sid';
$distid = 'debian60';
$distbaseid = 'debian';
} else {
$distname = 'Debian';
$distver = 'Unknown';
$distid = 'debian40';
$distbaseid = 'debian';
}
}
//** OpenSuSE
elseif (file_exists('/etc/SuSE-release')) {
if (stristr(file_get_contents('/etc/SuSE-release'), '11.0')) {
$distname = 'openSUSE';
$distver = '11.0';
$distid = 'opensuse110';
$distbaseid = 'opensuse';
} elseif (stristr(file_get_contents('/etc/SuSE-release'), '11.1')) {
$distname = 'openSUSE';
$distver = '11.1';
$distid = 'opensuse110';
$distbaseid = 'opensuse';
} elseif (stristr(file_get_contents('/etc/SuSE-release'), '11.2')) {
$distname = 'openSUSE';
$distver = '11.1';
$distid = 'opensuse110';
$distbaseid = 'opensuse';
} else {
$distname = 'openSUSE';
$distver = 'Unknown';
$distid = 'opensuse110';
$distbaseid = 'opensuse';
}
}
//** Redhat
elseif (file_exists('/etc/redhat-release')) {
$content = file_get_contents('/etc/redhat-release');
if (stristr($content, 'Fedora release 9 (Sulphur)')) {
$distname = 'Fedora';
$distver = '9';
$distid = 'fedora9';
$distbaseid = 'fedora';
} elseif (stristr($content, 'Fedora release 10 (Cambridge)')) {
$distname = 'Fedora';
$distver = '10';
$distid = 'fedora9';
$distbaseid = 'fedora';
} elseif (stristr($content, 'Fedora release 10')) {
$distname = 'Fedora';
$distver = '11';
$distid = 'fedora9';
$distbaseid = 'fedora';
} elseif (stristr($content, 'CentOS release 5.2 (Final)')) {
$distname = 'CentOS';
$distver = '5.2';
$distid = 'centos52';
$distbaseid = 'fedora';
} elseif (stristr($content, 'CentOS release 5.3 (Final)')) {
$distname = 'CentOS';
$distver = '5.3';
$distid = 'centos53';
$distbaseid = 'fedora';
} else {
$distname = 'Redhat';
$distver = 'Unknown';
$distid = 'fedora9';
$distbaseid = 'fedora';
}
}
//** Gentoo
elseif (file_exists('/etc/gentoo-release')) {
$content = file_get_contents('/etc/gentoo-release');
preg_match_all('/([0-9]{1,2})/', $content, $version);
$distname = 'Gentoo';
$distver = $version[0][0] . $version[0][1];
$distid = 'gentoo';
$distbaseid = 'gentoo';
} else {
die('Unrecognized GNU/Linux distribution');
}
return array('name' => $distname, 'version' => $distver, 'id' => $distid, 'baseid' => $distbaseid);
}
/*
This method is called every n minutes, when the module ist loaded.
The method then does a system-monitoring
/**
* This Method tries to rescue the server if a service is down.
*/
// TODO: what monitoring is done should be a config-var
function doMonitor() {
/* Calls the single Monitoring steps */
$this->monitorHDQuota();
$this->monitorServer();
$this->monitorOSVer();
$this->monitorIspCVer();
$this->monitorDiskUsage();
$this->monitorMemUsage();
$this->monitorCpu();
$this->monitorServices();
if (@file_exists('/proc/user_beancounters')) {
$this->monitorOpenVzHost();
$this->monitorOpenVzUserBeancounter();
}
$this->monitorMailLog();
$this->monitorMailWarnLog();
$this->monitorMailErrLog();
$this->monitorMessagesLog();
$this->monitorISPCCronLog();
$this->monitorFreshClamLog();
$this->monitorClamAvLog();
$this->monitorIspConfigLog();
$this->monitorSystemUpdate();
$this->monitorMailQueue();
$this->monitorRaid();
$this->monitorRkHunter();
$this->monitorFail2ban();
$this->monitorSysLog();
}
function monitorHDQuota() {
global $app;
global $conf;
/* Initialize data array */
$data = array();
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'harddisk_quota';
/** The state of the harddisk_quota. */
$state = 'ok';
/** Fetch the data for all users */
$dfData = shell_exec('repquota -au');
// split into array
$df = explode("\n", $dfData);
private function _doRescue() {
/*
* ignore the first 5 lines, process the rest
* do nothing, if the rescue-system is not enabled
*/
for ($i = 5; $i <= sizeof($df); $i++) {
if ($df[$i] != '') {
/*
* Make a array of the data
*/
$s = preg_split('/[\s]+/', $df[$i]);
$username = $s[0];
if (substr($username, 0, 3) == 'web') {
if (isset($data['user'][$username])) {
$data['user'][$username]['used'] += $s[2];
$data['user'][$username]['soft'] += $s[3];
$data['user'][$username]['hard'] += $s[4];
} else {
$data['user'][$username]['used'] = $s[2];
$data['user'][$username]['soft'] = $s[3];
$data['user'][$username]['hard'] = $s[4];
}
}
}
}
/** Fetch the data for all users */
$dfData = shell_exec('repquota -ag');
// split into array
$df = explode("\n", $dfData);
/*
* ignore the first 5 lines, process the rest
*/
for ($i = 5; $i <= sizeof($df); $i++) {
if ($df[$i] != '') {
/*
* Make a array of the data
*/
$s = preg_split('/[\s]+/', $df[$i]);
$groupname = $s[0];
if (substr($groupname, 0, 6) == 'client') {
if (isset($data['group'][$groupname])) {
$data['group'][$groupname]['used'] += $s[1];
$data['group'][$groupname]['soft'] += $s[2];
$data['group'][$groupname]['hard'] += $s[3];
} else {
$data['group'][$groupname]['used'] = $s[1];
$data['group'][$groupname]['soft'] = $s[2];
$data['group'][$groupname]['hard'] = $s[3];
}
}
}
}
/*
Insert the data into the database
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
}
function monitorServer() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'server_load';
/*
Fetch the data into a array
*/
$procUptime = shell_exec("cat /proc/uptime | cut -f1 -d' '");
$data['up_days'] = floor($procUptime / 86400);
$data['up_hours'] = floor(($procUptime - $data['up_days'] * 86400) / 3600);
$data['up_minutes'] = floor(($procUptime - $data['up_days'] * 86400 - $data['up_hours'] * 3600) / 60);
$data['uptime'] = shell_exec('uptime');
$tmp = explode(',', $data['uptime'], 4);
$tmpUser = explode(' ', trim($tmp[2]));
$data['user_online'] = intval($tmpUser[0]);
//* New Load Average code to fix "always zero" bug in non-english distros. NEEDS TESTING
$loadTmp = shell_exec("cat /proc/loadavg | cut -f1-3 -d' '");
$load = explode(' ', $loadTmp);
$data['load_1'] = floatval(str_replace(',', '.', $load[0]));
$data['load_5'] = floatval(str_replace(',', '.', $load[1]));
$data['load_15'] = floatval(str_replace(',', '.', $load[2]));
/** The state of the server-load. */
$state = 'ok';
if ($data['load_1'] > 20)
$state = 'info';
if ($data['load_1'] > 50)
$state = 'warning';
if ($data['load_1'] > 100)
$state = 'critical';
if ($data['load_1'] > 150)
$state = 'error';
/*
Insert the data into the database
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
}
function monitorOsVer() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'os_info';
/*
Fetch the data into a array
*/
$dist = $this->get_distname();
$data['name'] = $dist['name'];
$data['version'] = $dist['version'];
/* the OS has no state. It is, what it is */
$state = 'no_state';
/*
Insert the data into the database
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
}
function monitorIspcVer() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'ispc_info';
if ((!isset($conf['serverconfig']['rescue']['try_rescue'])) ||
((isset($conf['serverconfig']['rescue']['try_rescue'])) && ($conf['serverconfig']['rescue']['try_rescue'] !='y'))){
return;
}
/*
Fetch the data into a array
* First we get the monitoring data needed for rescuing the system
*/
$data['name'] = ISPC_APP_TITLE;
$data['version'] = ISPC_APP_VERSION;
/* the ISPC-Version has no state. It is, what it is */
$state = 'no_state';
$this->_monitoringData = $this->_getMonitoringData();
/*
Insert the data into the database
* Next we get the rescue data needed for rescuing the system
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
}
function monitorDiskUsage() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'disk_usage';
/** The state of the disk-usage */
$state = 'ok';
/** Fetch the data of ALL devices into a array (needed for monitoring!) */
$dfData = shell_exec('df -hT');
// split into array
$df = explode("\n", $dfData);
$this->_rescueData = $this->_getRescueData();
/*
* ignore the first line, process the rest
* rescue apache if needed
*/
for ($i = 1; $i <= sizeof($df); $i++) {
if ($df[$i] != '') {
/*
* Make an array of the data
*/
$s = preg_split('/[\s]+/', $df[$i]);
$data[$i]['fs'] = $s[0];
$data[$i]['type'] = $s[1];
$data[$i]['size'] = $s[2];
$data[$i]['used'] = $s[3];
$data[$i]['available'] = $s[4];
$data[$i]['percent'] = $s[5];
$data[$i]['mounted'] = $s[6];
/*
* calculate the state
*/
$usePercent = floatval($data[$i]['percent']);
//* We don't want to check some filesystem which have no sensible filling levels
switch ($data[$i]['type']) {
case 'iso9660':
case 'cramfs':
case 'udf':
case 'tmpfs':
case 'devtmpfs':
case 'udev':
break;
default:
if ($usePercent > 75)
$state = $this->_setState($state, 'info');
if ($usePercent > 80)
$state = $this->_setState($state, 'warning');
if ($usePercent > 90)
$state = $this->_setState($state, 'critical');
if ($usePercent > 95)
$state = $this->_setState($state, 'error');
break;
}
}
}
$this->_rescueApache();
/*
Insert the data into the database
* The last step is to save the rescue-data
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
$this->_saveRescueData();
}
function monitorMemUsage() {
/**
* This gets the Monitoring-Data, needed for rescuing the system.
* Because we can not be 100% sure, that the mysql-DB is up and running, so we use the
* file-system (this is no problem, because this module is the only one using this data,
* so we do not have parallel access.
*/
private function _getMonitoringData() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'mem_usage';
$dataFilename = dirname(__FILE__) . "/../lib/rescue_module_monitoringdata.ser.txt";
/*
Fetch the data into a array
* If the file containing the data is too old (older than 5 minutes) it is better to
* delete it, because it could be, that the server was down for some times and so the data
* are outdated
*/
$miData = shell_exec('cat /proc/meminfo');
$memInfo = explode("\n", $miData);
foreach ($memInfo as $line) {
$part = preg_split('/:/', $line);
$key = trim($part[0]);
$tmp = explode(' ', trim($part[1]));
$value = 0;
if ($tmp[1] == 'kB')
$value = $tmp[0] * 1024;
$data[$key] = $value;
if (file_exists($dataFilename) && (filemtime($dataFilename) < (time() - 5 * 60))) {
unlink($dataFilename);
}
/*
* actually this info has no state.
* maybe someone knows better...???...
*/
$state = 'no_state';
/*
Insert the data into the database
* Get the monitoring-data
*/
$sql = 'INSERT INTO monitor_data (server_id, type, created, data, state) ' .
'VALUES (' .
$server_id . ', ' .
"'" . $app->dbmaster->quote($type) . "', " .
time() . ', ' .
"'" . $app->dbmaster->quote(serialize($data)) . "', " .
"'" . $state . "'" .
')';
$app->dbmaster->query($sql);
/* The new data is written, now we can delete the old one */
$this->_delOldRecords($type, 4);
}
function monitorCpu() {
global $app;
global $conf;
/* the id of the server as int */
$server_id = intval($conf['server_id']);
/** The type of the data */
$type = 'cpu_info';