From a73335376ec33249486e9c38dd890043ecd3ae30 Mon Sep 17 00:00:00 2001 From: mcramer Date: Mon, 8 Jun 2009 20:22:27 +0000 Subject: [PATCH] Added: vlogger mysql usage to store traffic reports Added: interface module web traffic statistics Fixed: mail_traffic is now reported to master server instead of local Fixed: GRANTs for slave server database user on master database --- install/dist/conf/centos52.conf.php | 2 + install/dist/conf/centos53.conf.php | 2 + install/dist/conf/debian40.conf.php | 2 + install/dist/conf/fedora9.conf.php | 2 + install/dist/conf/gentoo.conf.php | 2 + install/dist/conf/opensuse110.conf.php | 3 + install/install.php | 8 + install/lib/installer_base.lib.php | 42 +- install/sql/ispconfig3.sql | 14 + install/tpl/apache_ispconfig.conf.master | 2 +- install/tpl/vlogger-dbi.conf.master | 4 + install/update.php | 4 + .../lib/lang/bg_web_sites_stats_list.lng | 8 + .../lib/lang/de_web_sites_stats_list.lng | 8 + .../lib/lang/en_web_sites_stats_list.lng | 8 + .../lib/lang/es_web_sites_stats_list.lng | 8 + .../lib/lang/fi_web_sites_stats_list.lng | 8 + .../lib/lang/fr_web_sites_stats_list.lng | 8 + .../lib/lang/it_web_sites_stats_list.lng | 8 + .../lib/lang/nl_web_sites_stats_list.lng | 8 + .../lib/lang/ru_web_sites_stats_list.lng | 8 + .../lib/lang/se_web_sites_stats_list.lng | 8 + interface/web/sites/lib/module.conf.php | 21 +- .../web/sites/list/web_sites_stats.list.php | 60 ++ .../sites/templates/web_sites_stats_list.htm | 52 ++ interface/web/sites/web_sites_stats.php | 64 ++ server/conf/apache_ispconfig.conf.master | 2 +- server/cron_daily.php | 4 +- server/scripts/vlogger | 564 ++++++++++++++++++ 29 files changed, 926 insertions(+), 8 deletions(-) create mode 100644 install/tpl/vlogger-dbi.conf.master create mode 100644 interface/web/sites/lib/lang/bg_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/de_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/en_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/es_web_sites_stats_list.lng create mode 100755 interface/web/sites/lib/lang/fi_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/fr_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/it_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/nl_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/ru_web_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/se_web_sites_stats_list.lng create mode 100644 interface/web/sites/list/web_sites_stats.list.php create mode 100644 interface/web/sites/templates/web_sites_stats_list.htm create mode 100644 interface/web/sites/web_sites_stats.php create mode 100755 server/scripts/vlogger diff --git a/install/dist/conf/centos52.conf.php b/install/dist/conf/centos52.conf.php index 6f169fd0d..0f7cac079 100644 --- a/install/dist/conf/centos52.conf.php +++ b/install/dist/conf/centos52.conf.php @@ -148,5 +148,7 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /bin/basename /usr/bin/dirname /usr/bin/nano'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; ?> \ No newline at end of file diff --git a/install/dist/conf/centos53.conf.php b/install/dist/conf/centos53.conf.php index d0f35f50a..51a5ba448 100644 --- a/install/dist/conf/centos53.conf.php +++ b/install/dist/conf/centos53.conf.php @@ -148,5 +148,7 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /bin/basename /usr/bin/dirname /usr/bin/nano'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; ?> \ No newline at end of file diff --git a/install/dist/conf/debian40.conf.php b/install/dist/conf/debian40.conf.php index 0d5842162..92fd4289b 100644 --- a/install/dist/conf/debian40.conf.php +++ b/install/dist/conf/debian40.conf.php @@ -148,5 +148,7 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /usr/bin/lesspipe /usr/bin/basename /usr/bin/dirname /usr/bin/nano /usr/bin/pico'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; ?> \ No newline at end of file diff --git a/install/dist/conf/fedora9.conf.php b/install/dist/conf/fedora9.conf.php index 8a69fc42d..fd4169035 100644 --- a/install/dist/conf/fedora9.conf.php +++ b/install/dist/conf/fedora9.conf.php @@ -148,5 +148,7 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /bin/basename /usr/bin/dirname /usr/bin/nano'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; ?> \ No newline at end of file diff --git a/install/dist/conf/gentoo.conf.php b/install/dist/conf/gentoo.conf.php index ff7b6268d..4d9a7308b 100644 --- a/install/dist/conf/gentoo.conf.php +++ b/install/dist/conf/gentoo.conf.php @@ -97,5 +97,7 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /usr/bin/lesspipe /usr/bin/basename /usr/bin/dirname /usr/bin/nano /usr/bin/pico'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; ?> \ No newline at end of file diff --git a/install/dist/conf/opensuse110.conf.php b/install/dist/conf/opensuse110.conf.php index e4991fe14..e46d77ece 100644 --- a/install/dist/conf/opensuse110.conf.php +++ b/install/dist/conf/opensuse110.conf.php @@ -148,5 +148,8 @@ $conf['jailkit']['jk_init'] = 'jk_init.ini'; $conf['jailkit']['jk_chrootsh'] = 'jk_chrootsh.ini'; $conf['jailkit']['jailkit_chroot_app_programs'] = '/usr/bin/groups /usr/bin/id /usr/bin/dircolors /usr/bin/basename /usr/bin/dirname /usr/bin/nano /usr/bin/pico'; +//* vlogger +$conf['vlogger']['config_dir'] = '/etc'; + ?> \ No newline at end of file diff --git a/install/install.php b/install/install.php index 67651701b..4378fd7e0 100644 --- a/install/install.php +++ b/install/install.php @@ -195,6 +195,10 @@ if($install_mode == 'standard') { swriteln('Configuring Apache'); $inst->configure_apache(); + //** Configure vlogger + swriteln('Configuring vlogger'); + $inst->configure_vlogger(); + //* Configure Firewall swriteln('Configuring Firewall'); $inst->configure_firewall(); @@ -359,6 +363,10 @@ if($install_mode == 'standard') { $conf['services']['web'] = true; swriteln('Configuring Apache'); $inst->configure_apache(); + + //** Configure vlogger + swriteln('Configuring vlogger'); + $inst->configure_vlogger(); } //** Configure Firewall diff --git a/install/lib/installer_base.lib.php b/install/lib/installer_base.lib.php index d68a20842..527a6e6ed 100644 --- a/install/lib/installer_base.lib.php +++ b/install/lib/installer_base.lib.php @@ -318,6 +318,21 @@ class installer_base { if(!$this->dbmaster->query($query)) { $this->error('Unable to create database user in master database: '.$conf['mysql']['master_ispconfig_user'].' Error: '.$this->dbmaster->errorMessage); } + + $query = "GRANT SELECT, INSERT, UPDATE ON ".$conf['mysql']['master_database'].".`mail_traffic` " + ."TO '".$conf['mysql']['master_ispconfig_user']."'@'".$src_host."' " + ."IDENTIFIED BY '".$conf['mysql']['master_ispconfig_password']."';"; + if(!$this->dbmaster->query($query)) { + $this->error('Unable to create database user in master database: '.$conf['mysql']['master_ispconfig_user'].' Error: '.$this->dbmaster->errorMessage); + } + + $query = "GRANT SELECT, INSERT, UPDATE ON ".$conf['mysql']['master_database'].".`web_traffic` " + ."TO '".$conf['mysql']['master_ispconfig_user']."'@'".$src_host."' " + ."IDENTIFIED BY '".$conf['mysql']['master_ispconfig_password']."';"; + if(!$this->dbmaster->query($query)) { + $this->error('Unable to create database user in master database: '.$conf['mysql']['master_ispconfig_user'].' Error: '.$this->dbmaster->errorMessage); + } + } } @@ -856,7 +871,32 @@ class installer_base { } - + public function configure_vlogger() + { + global $conf; + + //** Configure vlogger to use traffic logging to mysql (master) db + $configfile = 'vlogger-dbi.conf'; + if(is_file($conf["vlogger"]["config_dir"].'/'.$configfile)) copy($conf["vlogger"]["config_dir"].'/'.$configfile,$conf["vlogger"]["config_dir"].'/'.$configfile.'~'); + if(is_file($conf["vlogger"]["config_dir"].'/'.$configfile.'~')) exec('chmod 400 '.$conf["vlogger"]["config_dir"].'/'.$configfile.'~'); + $content = rf("tpl/".$configfile.".master"); + if($conf['mysql']['master_slave_setup'] == 'y') { + $content = str_replace('{mysql_server_ispconfig_user}',$conf['mysql']['master_ispconfig_user'],$content); + $content = str_replace('{mysql_server_ispconfig_password}',$conf['mysql']['master_ispconfig_password'], $content); + $content = str_replace('{mysql_server_database}',$conf['mysql']['master_database'],$content); + $content = str_replace('{mysql_server_ip}',$conf["mysql"]["master_host"],$content); + } else { + $content = str_replace('{mysql_server_ispconfig_user}',$conf['mysql']['ispconfig_user'],$content); + $content = str_replace('{mysql_server_ispconfig_password}',$conf['mysql']['ispconfig_password'], $content); + $content = str_replace('{mysql_server_database}',$conf['mysql']['database'],$content); + $content = str_replace('{mysql_server_ip}',$conf["mysql"]["host"],$content); + } + wf($conf["vlogger"]["config_dir"].'/'.$configfile,$content); + exec('chmod 600 '.$conf["vlogger"]["config_dir"].'/'.$configfile); + exec('chown root:root '.$conf["vlogger"]["config_dir"].'/'.$configfile); + + } + public function install_ispconfig() { global $conf; diff --git a/install/sql/ispconfig3.sql b/install/sql/ispconfig3.sql index 448c0a846..376ff6688 100644 --- a/install/sql/ispconfig3.sql +++ b/install/sql/ispconfig3.sql @@ -1016,6 +1016,20 @@ CREATE TABLE `web_database` ( ) ENGINE=MyISAM AUTO_INCREMENT=1; +-- -------------------------------------------------------- + +-- +-- Table structure for table `web_traffic` +-- + +CREATE TABLE `web_traffic` ( + `hostname` varchar(255) NOT NULL, + `traffic_date` date NOT NULL, + `traffic_bytes` bigint(32) unsigned NOT NULL default '0', + PRIMARY KEY (`hostname`,`traffic_date`) +) ENGINE=MyISAM; + + -- -------------------------------------------------------- -- diff --git a/install/tpl/apache_ispconfig.conf.master b/install/tpl/apache_ispconfig.conf.master index c6d044b63..49d0d9be7 100644 --- a/install/tpl/apache_ispconfig.conf.master +++ b/install/tpl/apache_ispconfig.conf.master @@ -5,7 +5,7 @@ ################################################ LogFormat "%v %h %l %u %t \"%r\" %>s %B \"%{Referer}i\" \"%{User-Agent}i\"" combined_ispconfig -CustomLog "| /usr/sbin/vlogger -s access.log -t \"%Y%m%d-access.log\" /var/log/ispconfig/httpd" combined_ispconfig +CustomLog "| /usr/local/ispconfig/server/scripts/vlogger -s access.log -t \"%Y%m%d-access.log\" -d \"/etc/vlogger-dbi.conf\" /var/log/ispconfig/httpd" combined_ispconfig AllowOverride None diff --git a/install/tpl/vlogger-dbi.conf.master b/install/tpl/vlogger-dbi.conf.master new file mode 100644 index 000000000..883bd6a8f --- /dev/null +++ b/install/tpl/vlogger-dbi.conf.master @@ -0,0 +1,4 @@ +dsn DBI:mysql:database={mysql_server_database};host={mysql_server_ip}:3306 +user {mysql_server_ispconfig_user} +pass {mysql_server_ispconfig_password} +dump 30 \ No newline at end of file diff --git a/install/update.php b/install/update.php index 2fb2d9b6a..47be7d6b3 100644 --- a/install/update.php +++ b/install/update.php @@ -300,6 +300,10 @@ if($reconfigure_services_answer == 'yes') { //** Configure Apache swriteln('Configuring Apache'); $inst->configure_apache(); + + //** Configure vlogger + swriteln('Configuring vlogger'); + $inst->configure_vlogger(); } diff --git a/interface/web/sites/lib/lang/bg_web_sites_stats_list.lng b/interface/web/sites/lib/lang/bg_web_sites_stats_list.lng new file mode 100644 index 000000000..b217f80cc --- /dev/null +++ b/interface/web/sites/lib/lang/bg_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/de_web_sites_stats_list.lng b/interface/web/sites/lib/lang/de_web_sites_stats_list.lng new file mode 100644 index 000000000..660abfb25 --- /dev/null +++ b/interface/web/sites/lib/lang/de_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/en_web_sites_stats_list.lng b/interface/web/sites/lib/lang/en_web_sites_stats_list.lng new file mode 100644 index 000000000..20f018b82 --- /dev/null +++ b/interface/web/sites/lib/lang/en_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + \ No newline at end of file diff --git a/interface/web/sites/lib/lang/es_web_sites_stats_list.lng b/interface/web/sites/lib/lang/es_web_sites_stats_list.lng new file mode 100644 index 000000000..b2dd3e9b5 --- /dev/null +++ b/interface/web/sites/lib/lang/es_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/fi_web_sites_stats_list.lng b/interface/web/sites/lib/lang/fi_web_sites_stats_list.lng new file mode 100755 index 000000000..cecf3e387 --- /dev/null +++ b/interface/web/sites/lib/lang/fi_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/fr_web_sites_stats_list.lng b/interface/web/sites/lib/lang/fr_web_sites_stats_list.lng new file mode 100644 index 000000000..b217f80cc --- /dev/null +++ b/interface/web/sites/lib/lang/fr_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/it_web_sites_stats_list.lng b/interface/web/sites/lib/lang/it_web_sites_stats_list.lng new file mode 100644 index 000000000..b217f80cc --- /dev/null +++ b/interface/web/sites/lib/lang/it_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/nl_web_sites_stats_list.lng b/interface/web/sites/lib/lang/nl_web_sites_stats_list.lng new file mode 100644 index 000000000..45bcd7766 --- /dev/null +++ b/interface/web/sites/lib/lang/nl_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/ru_web_sites_stats_list.lng b/interface/web/sites/lib/lang/ru_web_sites_stats_list.lng new file mode 100644 index 000000000..58a4db0d2 --- /dev/null +++ b/interface/web/sites/lib/lang/ru_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/lang/se_web_sites_stats_list.lng b/interface/web/sites/lib/lang/se_web_sites_stats_list.lng new file mode 100644 index 000000000..b217f80cc --- /dev/null +++ b/interface/web/sites/lib/lang/se_web_sites_stats_list.lng @@ -0,0 +1,8 @@ + diff --git a/interface/web/sites/lib/module.conf.php b/interface/web/sites/lib/module.conf.php index 1992fbb37..6a437e827 100644 --- a/interface/web/sites/lib/module.conf.php +++ b/interface/web/sites/lib/module.conf.php @@ -20,9 +20,9 @@ $items[] = array( 'title' => "Subdomain", 'link' => 'sites/web_subdomain_list.php'); -$items[] = array( 'title' => "Aliasdomain", - 'target' => 'content', - 'link' => 'sites/web_aliasdomain_list.php'); +$items[] = array( 'title' => "Aliasdomain", + 'target' => 'content', + 'link' => 'sites/web_aliasdomain_list.php'); $module["nav"][] = array( 'title' => 'Websites', 'open' => 1, @@ -76,6 +76,21 @@ $module["nav"][] = array( 'title' => 'Database', 'open' => 1, 'items' => $items); + +//**** Statistics menu +$items = array(); + +$items[] = array( 'title' => 'Web traffic', + 'target' => 'content', + 'link' => 'sites/web_sites_stats.php'); + + +$module['nav'][] = array( 'title' => 'Statistics', + 'open' => 1, + 'items' => $items); + + + // clean up unset($items); diff --git a/interface/web/sites/list/web_sites_stats.list.php b/interface/web/sites/list/web_sites_stats.list.php new file mode 100644 index 000000000..0824ba996 --- /dev/null +++ b/interface/web/sites/list/web_sites_stats.list.php @@ -0,0 +1,60 @@ + "domain", + 'datatype' => "VARCHAR", + 'formtype' => "TEXT", + 'op' => "like", + 'prefix' => "%", + 'suffix' => "%", + 'width' => "", + 'value' => ""); + + +?> \ No newline at end of file diff --git a/interface/web/sites/templates/web_sites_stats_list.htm b/interface/web/sites/templates/web_sites_stats_list.htm new file mode 100644 index 000000000..84a9b97d2 --- /dev/null +++ b/interface/web/sites/templates/web_sites_stats_list.htm @@ -0,0 +1,52 @@ +

+

+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 
{tmpl_var name="domain"}{tmpl_var name="this_month"} kB{tmpl_var name="last_month"} kB{tmpl_var name="this_year"} kB{tmpl_var name="last_year"} kB +
+
+
+
+
+ +
diff --git a/interface/web/sites/web_sites_stats.php b/interface/web/sites/web_sites_stats.php new file mode 100644 index 000000000..ee8fd5874 --- /dev/null +++ b/interface/web/sites/web_sites_stats.php @@ -0,0 +1,64 @@ +auth->check_module_permissions('sites'); + +$app->load('listform_actions'); + +class list_action extends listform_actions { + + function prepareDataRow($rec) + { + global $app; + + $rec = $app->listform->decode($rec); + + //* Alternating datarow colors + $this->DataRowColor = ($this->DataRowColor == '#FFFFFF') ? '#EEEEEE' : '#FFFFFF'; + $rec['bgcolor'] = $this->DataRowColor; + + //* Set the statistics colums + //** Traffic of the current month + $tmp_year = date('Y'); + $tmp_month = date('m'); + $tmp_rec = $app->db->queryOneRecord("SELECT SUM(traffic_bytes) as t FROM web_traffic WHERE hostname = '".$rec['domain']."' AND YEAR(traffic_date) = '$tmp_year' AND MONTH(traffic_date) = '$tmp_month'"); + $rec['this_month'] = number_format(intval($tmp_rec['t'])/1024, 0, '.', ' '); + + //** Traffic of the current year + $tmp_rec = $app->db->queryOneRecord("SELECT sum(traffic_bytes) as t FROM web_traffic WHERE hostname = '".$rec['domain']."' AND YEAR(traffic_date) = '$tmp_year'"); + $rec['this_year'] = number_format(intval($tmp_rec['t'])/1024, 0, '.', ' '); + + //** Traffic of the last month + $tmp_year = date('Y',mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $tmp_month = date('m',mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $tmp_rec = $app->db->queryOneRecord("SELECT sum(traffic_bytes) as t FROM web_traffic WHERE hostname = '".$rec['domain']."' AND YEAR(traffic_date) = '$tmp_year' AND MONTH(traffic_date) = '$tmp_month'"); + $rec['last_month'] = number_format(intval($tmp_rec['t'])/1024, 0, '.', ' '); + + //** Traffic of the last year + $tmp_year = date('Y',mktime(0, 0, 0, date("m"), date("d"), date("Y")-1)); + $tmp_rec = $app->db->queryOneRecord("SELECT sum(traffic_bytes) as t FROM web_traffic WHERE hostname = '".$rec['domain']."' AND YEAR(traffic_date) = '$tmp_year'"); + $rec['last_year'] = number_format(intval($tmp_rec['t'])/1024, 0, '.', ' '); + + //* The variable "id" contains always the index variable + $rec['id'] = $rec[$this->idx_key]; + return $rec; + } +} + +$list = new list_action; +$list->onLoad(); + + +?> \ No newline at end of file diff --git a/server/conf/apache_ispconfig.conf.master b/server/conf/apache_ispconfig.conf.master index a851b6e75..7db2a489a 100644 --- a/server/conf/apache_ispconfig.conf.master +++ b/server/conf/apache_ispconfig.conf.master @@ -4,7 +4,7 @@ ################################################ LogFormat "%v %h %l %u %t \"%r\" %>s %B \"%{Referer}i\" \"%{User-Agent}i\"" combined_ispconfig -CustomLog "| /usr/sbin/vlogger -s access.log -t \"%Y%m%d-access.log\" /var/log/ispconfig/httpd" combined_ispconfig +CustomLog "| /usr/local/ispconfig/server/scripts/vlogger -s access.log -t \"%Y%m%d-access.log\" -d \"/etc/vlogger-dbi.conf\" /var/log/ispconfig/httpd" combined_ispconfig AllowOverride None diff --git a/server/cron_daily.php b/server/cron_daily.php index f6966883f..2cac46c9c 100644 --- a/server/cron_daily.php +++ b/server/cron_daily.php @@ -68,7 +68,7 @@ foreach($records as $rec) { $tstamp = date("Y-m"); $sql = "SELECT * FROM mail_traffic WHERE month = '$tstamp' AND mailuser_id = ".$rec["mailuser_id"]; - $tr = $app->db->queryOneRecord($sql); + $tr = $app->dbmaster->queryOneRecord($sql); $mail_traffic += $tr["traffic"]; if($tr["traffic_id"] > 0) { @@ -76,7 +76,7 @@ foreach($records as $rec) { } else { $sql = "INSERT INTO mail_traffic (month,mailuser_id,traffic) VALUES ('$tstamp',".$rec["mailuser_id"].",$mail_traffic)"; } - $app->db->query($sql); + $app->dbmaster->query($sql); echo $sql; } diff --git a/server/scripts/vlogger b/server/scripts/vlogger new file mode 100755 index 000000000..d136af176 --- /dev/null +++ b/server/scripts/vlogger @@ -0,0 +1,564 @@ +#!/usr/bin/perl -T +# +# vlogger - smarter logging for apache +# steve j. kondik +# +# this script will take piped logs in STDIN, break off the first component +# and log the line into the proper directory under $LOGDIR. it will roll the +# logs over at midnight on-the-fly and maintain a symlink to the most recent log. +# +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# +# CHANGELOG: +# 0.1 initial release +# 0.2 cleanups, added -e option for errorlogs, added strict stuff +# 0.3 cleanups, bugfixes, docs, added -r size rotation option +# 0.4 added dbi usage tracking option, code cleanups from cz@digitalfreaks.org +# 1.0 small bugfixes, first production release +# 1.1 bugfix release +# 1.2 support for mod_logio +# 1.3 various contributed bugfixes +# +# +# TODO: +# configurable file compression using Compress::Zlib, maybe. +# + +package vlogger; + +$ENV{PATH} = "/bin:/usr/bin"; + +my $VERSION = "1.3"; + +=head1 NAME + +vlogger - flexible log rotation and usage tracking in perl + +=head1 SYNOPSIS + +vlogger [OPTIONS]... [LOGDIR] + +=head1 DESCRIPTION + +Vlogger is designed to make webserver log rotation simple and easy to manage. +It deals with VirtualHost logs automatically, so only one directive is required +to manage all hosts on a webserver. Vlogger takes piped output from Apache or +another webserver, splits off the first field, and writes the logs to logfiles +in subdirectories. It uses a filehandle cache to avoid resource limitations. +It will start a new logfile at the beginning of a new day, and optionally start +new files when a certain filesize is reached. It can maintain a symlink to +the most recent log for easy access. Optionally, host parsing can be disabled +for use in ErrorLog directives. + +To use vlogger, you need to add a "%v" to the first part of your LogFormat: + +LogFormat "%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + +Then call it from a customlog: + +CustomLog "| /usr/local/sbin/vlogger -s access.log -u www-logs -g www-logs /var/log/apache" combined + +=head1 OPTIONS + +Options are given in short format on the command line. + +-a + Do not autoflush files. This may improve performance but may break logfile +analyzers that depend on full entries in the logs. + +-e + ErrorLog mode. In this mode, the host parsing is disabled, and the file is +written out using the template under the specified LOGDIR. + +-n + Disables rotation. This option disables rotation altogether. + +-f MAXFILES + Maximum number of filehandles to keep open. Defaults to 100. Setting this +value too high may result in the system running out of file descriptors. +Setting it too low may affect performance. + +-u UID + Change user to UID when running as root. + +-g GID + Change group to GID when running as root. + +-t TEMPLATE + Filename template using Date::Format codes. Default is "%m%d%Y-access.log", +or "%m%d%Y-error.log". When using the -r option, the default becomes +"%m%d%Y-%T-access.log" or "%m%d%Y-%T-error.log". + +-s SYMLINK + Specifies the name of a symlink to the current file. + +-r SIZE + Rotate files when they reach SIZE. SIZE is given in bytes. + +-d CONFIG + Use the DBI usage tracker. + +-h + Displays help. + +-v + Prints version information. + +=head1 DBI USAGE TRACKER + + Vlogger can automatically keep track of per-virtualhost usage statistics in a +database. DBI and the relevant drivers (eg. DBD::mysql) needs to be installed for +this to work. Create a table in your database to hold the data. A "mysql_create.sql" +script is provided for using this feature with MySQL. Configure the dsn, user, pass +and dump values in the vlogger-dbi.conf file. The "dump" parameter controls how often +vlogger will dump its stats into the database (the default is 30 seconds). Copy this +file to somewhere convienient on your filesystem (like /etc/apache/vlogger-dbi.conf) and +start vlogger with "-d /etc/apache/vlogger-dbi.conf". You might want to use this feature +to easily bill customers on a daily/weekly/monthly basis for bandwidth usage. + +=head1 SEE ALSO +cronolog(1), httplog(1) + +=head1 BUGS +None, yet. + +=head1 AUTHORS +Steve J. Kondik + +WWW: http://n0rp.chemlab.org/vlogger + +=cut + +# a couple modules we need +use strict; +no strict "refs"; +use warnings; +use sigtrap qw(handler exitall HUP USR1 TERM INT PIPE); +use Date::Format; +use Getopt::Std; +use IO::Handle; + +# get command line options +our %OPTS; +getopts( 'f:t:s:hu:g:aeivr:d:', \%OPTS ); + +# print out version +if ( $OPTS{'v'} ) { + print "VLogger $VERSION (apache logfile parser)\n"; + print "Written by Steve J. Kondik \n\n"; + print "This is free software; see the source for copying conditions. There is NO\n"; + print "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"; + exit; +} + +# print help +if ( $OPTS{'h'} || !$ARGV[0] ) { + usage(); + exit; +} + +# log directory +my $LOGDIR; +if ( $ARGV[0] ) { + if ( !-d $ARGV[0] ) { + print STDERR + "[vlogger] target directory $ARGV[0] does not exist - exiting.\n\n"; + exit; + } + $LOGDIR = $ARGV[0]; +} +$LOGDIR =~ /(.*)/; +$LOGDIR = $1; + +# change uid/gid if requested (and running as root) +if ( $> == "0" ) { + if ( $OPTS{'g'} ) { + my $guid = getgrnam( $OPTS{'g'} ); + if ( !defined $guid || $guid == 0 ) { + print STDERR + "[vlogger] cannot run as root or nonexistant group.\n\n"; + exit; + } + + $) = $guid; + $( = $guid; + if ( $) != $guid and $) != ( $guid - 2**32 ) ) { + die "fatal: setgid to gid $guid failed\n"; + } + } + + if ( $OPTS{'u'} ) { + my $uuid = getpwnam( $OPTS{'u'} ); + if ( !defined $uuid || $uuid == 0 ) { + print STDERR + "[vlogger] cannot run as root or nonexistant user.\n\n"; + exit; + } + + $> = $uuid; + $< = $uuid; + if ( $> != $uuid and $> != ( $uuid - 2**32 ) ) { + die "fatal: setuid to uid $uuid failed\n"; + } + } +} + +# set up dbi stuffs + +my $DBI_DSN; +my $DBI_USER; +my $DBI_PASS; +my $DBI_DUMP; +if ( $OPTS{'d'} ) { + if ( $OPTS{'e'} ) { + print "-d not valid with -e. exiting.\n"; + exit; + } + + eval "use DBI"; + + open CONF, $OPTS{'d'}; + while () { + chomp; + my @conf = split (/\s/); + if ( $conf[0] eq "dsn" ) { + $DBI_DSN = $conf[1]; + } + elsif ( $conf[0] eq "user" ) { + $DBI_USER = $conf[1]; + } + elsif ( $conf[0] eq "pass" ) { + $DBI_PASS = $conf[1]; + } + elsif ( $conf[0] eq "dump" ) { + $DBI_DUMP = $conf[1]; + } + } + close CONF; + + unless ( $DBI_DSN && $DBI_USER && $DBI_PASS && $DBI_DUMP ) { + print "All values for DBI configuration are not properly defined.\n\n"; + exit; + } + + # test the connection + my $dbh = DBI->connect( $DBI_DSN, $DBI_USER, $DBI_PASS ) + or die "DBI Error: $!"; + $dbh->disconnect; + + # SIGALRM dumps the tracker hash + $SIG{ALRM} = \&dump_tracker; + + alarm $DBI_DUMP; + +} + +# max files to keep open +my $MAXFILES; +if ( $OPTS{'f'} ) { + $MAXFILES = $OPTS{'f'}; +} +else { + $MAXFILES = "100"; +} + +# filesize rotation +my $MAXSIZE; +if ( $OPTS{'r'} ) { + $MAXSIZE = $OPTS{'r'}; +} + +# filename template +my $TEMPLATE; +if ( $OPTS{'t'} ) { + $TEMPLATE = $OPTS{'t'}; + $TEMPLATE =~ /(.*)/; + $TEMPLATE = $1; + +} +elsif ( $OPTS{'e'} ) { + if ( $OPTS{'r'} ) { + $TEMPLATE = "%m%d%Y-%T-error.log"; + } + else { + $TEMPLATE = "%m%d%Y-error.log"; + } +} +else { + if ( $OPTS{'r'} ) { + $TEMPLATE = "%m%d%Y-%T-access.log"; + } + else { + $TEMPLATE = "%m%d%Y-access.log"; + } +} + +# symlink +if ( $OPTS{'s'} ) { + $OPTS{'s'} =~ /(.*)/; + $OPTS{'s'} = $1; +} + +# chroot to the logdir +chdir($LOGDIR); +#chroot("."); we better do not chroot as DBI requires to load a module on the fly -> error! + +my %logs = (); +my %tracker = (); +my $LASTDUMP = time(); + +# pick a mode +if ( $OPTS{'e'} ) { + + $0 = "vlogger (error log)"; + # errorlog mode + open ELOG, ">>" . time2str( $TEMPLATE, time() ) + or die ( "can't open $LOGDIR/" . time2str( $TEMPLATE, time() ) ); + + unless ( $OPTS{'a'} ) { + ELOG->autoflush(1); + } + if ( $OPTS{'s'} ) { + if ( -l $OPTS{'s'} ) { + unlink( $OPTS{'s'} ); + } + symlink( time2str( $TEMPLATE, time() ), $OPTS{'s'} ); + } + + my $LASTWRITE = time(); + + while ( my $log_line = ) { + unless ( $OPTS{'n'} ) { + if ( time2str( "%Y%m%d", time() ) > + time2str( "%Y%m%d", $LASTWRITE ) ) + { + + # open a new file + close ELOG; + open_errorlog(); + } + elsif ( $OPTS{'r'} ) { + + # check the size + my @filesize = ELOG->stat; + print $filesize[7] . "\n"; + if ( $filesize[7] > $MAXSIZE ) { + close ELOG; + open_errorlog(); + } + } + + $LASTWRITE = time(); + } + + # we dont need to do any other parsing at all, so write the line. + print ELOG $log_line; + } + +} +else { + + # accesslog mode + $0 = "vlogger (access log)"; + while ( my $log_line = ) { + + # parse out the first word (the vhost) + my @this_line = split ( /\s/, $log_line ); + my ($vhost) = $this_line[0]; + my $reqsize = $this_line[10]; + $vhost = lc($vhost) || "default"; + if ( $vhost =~ m#[/\\]# ) { $vhost = "default" } + $vhost =~ /(.*)/o; + $vhost = $1; + $vhost = 'default' unless $vhost; + + if ( $OPTS{'i'} ) { + $reqsize = $this_line[1] + $this_line[2]; + } + + # if we're writing to a log, and it rolls to a new day, close all files. + unless ( $OPTS{'n'} ) { + if ( $logs{$vhost} + && ( time2str( "%Y%m%d", time() ) > + time2str( "%Y%m%d", $logs{$vhost} ) ) ) + { + foreach my $key ( keys %logs ) { + close $key; + } + %logs = (); + } + elsif ( $OPTS{'r'} && $logs{$vhost} ) { + + # check the size + my @filesize = $vhost->stat; + if ( $filesize[7] > $MAXSIZE ) { + close $vhost; + delete( $logs{$vhost} ); + } + } + } + + # open a new log + if ( !$logs{$vhost} ) { + + # check how many files we have open, close the oldest one + if ( keys(%logs) > $MAXFILES ) { + my ( $key, $value ) = + sort { $logs{$a} <=> $logs{$b} } ( keys(%logs) ); + close $key; + delete( $logs{$key} ); + } + + # check if directory is there + unless ( -d "${vhost}" ) { + mkdir("${vhost}"); + } + + # open the file using the template + open $vhost, ">>${vhost}/" . time2str( $TEMPLATE, time() ) + or die ( "can't open $LOGDIR/${vhost}/" + . time2str( $TEMPLATE, time() ) ); + + # autoflush the handle unless -a + if ( !$OPTS{'a'} ) { + $vhost->autoflush(1); + } + + # make a symlink if -s + if ( $OPTS{'s'} ) { + chdir("${vhost}"); + if ( -l $OPTS{'s'} ) { + unlink( $OPTS{'s'} ); + } + symlink( time2str( $TEMPLATE, time() ), $OPTS{'s'} ); + chdir(".."); + } + } + + # update the timestamp and write the line + $logs{$vhost} = time(); + if ($OPTS{'i'}) { + $log_line =~ s/^\S*\s+\S*\s+\S*\s+//o; + } + else { + $log_line =~ s/^\S*\s+//o; + } + + if ( $reqsize =~ m/^\d*$/ && $reqsize > 0 ) { + $tracker{$vhost} += $reqsize; + } + + print $vhost $log_line; + + } +} + +# sub to close all files +sub closeall { + if ( $OPTS{'e'} ) { + close ELOG; + } + else { + foreach my $key ( keys %logs ) { + close $key; + } + %logs = (); + if ( $OPTS{'d'} ) { + vlogger::dump_tracker(); + } + } +} + +sub exitall { + vlogger::closeall; + exit; +} + +# sub to open new errorlog +sub open_errorlog { + open ELOG, ">>" . time2str( $TEMPLATE, time() ) + or die ( "can't open $LOGDIR/" . time2str( $TEMPLATE, time() ) ); + if ( $OPTS{'s'} ) { + if ( -l $OPTS{'s'} ) { + unlink( $OPTS{'s'} ); + } + symlink( time2str( $TEMPLATE, time() ), $OPTS{'s'} ); + } + + # autoflush it unless -a + unless ( $OPTS{'a'} ) { + ELOG->autoflush(1); + } +} + +# sub to update the database with the tracker data +sub dump_tracker { + if ( keys(%tracker) > 0 ) { + my $dbh = DBI->connect( $DBI_DSN, $DBI_USER, $DBI_PASS ) + or warn "DBI Error: $!"; + foreach my $key ( keys(%tracker) ) { + my $ts = time2str( "%Y-%m-%d", time() ); + my $sth = + $dbh->prepare( "select * from web_traffic where hostname='" . $key + . "' and traffic_date='" . $ts . "'" ); + $sth->execute; + if ( $sth->rows ) { + my $query = + "update web_traffic set traffic_bytes=traffic_bytes+" + . $tracker{$key} + . " where hostname='" . $key + . "' and traffic_date='" . $ts . "'"; + $dbh->do($query); + } + else { + my $query = "insert into web_traffic (hostname, traffic_date, traffic_bytes) values ('$key', '$ts', '$tracker{$key}')"; + $dbh->do($query); + } + } + $dbh->disconnect; + %tracker = (); + } + alarm $DBI_DUMP; +} + +# print usage info +sub usage { + print "Usage: vlogger [OPTIONS]... [LOGDIR]\n"; + print "Handles a piped logfile from a webserver, splitting it into it's\n"; + print "host components, and rotates the files daily.\n\n"; + print " -a do not autoflush files\n"; + print " -e errorlog mode\n"; + print " -n don't rotate files\n"; + print " -f MAXFILES max number of files to keep open\n"; + print " -u UID uid to switch to when running as root\n"; + print " -g GID gid to switch to when running as root\n"; + print " -t TEMPLATE filename template (see perldoc Date::Format)\n"; + print " -s SYMLINK maintain a symlink to most recent file\n"; + print " -r SIZE rotate when file reaches SIZE\n"; + print " -d CONFIG use DBI usage tracker (see perldoc vlogger)\n"; + print " -i extract mod_logio instead of filesize\n"; + print " -h display this help\n"; + print " -v output version information\n\n"; + print "TEMPLATE may be a filename with Date::Format codes. The default template\n"; + print "is %m%d%Y-access.log. SYMLINK is the name of a file that will be linked to\n"; + print "the most recent file inside the log directory. The default is access.log.\n"; + print "MAXFILES is the maximum number of filehandles to cache. This defaults to 100.\n"; + print "When running with -a, performance may improve, but this might confuse some\n"; + print "log analysis software that expects complete log entries at all times.\n"; + print "Errorlog mode is used when running with an Apache errorlog. In this mode,\n"; + print "virtualhost parsing is disabled, and a single file is written in LOGDIR\n"; + print "using the TEMPLATE (%m%d%Y-error.log is default for -e). When running with\n"; + print "-r, the template becomes %m%d%Y-%T-xxx.log. SIZE is given in bytes.\n\n"; + print "Report bugs to .\n"; +} + -- GitLab