From 5dd3d754422cd91cc0aa755f989e42c91786158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?LEVEILLE=20C=C3=A9dric?= Date: Mon, 18 Jan 2016 14:07:02 -0500 Subject: [PATCH] Ftp statistics feature --- install/sql/incremental/upd_0081.sql | 1 + interface/web/sites/ftp_sites_stats.php | 181 ++++++++++++++++++ .../lib/lang/en_ftp_sites_stats_list.lng | 10 + .../lib/lang/fr_ftp_sites_stats_list.lng | 10 + .../web/sites/list/ftp_sites_stats.list.php | 60 ++++++ .../sites/templates/ftp_sites_stats_list.htm | 60 ++++++ .../classes/cron.d/200-ftplogfiles.inc.php | 126 ++++++++++++ 7 files changed, 448 insertions(+) create mode 100644 install/sql/incremental/upd_0081.sql create mode 100644 interface/web/sites/ftp_sites_stats.php create mode 100644 interface/web/sites/lib/lang/en_ftp_sites_stats_list.lng create mode 100644 interface/web/sites/lib/lang/fr_ftp_sites_stats_list.lng create mode 100644 interface/web/sites/list/ftp_sites_stats.list.php create mode 100644 interface/web/sites/templates/ftp_sites_stats_list.htm create mode 100644 server/lib/classes/cron.d/200-ftplogfiles.inc.php diff --git a/install/sql/incremental/upd_0081.sql b/install/sql/incremental/upd_0081.sql new file mode 100644 index 0000000000..6f6a1cbbe4 --- /dev/null +++ b/install/sql/incremental/upd_0081.sql @@ -0,0 +1 @@ +CREATE TABLE `ftp_traffic` (`hostname` varchar(255) NOT NULL,`traffic_date` date NOT NULL,`in_bytes` bigint(32) unsigned NOT NULL,`out_bytes` bigint(32) unsigned NOT NULL, PRIMARY KEY (`hostname`,`traffic_date`)) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; \ No newline at end of file diff --git a/interface/web/sites/ftp_sites_stats.php b/interface/web/sites/ftp_sites_stats.php new file mode 100644 index 0000000000..7ffc3f769d --- /dev/null +++ b/interface/web/sites/ftp_sites_stats.php @@ -0,0 +1,181 @@ +auth->check_module_permissions('sites'); + +$app->uses('functions'); + +$app->load('listform_actions'); + +class list_action extends listform_actions { + + private $sum_this_month = 0; + private $sum_this_year = 0; + private $sum_last_month = 0; + private $sum_last_year = 0; + + function prepareDataRow($rec) + { + global $app; + + $rec = $app->listform->decode($rec); + + //* Alternating datarow colors + $this->DataRowColor = ($this->DataRowColor == '#FFFFFF') ? '#EEEEEE' : '#FFFFFF'; + $rec['bgcolor'] = $this->DataRowColor; + + //* Set the statistics colums + //** Traffic of the current month + $tmp_year = date('Y'); + $tmp_month = date('m'); + $tmp_rec = $app->db->queryOneRecord("SELECT SUM(in_bytes) AS ftp_in, SUM(out_bytes) AS ftp_out FROM ftp_traffic WHERE hostname = ? AND YEAR(traffic_date) = ? AND MONTH(traffic_date) = ?", $rec['domain'], $tmp_year, $tmp_month); + $rec['this_month_in'] = $app->functions->formatBytes($tmp_rec['ftp_in']); + $rec['this_month_out'] = $app->functions->formatBytes($tmp_rec['ftp_out']); + $this->sum_this_month += $tmp_rec['ftp_in']+$tmp_rec['ftp_out']; + + //** Traffic of the current year + $tmp_rec = $app->db->queryOneRecord("SELECT SUM(in_bytes) AS ftp_in, SUM(out_bytes) AS ftp_out FROM ftp_traffic WHERE hostname = ? AND YEAR(traffic_date) = ?", $rec['domain'], $tmp_year); + $rec['this_year_in'] = $app->functions->formatBytes($tmp_rec['ftp_in']); + $rec['this_year_out'] = $app->functions->formatBytes($tmp_rec['ftp_out']); + $this->sum_this_year += $tmp_rec['ftp_in']+$tmp_rec['ftp_out']; + + //** Traffic of the last month + $tmp_year = date('Y', mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $tmp_month = date('m', mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $tmp_rec = $app->db->queryOneRecord("SELECT SUM(in_bytes) AS ftp_in, SUM(out_bytes) AS ftp_out FROM ftp_traffic WHERE hostname = ? AND YEAR(traffic_date) = ? AND MONTH(traffic_date) = ?", $rec['domain'], $tmp_year, $tmp_month); + $rec['last_month_in'] = $app->functions->formatBytes($tmp_rec['ftp_in']); + $rec['last_month_out'] = $app->functions->formatBytes($tmp_rec['ftp_out']); + $this->sum_last_month += $tmp_rec['ftp_in']+$tmp_rec['ftp_out']; + + //** Traffic of the last year + $tmp_year = date('Y', mktime(0, 0, 0, date("m"), date("d"), date("Y")-1)); + $tmp_rec = $app->db->queryOneRecord("SELECT SUM(in_bytes) AS ftp_in, SUM(out_bytes) AS ftp_out FROM ftp_traffic WHERE hostname = ? AND YEAR(traffic_date) = ?", $rec['domain'], $tmp_year); + $rec['last_year_in'] = $app->functions->formatBytes($tmp_rec['ftp_in']); + $rec['last_year_out'] = $app->functions->formatBytes($tmp_rec['ftp_out']); + $this->sum_last_year += $tmp_rec['ftp_in']+$tmp_rec['ftp_out']; + + //* The variable "id" contains always the index variable + $rec['id'] = $rec[$this->idx_key]; + + return $rec; + } + + function onShowEnd() + { + global $app; + + $app->tpl->setVar('sum_this_month', $app->functions->formatBytes($this->sum_this_month)); + $app->tpl->setVar('sum_this_year', $app->functions->formatBytes($this->sum_this_year)); + $app->tpl->setVar('sum_last_month', $app->functions->formatBytes($this->sum_last_month)); + $app->tpl->setVar('sum_last_year', $app->functions->formatBytes($this->sum_last_year)); + $app->tpl->setVar('sum_txt', $app->listform->lng('sum_txt')); + + $app->tpl_defaults(); + $app->tpl->pparse(); + } + + function getQueryString($no_limit = false) { + global $app; + $sql_where = ''; + + //* Generate the search sql + if($app->listform->listDef['auth'] != 'no') { + if($_SESSION['s']['user']['typ'] == "admin") { + $sql_where = ''; + } else { + $sql_where = $app->tform->getAuthSQL('r', $app->listform->listDef['table']).' and'; + //$sql_where = $app->tform->getAuthSQL('r').' and'; + } + } + if($this->SQLExtWhere != '') { + $sql_where .= ' '.$this->SQLExtWhere.' and'; + } + + $sql_where = $app->listform->getSearchSQL($sql_where); + if($app->listform->listDef['join_sql']) $sql_where .= ' AND '.$app->listform->listDef['join_sql']; + $app->tpl->setVar($app->listform->searchValues); + + $order_by_sql = $this->SQLOrderBy; + + //* Generate SQL for paging + $limit_sql = $app->listform->getPagingSQL($sql_where); + $app->tpl->setVar('paging', $app->listform->pagingHTML); + + $extselect = ''; + $join = ''; + + if(!empty($_SESSION['search'][$_SESSION['s']['module']['name'].$app->listform->listDef["name"].$app->listform->listDef['table']]['order'])){ + $order = str_replace(' DESC', '', $_SESSION['search'][$_SESSION['s']['module']['name'].$app->listform->listDef["name"].$app->listform->listDef['table']]['order']); + list($tmp_table, $order) = explode('.', $order); + if($order == 'ftp_traffic_last_month'){ + $tmp_year = date('Y', mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $tmp_month = date('m', mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $extselect .= ', SUM(ft.in_bytes+out_bytes) as calctraffic'; + $join .= ' INNER JOIN ftp_traffic as ft ON '.$app->listform->listDef['table'].'.domain = ft.hostname '; + $sql_where .= " AND YEAR(ft.traffic_date) = '$tmp_year' AND MONTH(ft.traffic_date) = '$tmp_month'"; + $order_by_sql = str_replace($app->listform->listDef['table'].'.ftp_traffic_last_month', 'calctraffic', $order_by_sql); + $order_by_sql = "GROUP BY domain ".$order_by_sql; + } elseif($order == 'ftp_traffic_this_month'){ + $tmp_year = date('Y'); + $tmp_month = date('m'); + $extselect .= ', SUM(ft.in_bytes+out_bytes) as calctraffic'; + $join .= ' INNER JOIN ftp_traffic as ft ON '.$app->listform->listDef['table'].'.domain = ft.hostname '; + $sql_where .= " AND YEAR(ft.traffic_date) = '$tmp_year' AND MONTH(ft.traffic_date) = '$tmp_month'"; + $order_by_sql = str_replace($app->listform->listDef['table'].'.ftp_traffic_this_month', 'calctraffic', $order_by_sql); + $order_by_sql = "GROUP BY domain ".$order_by_sql; + } elseif($order == 'ftp_traffic_last_year'){ + $tmp_year = date('Y', mktime(0, 0, 0, date("m")-1, date("d"), date("Y"))); + $extselect .= ', SUM(ft.in_bytes+out_bytes) as calctraffic'; + $join .= ' INNER JOIN ftp_traffic as ft ON '.$app->listform->listDef['table'].'.domain = ft.hostname '; + $sql_where .= " AND YEAR(ft.traffic_date) = '$tmp_year'"; + $order_by_sql = str_replace($app->listform->listDef['table'].'.ftp_traffic_last_year', 'calctraffic', $order_by_sql); + $order_by_sql = "GROUP BY domain ".$order_by_sql; + } elseif($order == 'ftp_traffic_this_year'){ + $tmp_year = date('Y'); + $extselect .= ', SUM(ft.in_bytes+out_bytes) as calctraffic'; + $join .= ' INNER JOIN ftp_traffic as ft ON '.$app->listform->listDef['table'].'.domain = ft.hostname '; + $sql_where .= " AND YEAR(ft.traffic_date) = '$tmp_year'"; + $order_by_sql = str_replace($app->listform->listDef['table'].'.ftp_traffic_this_year', 'calctraffic', $order_by_sql); + $order_by_sql = "GROUP BY domain ".$order_by_sql; + } + } + + if($this->SQLExtSelect != '') { + if(substr($this->SQLExtSelect, 0, 1) != ',') $this->SQLExtSelect = ','.$this->SQLExtSelect; + $extselect .= $this->SQLExtSelect; + } + + $table_selects = array(); + $table_selects[] = trim($app->listform->listDef['table']).'.*'; + $app->listform->listDef['additional_tables'] = trim($app->listform->listDef['additional_tables']); + if($app->listform->listDef['additional_tables'] != ''){ + $additional_tables = explode(',', $app->listform->listDef['additional_tables']); + foreach($additional_tables as $additional_table){ + $table_selects[] = trim($additional_table).'.*'; + } + } + $select = implode(', ', $table_selects); + + $sql = 'SELECT '.$select.$extselect.' FROM '.$app->listform->listDef['table'].($app->listform->listDef['additional_tables'] != ''? ','.$app->listform->listDef['additional_tables'] : '')."$join WHERE $sql_where $order_by_sql $limit_sql"; + return $sql; + } +} + +$list = new list_action; +$list->SQLExtWhere = "(web_domain.type = 'vhost' or web_domain.type = 'vhostsubdomain')"; +$list->SQLOrderBy = 'ORDER BY web_domain.domain'; +$list->onLoad(); + +?> \ No newline at end of file diff --git a/interface/web/sites/lib/lang/en_ftp_sites_stats_list.lng b/interface/web/sites/lib/lang/en_ftp_sites_stats_list.lng new file mode 100644 index 0000000000..03894607db --- /dev/null +++ b/interface/web/sites/lib/lang/en_ftp_sites_stats_list.lng @@ -0,0 +1,10 @@ + \ No newline at end of file diff --git a/interface/web/sites/lib/lang/fr_ftp_sites_stats_list.lng b/interface/web/sites/lib/lang/fr_ftp_sites_stats_list.lng new file mode 100644 index 0000000000..d83ee755d7 --- /dev/null +++ b/interface/web/sites/lib/lang/fr_ftp_sites_stats_list.lng @@ -0,0 +1,10 @@ + \ No newline at end of file diff --git a/interface/web/sites/list/ftp_sites_stats.list.php b/interface/web/sites/list/ftp_sites_stats.list.php new file mode 100644 index 0000000000..361e3d36ab --- /dev/null +++ b/interface/web/sites/list/ftp_sites_stats.list.php @@ -0,0 +1,60 @@ + "domain", + 'datatype' => "VARCHAR", + 'filters' => array( 0 => array( 'event' => 'SHOW', + 'type' => 'IDNTOUTF8') + ), + 'formtype' => "TEXT", + 'op' => "like", + 'prefix' => "%", + 'suffix' => "%", + 'width' => "", + 'value' => ""); + +?> \ No newline at end of file diff --git a/interface/web/sites/templates/ftp_sites_stats_list.htm b/interface/web/sites/templates/ftp_sites_stats_list.htm new file mode 100644 index 0000000000..fb4aae1ed6 --- /dev/null +++ b/interface/web/sites/templates/ftp_sites_stats_list.htm @@ -0,0 +1,60 @@ +
+

+
+

+ +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{tmpl_var name='search_limit'}
+ +
{tmpl_var name="domain"}{tmpl_var name="this_month_in"}/{tmpl_var name="this_month_out"}{tmpl_var name="last_month_in"}/{tmpl_var name="last_month_out"}{tmpl_var name="this_year_in"}/{tmpl_var name="this_year_out"}{tmpl_var name="last_year_in"}/{tmpl_var name="last_year_out"}
{tmpl_var name='globalsearch_noresults_text_txt'}
{tmpl_var name="sum_txt"}{tmpl_var name="sum_this_month"}{tmpl_var name="sum_last_month"}{tmpl_var name="sum_this_year"}{tmpl_var name="sum_last_year"}
+
\ No newline at end of file diff --git a/server/lib/classes/cron.d/200-ftplogfiles.inc.php b/server/lib/classes/cron.d/200-ftplogfiles.inc.php new file mode 100644 index 0000000000..e471967555 --- /dev/null +++ b/server/lib/classes/cron.d/200-ftplogfiles.inc.php @@ -0,0 +1,126 @@ +db->queryAllRecords($sql, $conf['server_id']); + + function parse_ftp_log($line){ + if(preg_match('/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - (.+) \[(\d+\/\w+\/\d+):.+\] "(PUT|GET) .+" \d+ (\d+)$/', $line, $matches) == false) return false; + + $date = date('Y-m-d', strtotime(str_replace('/', '-', $matches[2]))); // Correction date + + switch($matches[3]) + { + case 'PUT': + $direction = 'in'; + break; + case 'GET': + $direction = 'out'; + break; + } + // Returned array + return array('username' => $matches[1], 'date' => $date,'direction' => $direction, 'size' => $matches[4]); + } + + function add_ftp_traffic(&$traffic_array, $parsed_line) + { + if(is_array($traffic_array[$parsed_line['date']]) && array_key_exists($parsed_line['domain'], $traffic_array[$parsed_line['date']])) + { + $traffic_array[$parsed_line['date']][$parsed_line['domain']][$parsed_line['direction']] += $parsed_line['size']; + } + else + { + $traffic_array[$parsed_line['date']][$parsed_line['domain']] = array('in' => 0, 'out' => 0 ); + $traffic_array[$parsed_line['date']][$parsed_line['domain']][$parsed_line['direction']] = $parsed_line['size']; + } + } + + $fp = fopen('/var/log/pure-ftpd/transfer.log.1', 'r'); + $ftp_traffic = array(); + + // cumule des stats journalière dans un tableau + while($line = fgets($fp)) + { + $parsed_line = parse_ftp_log($line); + + $sql = "SELECT wd.domain FROM ftp_user AS fu INNER JOIN web_domain AS wd ON fu.parent_domain_id = wd.domain_id WHERE fu.username = ? "; + $temp = $app->db->queryOneRecord($sql, $parsed_line['username'] ); + + $parsed_line['domain'] = $temp['domain']; + + add_ftp_traffic($ftp_traffic, $parsed_line); + } + + fclose($fp); + + // Save du tableau en BD + foreach($ftp_traffic as $traffic_date => $all_traffic) + { + foreach ( $all_traffic as $hostname =>$traffic) + { + $sql1 = "SELECT hostname FROM ftp_traffic WHERE hostname = ? AND traffic_date = ?"; + $tmp = $app->dbmaster->queryOneRecord($sql1, $hostname , $traffic_date); + + if(is_array($tmp) && count($tmp) > 0) { + $sql = "UPDATE ftp_traffic SET in_bytes=in_bytes+ ?, out_bytes=out_bytes+ ? WHERE hostname = ? AND traffic_date = ? "; + } + else + { + $sql = "INSERT INTO ftp_traffic (in_bytes, out_bytes, hostname, traffic_date ) VALUES ( ?, ?, ?, ? )"; + } + + $resultat = $app->dbmaster->query($sql, $traffic['in'], $traffic['out'], $hostname, $traffic_date ); + + //if($resultat == 1){ + //echo 'finished.'.PHP_EOL; // maybe you have a better solution ? } + } + } + + parent::onRunJob(); + } + + /* this function is optional if it contains no custom code */ + public function onAfterRun() { + global $app; + + parent::onAfterRun(); + } +} + +?> \ No newline at end of file -- GitLab