From 2370828b89186f38f776b5c63f9223801f31c190 Mon Sep 17 00:00:00 2001
From: Herman van Rink <rink@initfour.nl>
Date: Wed, 13 Mar 2024 13:54:27 +0100
Subject: [PATCH 1/5] Parse the dovecot/postfix logs to update the last login
 time for mail_user's. #5374

---
 .../cron.d/100-mailbox_stats_hourly.inc.php   | 117 ++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php

diff --git a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
new file mode 100644
index 0000000000..954de202da
--- /dev/null
+++ b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
@@ -0,0 +1,117 @@
+<?php
+
+/*
+Copyright (c) 2024, Herman van Rink, Initfour websolutions
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of ISPConfig nor the names of its contributors
+      may be used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+class cronjob_mailbox_stats_hourly extends cronjob {
+
+	// job schedule
+	protected $_schedule = '0 * * * *';
+	protected $mailbox_traffic = array();
+	protected $mail_boxes = array();
+	protected $mail_rewrites = array();
+
+	/* this function is optional if it contains no custom code */
+	public function onPrepare() {
+		global $app;
+
+		parent::onPrepare();
+	}
+
+	/* this function is optional if it contains no custom code */
+	public function onBeforeRun() {
+		global $app;
+
+		return parent::onBeforeRun();
+	}
+
+	public function onRunJob() {
+		global $app, $conf;
+
+		$sql = "SELECT mailuser_id FROM mail_user WHERE server_id = ?";
+		$records = $app->db->queryAllRecords($sql, $conf['server_id']);
+    if(count($records) > 0) {
+			$this->update_last_mail_login();
+		}
+
+		parent::onRunJob();
+	}
+
+	/* this function is optional if it contains no custom code */
+	public function onAfterRun() {
+		global $app;
+
+		parent::onAfterRun();
+	}
+
+	// Parse the dovecot/postfix logs to update the last login time for mail_user's.
+	private function update_last_mail_login() {
+		global $app;
+
+		// Command to get all successful dovecot and postfix logins from the last hour.
+		$journalCmd = "journalctl -u dovecot.service -u postfix@-.service --since='60 minutes ago' --grep '((imap|pop3)-login: Login|sasl_method=PLAIN)'";
+
+		$process = popen($journalCmd, 'r');
+		if ($process === false) {
+			die("Failed to execute the command");
+		}
+
+		$updatedUsers = [];
+
+		// Loop over all lines.
+		while (!feof($process)) {
+			$line = fgets($process);
+			if ($line === false) {
+				break;
+			}
+
+			$matches = [];
+			// Match pop3/imap logings, or alternately smtp logins.
+			if (preg_match('/(.*) dovecot\[.*\]: (imap|pop3)-login: Login: user=\<([\w\.@-]+)\>/', $line, $matches) || preg_match('/(.*) sasl_method=PLAIN, sasl_username=([\w\.@-]+)/', $line, $matches)) {
+				$user = $matches[3] ?? $matches[2];
+				$updatedUsers[] = $user;
+			}
+		}
+
+		pclose($process);
+
+		$uniqueUsers = array_unique($updatedUsers);
+
+		$app->log('Updating last_access stats for ' . count($uniqueUsers) . ' mail users', LOGLEVEL_DEBUG);
+
+		// Date/time rounded to hours.
+		$now = time() - (time() % (60 * 60));
+		$nowFormatted = date('Y-m-d H:i:s', $now);
+		$sqlStatement = "UPDATE mail_user SET last_access=? WHERE email=?";
+
+		// Save to master db.
+		foreach ($uniqueUsers as $user) {
+			$ret = $app->dbmaster->query($sql, $now, $user);
+		}
+	}
+}
-- 
GitLab


From 50de911df31fdb3da189fcf0f418f5116fd3918a Mon Sep 17 00:00:00 2001
From: Herman van Rink <rink@initfour.nl>
Date: Mon, 18 Mar 2024 23:09:43 +0100
Subject: [PATCH 2/5] var typo

---
 server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
index 954de202da..e0a41a4a59 100644
--- a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
+++ b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
@@ -111,7 +111,7 @@ class cronjob_mailbox_stats_hourly extends cronjob {
 
 		// Save to master db.
 		foreach ($uniqueUsers as $user) {
-			$ret = $app->dbmaster->query($sql, $now, $user);
+			$ret = $app->dbmaster->query($sqlStatement, $now, $user);
 		}
 	}
 }
-- 
GitLab


From 0db552e5ec43567e042742a1783a785c41e8bc65 Mon Sep 17 00:00:00 2001
From: Herman van Rink <rink@initfour.nl>
Date: Mon, 18 Mar 2024 23:29:55 +0100
Subject: [PATCH 3/5] Reduce to 24hour precision, more would require more
 filtering work

---
 .../cron.d/100-mailbox_stats_hourly.inc.php   | 28 ++++++++-----------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
index e0a41a4a59..8a8d9a6e5d 100644
--- a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
+++ b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
@@ -73,39 +73,35 @@ class cronjob_mailbox_stats_hourly extends cronjob {
 	private function update_last_mail_login() {
 		global $app;
 
-		// Command to get all successful dovecot and postfix logins from the last hour.
-		$journalCmd = "journalctl -u dovecot.service -u postfix@-.service --since='60 minutes ago' --grep '((imap|pop3)-login: Login|sasl_method=PLAIN)'";
+		// used for all monitor cronjobs
+		$app->load('monitor_tools');
+		$this->_tools = new monitor_tools();
 
-		$process = popen($journalCmd, 'r');
-		if ($process === false) {
-			die("Failed to execute the command");
-		}
+		// Get the data of the log
+		$log_lines = $this->_tools->_getLogData('log_mail', 1000000);
 
 		$updatedUsers = [];
 
 		// Loop over all lines.
-		while (!feof($process)) {
-			$line = fgets($process);
-			if ($line === false) {
-				break;
-			}
-
+		$line = strtok($log_lines, PHP_EOL);
+		while ($line !== FALSE) {
 			$matches = [];
 			// Match pop3/imap logings, or alternately smtp logins.
-			if (preg_match('/(.*) dovecot\[.*\]: (imap|pop3)-login: Login: user=\<([\w\.@-]+)\>/', $line, $matches) || preg_match('/(.*) sasl_method=PLAIN, sasl_username=([\w\.@-]+)/', $line, $matches)) {
+			if (preg_match('/(.*) (imap|pop3)-login: Login: user=\<([\w\.@-]+)\>/', $line, $matches) || preg_match('/(.*) sasl_method=PLAIN, sasl_username=([\w\.@-]+)/', $line, $matches)) {
 				$user = $matches[3] ?? $matches[2];
 				$updatedUsers[] = $user;
 			}
-		}
 
-		pclose($process);
+			// get the next line
+			$line = strtok(PHP_EOL);
+		}
 
 		$uniqueUsers = array_unique($updatedUsers);
 
 		$app->log('Updating last_access stats for ' . count($uniqueUsers) . ' mail users', LOGLEVEL_DEBUG);
 
 		// Date/time rounded to hours.
-		$now = time() - (time() % (60 * 60));
+		$now = time() - (time() % (60 * 60 * 24));
 		$nowFormatted = date('Y-m-d H:i:s', $now);
 		$sqlStatement = "UPDATE mail_user SET last_access=? WHERE email=?";
 
-- 
GitLab


From 1c74d2490ed30d920195f5eda284f8862b604863 Mon Sep 17 00:00:00 2001
From: Herman van Rink <rink@initfour.nl>
Date: Mon, 18 Mar 2024 23:30:25 +0100
Subject: [PATCH 4/5] Allow reading more then 4MB of data

---
 server/lib/classes/monitor_tools.inc.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/lib/classes/monitor_tools.inc.php b/server/lib/classes/monitor_tools.inc.php
index 86fe3e672d..c6128fc7b5 100644
--- a/server/lib/classes/monitor_tools.inc.php
+++ b/server/lib/classes/monitor_tools.inc.php
@@ -677,14 +677,14 @@ class monitor_tools {
 				$log = 'Logfile path error.';
 			} else {
 				if (is_readable($logfile)) {
-					$log = $this->_getOutputFromExecCommand('tail -n '.intval($max_lines).' ' . escapeshellarg($logfile));
+					$log = $this->_getOutputFromExecCommand('tail -n '.intval($max_lines).' ' . escapeshellarg($logfile), $max_lines);
 				} else {
 					$log = 'Unable to read ' . $logfile;
 				}
 			}
 		} else {
 			if($journalmatch != ''){
-				$log = $this->_getOutputFromExecCommand('journalctl -n '.intval($max_lines).' --no-pager ' . escapeshellcmd($journalmatch));
+				$log = $this->_getOutputFromExecCommand('journalctl -n '.intval($max_lines).' --no-pager ' . escapeshellcmd($journalmatch), $max_lines);
 			}else{
 				$log = 'Unable to read logfile';
 			}
@@ -694,7 +694,7 @@ class monitor_tools {
 		return $log;
 	}
 
-	private function _getOutputFromExecCommand ($command) {
+	private function _getOutputFromExecCommand ($command, $max4k = 1000) {
 		$log = '';
 		$fd = popen($command, 'r');
 		if ($fd) {
@@ -702,7 +702,7 @@ class monitor_tools {
 			while (!feof($fd)) {
 				$log .= fgets($fd, 4096);
 				$n++;
-				if ($n > 1000)
+				if ($n > $max4k)
 					break;
 			}
 			fclose($fd);
-- 
GitLab


From 7012392939d9e40dc1b180aee2a7ba6042310c23 Mon Sep 17 00:00:00 2001
From: Herman van Rink <rink@initfour.nl>
Date: Mon, 18 Mar 2024 23:40:40 +0100
Subject: [PATCH 5/5] 10 milion to be safe

---
 server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
index 8a8d9a6e5d..4ad612f601 100644
--- a/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
+++ b/server/lib/classes/cron.d/100-mailbox_stats_hourly.inc.php
@@ -78,7 +78,7 @@ class cronjob_mailbox_stats_hourly extends cronjob {
 		$this->_tools = new monitor_tools();
 
 		// Get the data of the log
-		$log_lines = $this->_tools->_getLogData('log_mail', 1000000);
+		$log_lines = $this->_tools->_getLogData('log_mail', 10000000);
 
 		$updatedUsers = [];
 
-- 
GitLab