#! /bin/bash

# Autolist: Automated Anti-Spam Blacklists Management System

# Or: Some madness written in bash with tons of regexps inside
# Someday I should learn some other language, but I love bash ;-)

# Reads an email piped from stdin (and previsouly classified good or spam
# by a content analyzer such as DSPAM or SpamAssassin, etc...), parses its
# headers, updates autolist MySQL DB, blacklists or whitelists sending IP
# depending on message nature, automatically maintaining a MySQL black-
# list that can be directly used by an MTA such as Postfix.

# Also optionally parses a Postfix log for maintaining yet more statistics
# in the database.

# Plus some Fine Interactive commands.

# This script is taylored for Postfix and may not be able
# to parse a different MTA log or message headers.

# Swami Petaramesh, 2009/02/03
#
# Copyright Swami Petaramesh 2009. < swami AT petaramesh DOT org >

version="0.55"

# #########################################################################

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

# #########################################################################

# Runtime parameters: See Usage

# Needed specific binaries:
# - MySQL server, client and database
# - egrep
# - formail (part of procmail package)
# - gawk
# - mktemp
# - sed
# - If it breaks, something else is missing ;-)

# #########################################################################
# # Script parameters                                                     #
# #########################################################################

# Database parameters
dbhost="127.0.0.1"		# MySQL database server
dbname="autolist"		# Database name
dbuser="autolist"		# MySQL database user
dbpwd="AZERTYUIOP"		# MySQL user password
dbprefix="al_"			# Database tables prefix

dbpurge="365"			# Days before unused DB records are purged

bldur="1"	# Auto blacklist initial duration (hours)
blmult="2"	# Auto blacklist duration multiplier
blperm="64"	# Blacklisting duration over which we send a permanent (5xx)
		# error code. Below this we send a temporary (4xx) error.
minspam="2"	# Minimum number of spams for blacklisting IP

# Spam / Good ratio : IP gets blacklisted if there is more or equal than
# 1 spam per defined number of "good" messages.
spamratio="2"

# Our mailservers (1-9) : Put secondary MXes first !
ourmx1="otherserver.otherdomain.org"
# Received: from someserver.some.where (localhost [127.0.0.1]) by someserver.some.where
ourmx2="someserver.somedomain.net"

# Postfix rejection message, 3 parts.
pf_msg1="Sending server at"
# (IP address will be inserted)
pf_msg2="is blacklisted for SPAM until"
# (Blacklisted end time will be inserted)
# pf_msg3="Please refer to http://some.web.page"
pf_msg3="Please resend your message later."

# Our MTA name (that we will grep in MTA log)
mta_name="someserver"

# Shall we insert in database hosts that were seen in MTA log but never
# delivered an actual message (Never seen by DSPAM) ? (yes|no)
insert_mta_only="yes"

# Optional log file. Will be rather verbose if set.
logfile="/var/log/autolist.log"

# #########################################################################
# # Usage                                                                 #
# #########################################################################

usage() {
	echo "$0 Version ${version} is Free Software released under GPL V.2"
	echo "(c) Swami Petaramesh 2009."
	echo ""
	echo "$0 <mode> < file"
	echo ""
	echo "A file needs be piped in only for modes analyzing a file, that can be :"
	echo "- Either an email message classified 'good' or 'spam' ;"
	echo "- Or a Postfix log file."
	echo ""
	echo "Mode :"
	echo "--scan:		Just scans message and output headers"
	echo "--status <IP>	Shows IP status in database."
	echo "--good		Treat as good message."
	echo "--spam		Treat as spam (and possibly blacklist source)."
	echo "--trap		Treat as spamtrap (and possibly blacklist source)."
	echo "--regood		Reclassify as good (and reverses spam count)."
	echo "--respam		Reclassify as spam (and reverse good count)."
	echo "--block <IP>	Doesn't process a message but manually blacklists provided"
	echo "			IP address (without auto expiration)."
	echo "--unlist <IP>	Doesn't process a message but manually unlists provided IP."
	echo "--whitelist <IP>	Doesn't process a message but manually whitelists provided"
	echo "			IP (and prevents it from being auto-blacklisted in the"
	echo "			future)."
	echo "--forget <IP>	Removes IP entry from database."
	echo "--process-log	Process a Postfix log file."
	echo "--db-reset	Create or reset DB tables."
	echo "--db-analyze	Analyses DB."
	echo "--db-optimize	Optimizes DB."
	echo "--db-purge	Purge old records from database."
	exit 1
}

notimpl() {
	echo "Sorry ! This feature is to come but not implemented yet !"
	exit 1
}	

# #########################################################################
# # Functions                                                             #
# #########################################################################

cleanup() {
	for tempfile in ${headerfile} ${mtalog}; do
		[ -f ${tempfile} ] && rm -f ${tempfile}
	done
}

bailout() {
	cleanup
	exit 1
}

# Converts date in Unix epoch seconds to YYYY-MM-DD hh:mm:ss format -------
epochdate() {
	[ $# -eq 1 ] || return 1
	# 2 ways of doing it: Either "date" supports it with "@"
	# or we'll use gawk...
	epochstring="`date -d @$1 '+%Y-%m-%d %H:%M:%S'`"
	#
	# epochstring="`gawk 'BEGIN { print strftime(\"%Y-%m-%d %H:%M:%S\",$1) }'`"
}

# Variables initialization ------------------------------------------------

init_sql_vars() {
	# mysql_options=""
	mysql_options="--connect_timeout=3 --disable-pager --no-beep --batch --silent"

	now="`date '+%Y-%m-%d %H:%M:%S'`"
	nowsec="`date '+%s'`"

	ip_address=""
	hostname=""
	created=""
	last_helo=""
	last_rdns_update=""
	fake_hostname="0"
	host_karma="0"
	listed=" "
	mode="A"
	list_duration="0"
	list_expires=""
	last_seen=""
	times_mta_seen="0"
	last_mta_seen=""
	times_dspam_seen="0"
	last_dspam_seen=""
	times_blacklisted="0"
	last_blacklisted=""
	times_whitelisted="0"
	last_whitelisted=""
	times_mta_accepted="0"
	last_mta_accepted=""
	times_mta_refused="0"
	last_mta_refused=""
	times_greylisted="0"
	last_greylisted=""
	autolist_hits="0"
	last_autolist_hit=""
	last_mta_rej_msg=""
	times_dspam_good="0"
	last_dspam_good=""
	times_dspam_spam="0"
	last_dspam_spam=""
	times_spamtrap="0"
	last_spamtrap=""
	mta_action=""
	last_sender=""
	unblock_sender=""
	unblock_code=""
	last_rcv_mx=""
	last_subject=""
}

init_vars() {
	epochstring=""
	logevent=""
	init_sql_vars
}

make_tempfiles() {
	headerfile=`mktemp -t autolist.headers.tmp.XXXXXXXX`
	mtalog=`mktemp -t autolist.mtalog.tmp.XXXXXXXX`
}

# Calculate blacklisting --------------------------------------------------

black_calc() {

	# Manual forced or already listed: Do nothing
	[ "${mode}" == "M" -o "${listed}" == "B" ] && return

	# Shall we blacklist ?
	local totspam=$(( times_dspam_spam + times_spamtrap ))
	local needspam=$(( times_dspam_good * spamratio ))
	[ ${totspam} -lt ${minspam} -o ${totspam} -lt ${needspam} ] && return

	local blcode="450 #4.7.1 " print_expires=""

	# Yes we shall
	listed="B"
	(( times_blacklisted++ ))
	last_blacklisted="${now}"
	if [ -z "${list_duration}" ]; then
		list_duration="${bldur}"
	elif [ ${list_duration} -lt ${bldur} ]; then
		list_duration="${bldur}"
	else
		list_duration=$(( list_duration * blmult ))
	fi

	[ ${list_duration} -ge ${blperm} ] && blcode="554 #5.7.1 "

	epochdate "$(( list_duration * 3600 + nowsec ))"
	list_expires="${epochstring}"
	print_expires="`date -d \"${list_expires}" '+%Y-%m-%d %H:%M:%S (%z %Z)'`"

	mta_action="${blcode} ${pf_msg1} ${ip_address} ${pf_msg2} ${print_expires}. ${pf_msg3} (Autolist)"

	if [ -n "${logfile}" ]; then
		echo -n "BLACKLISTING: ${mta_action}   " >> ${logfile}
	fi
}

# Calculates a server karma ----------------------------------------------

karma_calc() {
	s_karma=0
	s_karma=$(( times_whitelisted * 50 - times_blacklisted * 20 + times_dspam_good - 3 * times_spamtrap - 2 * times_dspam_spam - times_mta_refused ))
}

# #########################################################################

# Create database tables -------------------------------------------------

# mysqldump -u autolist -p -d -a --add-drop-table autolist > autolist_db_struct

tbcreate() {
	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${dbname} << 'DB_EOT'
-- MySQL dump 10.11
--
-- Host: localhost    Database: autolist
-- ------------------------------------------------------
-- Server version	5.0.67-0ubuntu6-log

/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;

--
-- Table structure for table `al_hosts`
--

DROP TABLE IF EXISTS `al_hosts`;
SET @saved_cs_client     = @@character_set_client;
SET character_set_client = utf8;
CREATE TABLE `al_hosts` (
  `ip_address` varchar(15) NOT NULL,
  `hostname` varchar(63) default NULL,
  `created` timestamp NOT NULL default CURRENT_TIMESTAMP,
  `last_helo` varchar(63) default NULL,
  `last_rdns_update` timestamp NULL default NULL,
  `fake_hostname` tinyint(1) NOT NULL default '0',
  `host_karma` int(11) NOT NULL default '0',
  `listed` enum('','W','B') NOT NULL default '',
  `mode` enum('A','M') NOT NULL default 'A',
  `list_duration` int(11) NOT NULL default '0',
  `list_expires` timestamp NULL default NULL,
  `last_seen` timestamp NULL default NULL,
  `times_mta_seen` int(11) unsigned NOT NULL default '0',
  `last_mta_seen` timestamp NULL default NULL,
  `times_dspam_seen` int(11) unsigned NOT NULL default '0',
  `last_dspam_seen` timestamp NULL default NULL,
  `times_blacklisted` int(11) unsigned NOT NULL default '0',
  `last_blacklisted` timestamp NULL default NULL,
  `times_whitelisted` int(11) unsigned NOT NULL default '0',
  `last_whitelisted` timestamp NULL default NULL,
  `times_mta_accepted` int(11) unsigned NOT NULL default '0',
  `last_mta_accepted` timestamp NULL default NULL,
  `times_mta_refused` int(11) unsigned NOT NULL default '0',
  `last_mta_refused` timestamp NULL default NULL,
  `times_greylisted` int(11) unsigned NOT NULL default '0',
  `last_greylisted` timestamp NULL default NULL,
  `autolist_hits` int(11) unsigned NOT NULL default '0',
  `last_autolist_hit` timestamp NULL default NULL,
  `last_mta_rej_msg` varchar(255) default NULL,
  `times_dspam_good` int(11) unsigned NOT NULL default '0',
  `last_dspam_good` timestamp NULL default NULL,
  `times_dspam_spam` int(11) unsigned NOT NULL default '0',
  `last_dspam_spam` timestamp NULL default NULL,
  `times_spamtrap` int(11) unsigned NOT NULL default '0',
  `last_spamtrap` timestamp NULL default NULL,
  `mta_action` varchar(255) default NULL,
  `last_sender` varchar(63) default NULL,
  `unblock_sender` varchar(63) default NULL,
  `unblock_code` varchar(16) default NULL,
  `last_rcv_mx` varchar(63) default NULL,
  `last_subject` varchar(255) default NULL,
  PRIMARY KEY  (`ip_address`),
  KEY `blacklist_exp` (`listed`,`list_expires`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
SET character_set_client = @saved_cs_client;

--
-- Temporary table structure for view `al_mta_blacklist`
--

DROP TABLE IF EXISTS `al_mta_blacklist`;
/*!50001 DROP VIEW IF EXISTS `al_mta_blacklist`*/;
/*!50001 CREATE TABLE `al_mta_blacklist` (
  `ip_address` varchar(15),
  `mta_action` varchar(255)
) */;

--
-- Final view structure for view `al_mta_blacklist`
--

/*!50001 DROP TABLE `al_mta_blacklist`*/;
/*!50001 DROP VIEW IF EXISTS `al_mta_blacklist`*/;
/*!50001 CREATE ALGORITHM=MERGE */
/*!50013 DEFINER=`autolist`@`localhost` SQL SECURITY DEFINER */
/*!50001 VIEW `al_mta_blacklist` AS select `al_hosts`.`ip_address` AS `ip_address`,`al_hosts`.`mta_action` AS `mta_action` from `al_hosts` where ((`al_hosts`.`listed` = _latin1'B') and (`al_hosts`.`list_expires` > now())) */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;

-- Dump completed on 2009-02-01  9:38:28
DB_EOT
}

# DB tables maintenance ---------------------------------------------------

do_analyze() {
	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${dbname} << 'DB_EOT'
	ANALYZE TABLE al_hosts;
DB_EOT
}

do_optimize() {
	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${dbname} << 'DB_EOT'
	OPTIMIZE TABLE al_hosts;
DB_EOT
}

# Read database -----------------------------------------------------------

db_read_host() {
	[ -n "$1" ] || return 1
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || return 1
	echo "$1" | egrep -q "^127\." && return 1

	local msqlout=""

	# | sed -e "s/^/'/" -e "s/$/'/" -e "s/\t/','/g" )$

	msqlout=`{ mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${mysql_options} ${dbname} | sed -e "s/|/_/g" -e "s/\t/|/g" 2>/dev/null; } << DB_EOT
SELECT ip_address, hostname, created, last_helo, last_rdns_update, fake_hostname, host_karma, listed, mode, list_duration, list_expires, last_seen, times_mta_seen, last_mta_seen, times_dspam_seen, last_dspam_seen, times_blacklisted, last_blacklisted, times_whitelisted, last_whitelisted, times_mta_accepted, last_mta_accepted, times_mta_refused, last_mta_refused, times_greylisted, last_greylisted, autolist_hits, last_autolist_hit, last_mta_rej_msg, times_dspam_good, last_dspam_good, times_dspam_spam, last_dspam_spam, times_spamtrap, last_spamtrap, mta_action, last_sender, unblock_sender, unblock_code, last_rcv_mx, last_subject FROM al_hosts WHERE ip_address = '$1';
DB_EOT
`

# if [ -n "${logfile}" ]; then
# 	echo "----------" >> ${logfile}
# 	echo $msqlout >> ${logfile}
# fi
	IFS="|" read ip_address hostname created last_helo last_rdns_update fake_hostname host_karma listed mode list_duration list_expires last_seen times_mta_seen last_mta_seen times_dspam_seen last_dspam_seen times_blacklisted last_blacklisted times_whitelisted last_whitelisted times_mta_accepted last_mta_accepted times_mta_refused last_mta_refused times_greylisted last_greylisted autolist_hits last_autolist_hit last_mta_rej_msg times_dspam_good last_dspam_good times_dspam_spam last_dspam_spam times_spamtrap last_spamtrap mta_action last_sender unblock_sender unblock_code last_rcv_mx last_subject filler <<< "$msqlout"

}

# Write to database -------------------------------------------------------

db_write_host() {

	local msqlin="" mysqlresult=0

	hostname="${hostname//\\/\\\\}"
	last_helo="${last_helo//\\/\\\\}"
	last_sender="${last_sender//\\/\\\\}"
	last_mta_rej_msg="${last_mta_rej_msg//\\/\\\\}"
	unblock_sender="${unblock_sender//\\/\\\\}"
	last_subject="${last_subject//\\/\\\\}"

	msqlin="REPLACE INTO al_hosts SET \
	ip_address='${ip_address}', \
	hostname='${hostname//\'/\'}', \
	created='${created}', \
	last_helo='${last_helo//\'/\'}', \
	last_rdns_update='${last_rdns_update}', \
	fake_hostname='${fake_hostname}', \
	host_karma='${host_karma}', \
	listed='${listed}', \
	mode='${mode}', \
	list_duration='${list_duration}', \
	list_expires='${list_expires}', \
	last_seen='${last_seen}', \
	times_mta_seen='${times_mta_seen}', \
	last_mta_seen='${last_mta_seen}', \
	times_dspam_seen='${times_dspam_seen}', \
	last_dspam_seen='${last_dspam_seen}', \
	times_blacklisted='${times_blacklisted}', \
	last_blacklisted='${last_blacklisted}', \
	times_whitelisted='${times_whitelisted}', \
	last_whitelisted='${last_whitelisted}', \
	times_mta_accepted='${times_mta_accepted}', \
	last_mta_accepted='${last_mta_accepted}', \
	times_mta_refused='${times_mta_refused}', \
	last_mta_refused='${last_mta_refused}', \
	times_greylisted='${times_greylisted}', \
	last_greylisted='${last_greylisted}', \
	autolist_hits='${autolist_hits}', \
	last_autolist_hit='${last_autolist_hit}', \
	last_mta_rej_msg='${last_mta_rej_msg//\'/\'}', \
	times_dspam_good='${times_dspam_good}', \
	last_dspam_good='${last_dspam_good}', \
	times_dspam_spam='${times_dspam_spam}', \
	last_dspam_spam='${last_dspam_spam}', \
	times_spamtrap='${times_spamtrap}', \
	last_spamtrap='${last_spamtrap}', \
	mta_action='${mta_action}', \
	last_sender='${last_sender//\'/\'}', \
	unblock_sender='${unblock_sender//\'/\'}', \
	unblock_code='${unblock_code}', \
	last_rcv_mx='${last_rcv_mx}', \
	last_subject='${last_subject//\'/\'}'"

	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${mysql_options} ${dbname} <<< ${msqlin}
	mysqlresult="$?"

	if [ ${mysqlresult} -ne 0 ]; then
		if [ -n "${logfile}" ]; then
			echo "MySQL ERROR processing: ${msqlin}">> ${logfile}
		else
			echo "MySQL ERROR processing: ${msqlin}"
		fi
	fi
	
	return ${mysqlresult}
}

# Expire database blacklisted status --------------------------------------

db_expire_list() {

	local mysqlcmd="UPDATE al_hosts SET listed = ' ' WHERE listed = 'B' AND mode != 'M' AND list_expires < NOW();"

	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${mysql_options} ${dbname} <<< ${mysqlcmd}
}

# Host database status  ---------------------------------------------------

status() {
	[ -n "$1" ] || usage
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || usage
	echo "$1" | egrep -q "^127\." && usage

	init_vars
	db_expire_list
	db_read_host $1
	# echo "Code retour: $?"

	echo ""

	if [ -n "${ip_address}" ]; then
		echo -n "IP: ${ip_address}	"
		echo -n "Host: ${hostname} "
		[ "${fake_hostname}" != "0" ] && echo -n "(FAKE !)	" || echo -n "	"
		echo -n "Helo: ${last_helo}	"
		echo -n "Created: ${created}	"
		echo -n "Last seen: ${last_seen}	"
		echo "Last DNS: ${last_rdns_update}"
		echo -n "Karma: ${host_karma}	"
		echo -n "Listed: ${listed}${mode}	"
		echo -n "Duration: ${list_duration}	"
		echo -n "Until: ${list_expires}	"
		echo -n "Times blacklisted: ${times_blacklisted}	"
		echo -n "${last_blacklisted}	"
		echo -n "Times whitelisted: ${times_whitelisted}	"
		echo "${last_whitelisted}"
		echo ""
		echo -n "Times Postfix seen: ${times_mta_seen}	"
		echo -n "Last: ${last_mta_seen}	"
		echo -n "Accepted: ${times_mta_accepted}	"
		echo -n "${last_mta_accepted}	"
		echo -n "Refused: ${times_mta_refused}	"
		echo -n "${last_mta_refused}	"
		echo -n "Greylisted: ${times_greylisted}	"
		echo "${last_greylisted}"
		echo "Last rejection message : ${last_mta_rej_msg}"
		echo -n "Times DSPAM seen: ${times_dspam_seen}	"
		echo -n "Last: ${last_dspam_seen}	"
		echo -n "Good: ${times_dspam_good}	"
		echo -n "${last_dspam_good}	"
		echo -n "Spam: ${times_dspam_spam}	"
		echo -n "${last_dspam_spam}	"
		echo -n "Spamtrap: ${times_spamtrap}	"
		echo "${last_spamtrap}"
		echo "Postfix action: ${mta_action}"
		echo -n "Last receiver MX: ${last_rcv_mx}	"
		echo "Last sender: ${last_sender}"
		echo "Last Subject: ${last_subject}"
	else
		echo "IP: ${1} NOT FOUND !"
	fi

	echo ""
}

# Scans provided email message --------------------------------------------

parsemsg() {
	mdate=""; mrpath=""; mhelo=""; mhostname=""; msourceip=""; mrmx=""; msubj=""

	formail -c -X Return-Path: -X Date: -X Received: -X Subject: \
		| egrep "^(Return-Path:|Date:|Subject:|Received: from .*[[:space:]]+by (${ourmx1}|${ourmx2}|${ourmx3}|${ourmx4}|${ourmx5}|${ourmx6}|${ourmx7}|${ourmx8}|${ourmx9}))" \
		| egrep -v "^Received: from (${ourmx1}|${ourmx2}|${ourmx3}|${ourmx4}|${ourmx5}|${ourmx6}|${ourmx7}|${ourmx8}|${ourmx9}) " > ${headerfile}

	mdate=`gawk '/^Date: /{print $2 " " $3 " "  $4 " " $5 " " $6 " " $7 " " $8 " " $9 " " $10}' ${headerfile} | sed -e 's/\s*$//' | head -n 1`
	mdate="${mdate:0:255}"

	mrpath=`gawk 'BEGIN{FS="[<>]"} /^Return-Path: /{print $2}' ${headerfile} | head -n 1`
	mrpath="${mrpath:0:63}"

	msubj=`egrep '^Subject:' ${headerfile} | head -n 1 | cut -d' ' -f2-`

	mrmx=`gawk --re-interval 'BEGIN{FS="[[:space:]]by "} /^Received: from .*\(.*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\].*\).*[[:space:]]+by /{print $2}' ${headerfile} | sed -e 's/\s*$//' | head -n 1 | cut -d' ' -f1`
	mrmx="${mrmx:0:63}"

	mhostname=`gawk --re-interval 'BEGIN{FS="[([]"} /^Received: from .*\(.*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\].*\).*[[:space:]]+by/{print $2}' ${headerfile} | sed -e 's/\s*$//' | head -n 1`
	mhostname="${mhostname:0:255}"

	mhelo=`gawk --re-interval '/^Received: from .*\(.*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\].*\).*[[:space:]]+by/{print $3}' ${headerfile} | head -n 1`
	mhelo="${mhelo:0:63}"

	msourceip=`gawk --re-interval 'BEGIN{FS="[][]"} /^Received: from .*\(.*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\].*\).*[[:space:]]+by/{print $2}' ${headerfile} | head -n 1`
	msourceip="${msourceip:0:15}"

	if echo "${msourceip}" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$"; then
		if ! echo "${msourceip}" | egrep -q "^127\."; then
			return 0
		else
			msourceip=""
			return 1
		fi
	else
		msourceip=""
		return 1
	fi

}

scanmsg() {
	local scanresult

	echo "Scanning Message headers :"

	init_vars
	make_tempfiles

	parsemsg
	scanresult="$?"

	cat ${headerfile}
	echo ""

	[ ${scanresult} -eq 0 ] && {
		echo -n "Source IP: ${msourceip}	"
	} || {
		echo -n "Source IP NOT FOUND !	"
	}

	echo -n "Host: ${mhostname}	"
	echo -n "Helo: ${mhelo}	"
	echo -n "Date: ${mdate}	"
	echo "MX: ${mrmx}"
	echo -n "Return-Path: ${mrpath}	"
	echo "Subject: ${msubj}"

	cleanup
	return ${scanresult}
}

# Process provided email message ------------------------------------------

process_msg() {
	# if [ -n "${logfile}" ]; then
	#	echo "Entering process_msg() $@">> ${logfile}
	# fi

	[ $# -eq 1 ] || return 1
	echo "$1" | egrep -q "^([GST]|R[GS])$" || return 1

	init_vars
	make_tempfiles

	# Parse message
	parsemsg
	[ $? -eq 0 ] || bailout

	if [ -n "${logfile}" ]; then
		echo "Parsed msg. IP: ${msourceip} Host: ${mhostname} Helo: ${mhelo} Return-Path: ${mrpath} MX: ${mrmx} Subject: ${msubj}" >> ${logfile}
	fi

	local last_seen_sec=0 db_write_result=0

	# Read host record from database
	db_read_host ${msourceip}

	if [ -n "${logfile}" ]; then
		echo "----------" >> ${logfile}
		echo -n "Read DB. " >> ${logfile}
		if  [ -n "${ip_address}" ]; then
			echo "IP: ${ip_address} Host: ${hostname} Helo: ${last_helo} Created: ${created} Last seen: ${last_seen}" >> ${logfile}
			echo "Karma: ${host_karma} Listed: ${listed}${mode} Duration: ${list_duration} Until: ${list_expires} Times blacklisted: ${times_blacklisted} ${last_blacklisted} Times whitelisted: ${times_whitelisted} ${last_whitelisted}" >> ${logfile}
			echo "Times Postfix seen: ${times_mta_seen} Last: ${last_mta_seen} Accepted: ${times_mta_accepted} ${last_mta_accepted} Refused: ${times_mta_refused} ${last_mta_refused} Greylisted: ${times_greylisted} ${last_greylisted}" >> ${logfile}
			echo "Times DSPAM seen: ${times_dspam_seen} Last: ${last_dspam_seen} Good: ${times_dspam_good} ${last_dspam_good} Spam: ${times_dspam_spam} ${last_dspam_spam} Spamtrap: ${times_spamtrap} ${last_spamtrap}" >> ${logfile}
		else
			echo "IP ${msourceip} Not found in database." >> ${logfile}
		fi
	fi

	# Initialize record if no record was found
	[ -z "${ip_address}" ] && init_sql_vars
	ip_address="${msourceip}"

	[ -n "${mhostname}" ] && hostname="${mhostname}"
	[ -n "${mhelo}" ] && last_helo="${mhelo}"
	[ -n "${mhelo}" ] && last_rdns_update="${now}"
	[ -n "${mrpath}" ] && last_sender="${mrpath}"
	[ -n "${mrmx}" ] && last_rcv_mx="${mrmx}"
	[ -n "${msubj}" ] && last_subject="${msubj}"

	case "$1" in
		G)
			(( times_mta_accepted++ ))
			last_mta_accepted="${now}"
			(( times_dspam_seen++ ))
			last_dspam_seen="${now}"
			(( times_dspam_good++ ))
			last_dspam_good="${now}"

			# Now useless as we expire systematically all old records
			# if [ "${mode}" != "M" -a "${listed}" == "B"] ; then
			#	local list_expires_sec=0
			#	list_expires_sec="`date -d \"${list_expires}\" '+%s' 2>/dev/null || echo 0`"
			#	[ ${list_expires_sec} -le ${nowsec} ] && listed=" "
			#	if [ -n "${logfile}" ]; then
			#		echo -n "CLEARING OLD BLACKLISTING.   " >> ${logfile}
			#	fi
			# fi
			;;
		S)
			(( times_mta_accepted++ ))
			last_mta_accepted="${now}"
			(( times_dspam_seen++ ))
			last_dspam_seen="${now}"
			(( times_dspam_spam++ ))
			last_dspam_spam="${now}"
			black_calc
			;;
		T)
			(( times_mta_accepted++ ))
			last_mta_accepted="${now}"
			(( times_spamtrap++ ))
			last_spamtrap="${now}"
			black_calc
			;;
		RG)
			(( times_dspam_good++ ))
			last_dspam_good="${now}"
			(( times_dspam_spam-- ))

			if [ "${mode}" != "M" -a "${listed}" == "B"] ; then
				listed=" "
				list_expires="${now}"
				(( times_blacklisted-- ))

				if [ -n "${list_duration}" ]; then
					if [ ${list_duration} -gt ${bldur} ]; then
						(( list_duration / blmult ))
					fi
					if [ ${list_duration} -lt ${bldur} ]; then
						list_duration="0"
					fi
				fi
				if [ -n "${logfile}" ]; then
					echo -n "CLEARING OLD BLACKLISTING.   " >> ${logfile}
				fi
			fi
			;;
		RS)
			(( times_dspam_spam++ ))
			last_dspam_spam="${now}"
			(( times_dspam_good-- ))
			black_calc
			;;
	esac

	[ -n "${mode}" ] || mode="A"

	karma_calc
	host_karma="${s_karma}"
	[ -n "${mta_action}" -a "${listed}" != "B" ] && mta_action=""

	[ -z "${created}" ] && created="${now}"

	[ -n "${last_seen}" ] && last_seen_sec="`date -d \"${last_seen}\" '+%s'`"
	
	# echo "Now: ${nowsec} Last: ${last_seen} Secs: ${last_seen_sec}"

	[ ${nowsec} -gt ${last_seen_sec} ] && last_seen="${now}"

	if [ -n "${logfile}" ]; then
		echo "New host karma : ${host_karma}" >> ${logfile}
	fi

	# Writes host record to database
	db_write_host
	db_write_result="$?"

	db_expire_list

	cleanup
	return ${db_write_result}
}

# Scans MTA log -----------------------------------------------------------

process_log() {
	# if [ -n "${logfile}" ]; then
	#	echo "Entering process_log() $@">> ${logfile}
	# fi


	make_tempfiles
	init_vars

	local logline="" logevent="" linedate="" m_sec="" test_sec="" connip="" mhostname="" mhelo="" mfrom="" mrej="" alist_hit=0

	egrep "^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} (postfix/smtpd\[[0-9]+\]: (NOQUEUE:|connect from)|sqlgrey: grey: (new|early reconnect|throttling):)" > ${mtalog}

	while read logline; do
		# echo "${logline}"

		logevent=""; linedate=""; connip=""; mhostname=""; mhelo=""; mfrom=""; mrej=""; alist_hit=0

		if echo "${logline}" | egrep -q "^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} postfix/smtpd\[[0-9]+\]: connect from"; then
			logevent="CONNECT"
		elif echo "${logline}" | egrep -q "^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} sqlgrey: grey: (new|early reconnect|throttling):"; then
			logevent="GREYLIST"
		elif echo "${logline}" | egrep -q "^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} postfix/smtpd\[[0-9]+\]: NOQUEUE:"; then
			logevent="REJECT"
		else
			logevent="UNKNOWN"
		fi

		linedate="`echo \"${logline}\" | egrep -o \"^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2}\"`"
		linedate="`date -d \"${linedate}\" '+%Y-%m-%d %H:%M:%S'`"

		# echo "${logevent} ${linedate}"

		case ${logevent} in
			CONNECT)
				connip=`echo "${logline}" | gawk --re-interval "BEGIN{FS=\"[][]\"} /^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} postfix\/smtpd\[[0-9]+\]: connect from .*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\]/{print \\\$4}"`
				# echo "${logevent} ${linedate} IP: ${connip}"
				;;
			GREYLIST)
				connip=`echo "${logline}" | gawk --re-interval "BEGIN{FS=\"[()]\"} /^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} sqlgrey: grey: (new|early reconnect|throttling): .*\([1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\)/{print \\\$2}"`
				mfrom=`echo "${logline}" | gawk --re-interval "BEGIN{FS=\", | ->\"} /^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} sqlgrey: grey: (new|early reconnect|throttling): .*\([1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\), [[:graph:]]+@[[:graph:]]+ ->/{print \\\$2}"`
				# echo "${logevent} ${linedate} IP: ${connip} From: ${mfrom}"
				;;
			REJECT)
				# echo "${logline}"
				connip=`echo "${logline}" | gawk --re-interval "BEGIN{FS=\"[][]\"} /^[A-Z][a-z]+ +[0-9]{1,2} ([0-9]{2}:){2}[0-9]{2} ${mta_name} postfix\/smtpd\[[0-9]+\]: NOQUEUE: reject: [A-Z]+ from .*\[[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}\]:/{print \\\$4}"`
				mhostname="`echo \"${logline}\" | egrep -o 'reject: RCPT from [[:alnum:]._-]+\[' | cut -d' ' -f4`"; mhostname="${mhostname%%\[*}"
				mhelo="`echo \"${logline}\" | egrep -o ' helo=<[[:graph:]]+>' | cut -d'<' -f2`"; mhelo="${mhelo%%>*}"
				mfrom="`echo \"${logline}\" | egrep -o ' from=<[[:graph:]]+>' | cut -d'<' -f2`"; mfrom="${mfrom%%>*}"
				mrej="`echo \"${logline}\" | egrep -o '[45][0-9]{2} [^;]+;'`"
				echo "${mrej}" | egrep -q "\(Autolist\)" && alist_hit=1

				# echo "${logevent} ${linedate} IP: ${connip} Host: ${mhostname} Helo: ${mhelo} From: ${mfrom} Alist_Hit: ${alist_hit}"
				# echo "-- ${mrej}"
				;;
		esac

		# Could we build a record with a correct source IP address ?
		echo "${connip}" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || continue
		echo "${connip}" | egrep -q "^127\." && continue

		# Read host record from database
		db_read_host ${connip}

		# Do we need to bother if no previous record exists ?
		[ -z "${ip_address}" -a "${insert_mta_only}" != "yes" ] && continue

		if [ -n "${logfile}" ]; then
			echo "Processing: ${logevent} ${linedate} IP: ${connip} Host: ${mhostname} Helo: ${mhelo} From: ${mfrom} Alist_Hit: ${alist_hit}" >> ${logfile}
			[ -n "${mrej}" ] && echo "-- ${mrej}" >> ${logfile}
		fi

		# Initialize record if no record was found
		[ -z "${ip_address}" ] && init_sql_vars

		m_sec="`date -d \"${linedate}\" '+%s' 2>/dev/null || echo 0`"

		ip_address="${connip}"
	
		test_sec="`date -d \"${created}\" '+%s' 2>/dev/null || echo 0`"
		[ ${m_sec} -ge ${test_sec} ] && created="${linedate}"

		test_sec="`date -d \"${last_seen}\" '+%s' 2>/dev/null || echo 0`"

		if [ ${m_sec} -ge ${test_sec} -o -z "${last_helo}" ]; then
			[ -n "${mhelo}" ] && last_helo="${mhelo}"
		fi

		if [ ${m_sec} -ge ${test_sec} -o -z "${last_mta_rej_msg}" ]; then
			[ -n "${mrej}" ] && last_mta_rej_msg="${mrej}"
		fi

		if [ ${m_sec} -ge ${test_sec} -o -z "${last_sender}" ]; then
			[ -n "${mfrom}" ] && last_sender="${mfrom}"
		fi

		if [ -n "${mhostname}" ]; then
			test_sec="`date -d \"${last_rdns_update}\" '+%s' 2>/dev/null || echo 0`"
			if [ ${m_sec} -ge ${test_sec} ]; then
				hostname="${mhostname}"
				last_rdns_update="${linedate}"
				fake_hostname="0"
			fi
		fi

		case ${logevent} in
			CONNECT)
				test_sec="`date -d \"${last_seen}\" '+%s' 2>/dev/null || echo 0`"
				[ ${m_sec} -gt ${test_sec} ] && last_seen="${linedate}"

				test_sec="`date -d \"${last_mta_seen}\" '+%s' 2>/dev/null || echo 0`"
				if [ ${m_sec} -gt ${test_sec} ]; then
					(( times_mta_seen++ ))
					last_mta_seen="${linedate}"
				fi
				;;
			GREYLIST)
				test_sec="`date -d \"${last_greylisted}\" '+%s' 2>/dev/null || echo 0`"
				if [ ${m_sec} -gt ${test_sec} ]; then
					(( times_greylisted++ ))
					last_greylisted="${linedate}"
				fi
				;;
			REJECT)
				test_sec="`date -d \"${last_mta_refused}\" '+%s' 2>/dev/null || echo 0`"
				if [ ${m_sec} -gt ${test_sec} ]; then
					(( times_mta_refused++ ))
					last_mta_refused="${linedate}"
				fi

				if [ "${alist_hit}" == "1" ]; then
					test_sec="`date -d \"${last_autolist_hit}\" '+%s' 2>/dev/null || echo 0`"
					if [ ${m_sec} -gt ${test_sec} ]; then
						(( autolist_hits++ ))
						last_autolist_hit="${linedate}"
					fi
				fi
				;;
		esac

		karma_calc
		host_karma="${s_karma}"

		# Writes host record to database
		db_write_host

	done < ${mtalog}

	cleanup
}

# Manual operations on IP addresses ---------------------------------------

block() {
	[ -n "$1" ] || usage
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || usage
	echo "$1" | egrep -q "^127\." && usage

	init_vars
	db_read_host $1

	ip_address="$1"
	mode="M"
	listed="B"
	(( times_blacklisted++ ))
	last_blacklisted="${now}"
	if [ -z "${list_duration}" ]; then
		list_duration="${bldur}"
	elif [ ${list_duration} -lt ${bldur} ]; then
		list_duration="${bldur}"
	fi
	list_expires="2037-12-31 23:59:59"

	mta_action="554 #5.7.1 ${pf_msg1} ${ip_address} is blacklisted for SPAM. ${pf_msg3} (Autolist)"

	karma_calc
	host_karma="${s_karma}"

	[ -z "${created}" ] && created="${now}"

	# Writes host record to database
	db_write_host
	db_write_result="$?"

	db_expire_list

	cleanup
	return ${db_write_result}
}

unlist() {
	[ -n "$1" ] || usage
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || usage
	echo "$1" | egrep -q "^127\." && usage

	init_vars
	db_read_host $1

	ip_address="$1"
	mode=" "
	listed=" "
	local list_expires_sec="`date -d \"${list_expires}\" '+%s' 2>/dev/null || echo 0`"
	[ ${list_expires_sec} -gt ${nowsec} ] && list_expires=${now}
	mta_action=""

	[ -z "${created}" ] && created="${now}"

	# Writes host record to database
	db_write_host
	db_write_result="$?"

	db_expire_list

	cleanup
	return ${db_write_result}
}

whitelist() {
	[ -n "$1" ] || usage
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || usage
	echo "$1" | egrep -q "^127\." && usage

	init_vars
	db_read_host $1

	ip_address="$1"
	mode="M"
	listed="W"
	(( times_whitelisted++ ))
	last_whitelisted="${now}"
	local list_expires_sec="`date -d \"${list_expires}\" '+%s' 2>/dev/null || echo 0`"
	[ ${list_expires_sec} -gt ${nowsec} ] && list_expires=${now}
	mta_action=""

	karma_calc
	host_karma="${s_karma}"

	[ -z "${created}" ] && created="${now}"

	# Writes host record to database
	db_write_host
	db_write_result="$?"

	db_expire_list

	cleanup
	return ${db_write_result}
}

forget() {
	[ -n "$1" ] || usage
	echo "$1" | egrep -q "^[1-9][0-9]{0,2}(\.[0-9]{1,3}){3}$" || usage
	echo "$1" | egrep -q "^127\." && usage

	init_vars

	db_expire_list

	db_read_host $1

	local mysqlcmd="DELETE FROM al_hosts WHERE ip_address = '$1';"

	if [ -z "${ip_address}" ]; then
		echo "IP: ${1} NOT FOUND !"
		return 1
	fi

	mysql -h ${dbhost} -u ${dbuser} -p${dbpwd} ${mysql_options} ${dbname} <<< ${mysqlcmd}

	return $?
}

# #########################################################################
# # Main                                                                  #
# #########################################################################

[ $# -ge 1 ] || usage

if [ -n "${logfile}" ]; then
	echo "==================================================" >> ${logfile}	
	echo "`date` : $0 called with $@">> ${logfile}
fi

case $1 in
	--scan)
		[ $# -ne 1 ] && usage
		scanmsg
		;;
	--status)
		[ $# -ne 2 ] && usage
		status $2
		;;
	--good)
		[ $# -ne 1 ] && usage
		process_msg G
		;;
	--spam)
		[ $# -ne 1 ] && usage
		process_msg S
		;;
	--trap)
		[ $# -ne 1 ] && usage
		process_msg T
		;;
	--regood)
		[ $# -ne 1 ] && usage
		process_msg RG
		;;
	--respam)
		[ $# -ne 1 ] && usage
		process_msg RS
		;;
	--block)
		[ $# -ne 2 ] && usage
		block $2
		;;
	--unlist)
		[ $# -ne 2 ] && usage
		unlist $2
		;;
	--whitelist)
		[ $# -ne 2 ] && usage
		whitelist $2
		;;
	--forget)
		[ $# -ne 2 ] && usage
		forget $2
		;;
	--process-log)
		[ $# -ne 1 ] && usage
		process_log
		;;
	--db-reset)
		[ $# -ne 1 ] && usage
		tbcreate
		;;
	--db-analyze)
		[ $# -ne 1 ] && usage
		do_analyze
		;;
	--db-optimize)
		[ $# -ne 1 ] && usage
		do_optimize
		;;
	--db-purge)
		[ $# -ne 1 ] && usage
		notimpl
		;;
	*)
		usage
		;;
esac

exit $?