jaromail

a commandline tool to easily and privately handle your e-mail
git clone git://parazyd.org/jaromail.git
Log | Files | Refs | Submodules | README

commit fa80ed43bb404dadb434ebe2b55d8db6af054045
parent cc958892a5af48c04733acb72745a986402613a2
Author: Jaromil <jaromil@dyne.org>
Date:   Wed,  7 May 2014 00:28:37 +0200

Complete rewrite of the filter engine

Eliminated use of procmail. Maildir filtering now works using
zsh arrays and is natively implemented by jaromail.
Sieve filters are still generated.

More refactoring or related will follow.

Diffstat:
Msrc/jaro | 39+++++++++++++++++++++++----------------
Msrc/zlibs/addressbook | 12+++++++++---
Msrc/zlibs/email | 15++++++++-------
Msrc/zlibs/filters | 398+++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------
Msrc/zlibs/maildirs | 75+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------
Msrc/zlibs/stats | 4++--
6 files changed, 375 insertions(+), 168 deletions(-)

diff --git a/src/jaro b/src/jaro @@ -68,8 +68,6 @@ typeset -h hostname addressbook addressbook_tmp # global array for maildirs (filled by list_maildirs) typeset -al maildirs -# global variable formail cache (used by rmdupes) -typeset -h formail_cache # global variable for mutt binary typeset -h mutt pgpewrap dotlock @@ -564,6 +562,7 @@ main() subcommands_opts[rmdupes]="" subcommands_opts[merge]="" subcommands_opts[filter]="" + subcommands_opts[deliver]="" subcommands_opts[passwd]="" subcommands_opts[cert]="" @@ -666,13 +665,18 @@ main() case "$subcommand" in compose) compose ${PARAM} ;; queue) queue ${PARAM} ;; - fetch) fetch ${PARAM} ;; # was checking is_online + fetch) fetch ${PARAM};; # was checking is_online send) send ${PARAM} ;; # was checking is_online peek) peek ${PARAM} ;; # was checking is_online later) later ${PARAM} ;; - update|init) update ;; + update|init) + init_inbox + update_filters + update_mutt + update_sieve + ;; help) CLEANEXIT=0; usage ;; @@ -680,23 +684,25 @@ main() stat) CLEANEXIT=0; stats ${PARAM} ;; - complete) CLEANEXIT=0; complete ${PARAM} ;; - isknown) CLEANEXIT=0; isknown ${PARAM} ;; - learn) CLEANEXIT=0; learn ${PARAM} ;; - forget) CLEANEXIT=0; forget ${PARAM} ;; - list) CLEANEXIT=0; list_addresses ${PARAM} ;; + complete) CLEANEXIT=0; complete ${PARAM} ;; + isknown) CLEANEXIT=0; isknown ${PARAM} ;; + learn) CLEANEXIT=0; learn ${PARAM} ;; + forget) CLEANEXIT=0; forget ${PARAM} ;; + list) CLEANEXIT=0; list_addresses ${PARAM} ;; + import) import_addressbook ${PARAM} ;; - "export") export_vcard ${PARAM} ;; - abook) edit_abook ${PARAM} ;; + "export") export_vcard ${PARAM} ;; + abook) edit_abook ${PARAM} ;; - edit) CLEANEXIT=0; edit_file ${PARAM} ;; - open) CLEANEXIT=0; open_folder ${PARAM} ;; + edit) CLEANEXIT=0; edit_file ${PARAM} ;; + open) CLEANEXIT=0; open_folder ${PARAM} ;; preview) CLEANEXIT=0; preview_file ${PARAM} ;; - backup) backup ${PARAM} ;; + backup) backup ${PARAM} ;; rmdupes) rmdupes ${PARAM} ;; - merge) merge ${PARAM} ;; - filter) filter ${PARAM} ;; + merge) merge ${PARAM} ;; + filter) filter_maildir incoming ${PARAM} ;; + deliver) deliver ${PARAM} ;; passwd) change_password ${PARAM} ;; @@ -722,6 +728,7 @@ main() } ;; esac + exitcode=$? return 0 } diff --git a/src/zlibs/addressbook b/src/zlibs/addressbook @@ -154,13 +154,19 @@ complete() { } isknown() { - func "is known in $list: (string from stdin)" head="`${WORKDIR}/bin/fetchaddr -x From -a`" + email="${head[(ws:,:)1]}" exitcode=1 + { test "$email" = "" } && { return 1 } + lookup="`lookup_email ${email}`" - { test "$lookup" != "" } && { exitcode=0 } - act "Email <$email> found in $list with id $lookup" + + { test "$lookup" = "" } || { + func "isknown() found <$email> in $list (id $lookup)" + return 0 } + + return 1 } learn() { diff --git a/src/zlibs/email b/src/zlibs/email @@ -146,6 +146,7 @@ queue() { { test -r "${TMPDIR}/${queue_body}.mail" } && { ${=rm} "${TMPDIR}/${queue_body}.mail" } + return 0 } ########### @@ -216,7 +217,7 @@ fetch() { if ! [ -z $folders ]; then # add folder configuration fmconf+=(" folder ${folders} "); fi - fmconf+=(" ssl warnings 3600 and wants mda \"procmail -m $PROCMAILDIR/rc\" ") + fmconf+=(" ssl warnings 3600 and wants mda \"jaro -q deliver\" ") if [ "$cert" = "check" ]; then # we now use system-wide certs @@ -263,18 +264,18 @@ fetch() { unlock "$MAILDIRS/logs/procmail-${datestamp}.log" fi - act "please wait while downloading mails..." + act "please wait while downloading mails to incoming..." print " $fmconf " | fetchmail -f - unset $fmconf + filter_maildir incoming - - total=`mailstat -k "$MAILDIRS/logs/procmail.log" | tail -n1 | awk '{print $2}'` - briefing=`mailstat -kt "$MAILDIRS/logs/procmail.log" | awk '!/procmail/ { print " " $2 "\t" $3 }'|sort -nr` - notice "$total emails fetched" - print "${briefing}" + # total=`mailstat -k $MAILDIRS/logs/procmail.log | tail -n1 | awk '{print $2}'` + # briefing=`mailstat -kt $MAILDIRS/logs/procmail.log |awk '!/procmail/ { print " " $2 "\t" $3 }'|sort -nr` + # notice "$total emails fetched" + # print "${briefing}" } # DRYRUN diff --git a/src/zlibs/filters b/src/zlibs/filters @@ -4,7 +4,7 @@ # # a tool to easily and privately handle your e-mail communication # -# Copyleft (C) 2010-2012 Denis Roio <jaromil@dyne.org> +# Copyleft (C) 2010-2014 Denis Roio <jaromil@dyne.org> # # This source code is free software; you can redistribute it and/or # modify it under the terms of the GNU Public License as published by @@ -20,35 +20,246 @@ # this source code; if not, write to: # Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -update() { - notice "Updating all configurations and filters" - # this function should: - # parse all filters - # generate procmailrc - # generate muttrc - # backup what's too old in the maildirs - # ... - - # debug configuration - func "MAILDIRS: $MAILDIRS" - func "WORKDIR: $WORKDIR" - func "MUTTDIR: $MUTTDIR" - func "PROCMAILDIR: $PROCMAILDIR" +################### +# Filters workflow: +# +# 1. Check if From in blacklist -> zz.blacklist +# 2. Check if /Sender.*bounce/ -> zz.bounces +# 3. Check if From Filters match -> own.setup +# 4. Check if To Filters match -> own.setup +# 5. Check if From in whitelist -> known +# 6. Check if /X-Spam-Flag.*YES/ -> zz.spam +# 7. Check if To own address -> priv +# 8. All the rest -> unsorted +# + +# load zsh filter cache arrays +{ test -r $MAILDIRS/cache/filters } && { + source $MAILDIRS/cache/filters } +init_inbox() { # make sure maildirs where to put mails exist - ${=mkdir} "$MAILDIRS" - maildirmake "$MAILDIRS/known" - maildirmake "$MAILDIRS/sent" - maildirmake "$MAILDIRS/priv" - maildirmake "$MAILDIRS/postponed" - maildirmake "$MAILDIRS/unsorted" - maildirmake "$MAILDIRS/unsorted.ml" - ${=mkdir} "$MAILDIRS/outbox" - - - ###### - # MUTT - act "configuring Mutt's environment" + ${=mkdir} $MAILDIRS + maildirmake $MAILDIRS/incoming + maildirmake $MAILDIRS/known + maildirmake $MAILDIRS/sent + maildirmake $MAILDIRS/priv + maildirmake $MAILDIRS/postponed + maildirmake $MAILDIRS/unsorted + maildirmake $MAILDIRS/unsorted.ml + maildirmake $MAILDIRS/remember + maildirmake $MAILDIRS/outbox + + ${=mkdir} $MAILDIRS/cache + ${=mkdir} $MAILDIRS/logs + ${=mkdir} $MAILDIRS/tmp + + return 0 +} + +# short utility to print only mail headers +hdr() { + { test -r "$1" } || { + error "hdr() called on non existing file: $1" + return 1 } + awk '{ print $0 } +/^$/ { exit }' "$1" +} + +update_filters() { + { test -r "$MAILDIRS/Filters.txt" } || { + error "Filters not found in $MAILDIRS/Filters.txt" + return 1 } + + notice "Updating filters..." + + ff="$MAILDIRS/cache/filters" + ${=mkdir} "$MAILDIRS/cache" + + { test -r "$ff" } && { rm -f "$ff" } + newlock "$ff" + cat <<EOF >> "$ff" +# automatically generated by jaromail +typeset -Al filter_from +typeset -Al filter_to + +EOF + ffilters=`cat "$MAILDIRS/Filters.txt" | awk ' + /^#/ {next} + /^./ { print $1 ";" $2 ";" $3 ";" $4 }'` + + # insert filter rules in the cache + for f in ${(f)ffilters}; do + header="${f[(ws:;:)1]}" + regexp="${f[(ws:;:)2]}" + action="${f[(ws:;:)3]}" + destination="${f[(ws:;:)4]}" + case $header in + to) + cat <<EOF >> "$ff" +filter_to+=("${regexp}" "${destination}") +EOF + func "from: <${regexp}> -> ${destination}" + maildirmake $MAILDIRS/$destination + ;; + from) + cat <<EOF >> "$ff" +filter_from+=("${regexp}" "${destination}") +EOF + func "to: <${regexp}> -> ${destination}" + maildirmake $MAILDIRS/$destination + ;; + + *) + error "invalid filter: $f" + ;; + esac + done + + unlock "$ff" + zcompile "$ff" + source "$ff" + return 0 +} + +filter_maildir() { + + # for safety we bail out in case the final fallback + # maildir is not existing. unsorted should always + # be there. + maildircheck "$MAILDIRS/unsorted" + { test $? = 0 } || { + error "Invalid fallback maildir destination, operation aborted." + func "Returning error to caller." + return 1; } + + # loads up the filter cache (zsh compiled arrays) + { test -r "$MAILDIRS/cache/filters" } && { + source $MAILDIRS/cache/filters + ownfilters=1 } + + if [ "$1" = "" ]; then + input="incoming" + else + input="$1" + fi + + maildircheck "$MAILDIRS/$input" + { test $? = 0 } || { + error "Invalid maildir to filter: $input" + return 1; } + + numm=`${=find} "$MAILDIRS/$input" -maxdepth 2 -type f|wc -l` + mails=`${=find} "$MAILDIRS/$input" -maxdepth 2 -type f` + + { test "$numm" = "0" } && { + error "Nothing to filter inside maildir $input" + return 1 } + + notice "Filtering maildir: $input ($numm mails}" + c=0 + for m in ${(f)mails}; do + match=0 + c=$(($c + 1)) + + list="blacklist" + hdr "$m" | isknown + { test $? = 0 } && { + cat "$m" | deliver zz.blacklist + { test $? = 0 } && { ${=rm} "$m" } + act "$c\t\t/ $numm\t\t->\tzz.blacklist" + continue } + + hdr "$m" | awk '/Sender.*bounce/ { exit 1 }' + { test $? = 0 } || { + act "$c\t\t/ $numm\t\t->\tzz.bounce" + cat "$m" | deliver zz.bounces + { test $? = 0 } && { ${=rm} "$m" } + continue } + + { test "$ownfilters" = "1" } && { + + func "processing through own filters" + ffrom=`hdr "$m" | ${WORKDIR}/bin/fetchaddr -x From -a` + + # run all filter regexps on the from: field + { test "$ffrom" = "" } || { + femail="${ffrom[(ws:,:)1]}" + for exp in ${(k)filter_from}; do + if [[ "$femail" =~ "$exp" ]]; then + act "$c\t\t/ $numm\t\t-> ${filter_from[$exp]}" + cat "$m" | deliver ${filter_from[$exp]} + if [ $? = 0 ]; then + func "from filter match: $exp" + ${=rm} $m + match=1 + else + error "Error filtering to maildir ${filter_from[$exp]}" + error "File: $m" + fi + fi + done + } + { test "$match" = "1" } && { continue } + + ftos=`hdr "$m" | ${WORKDIR}/bin/fetchaddr -x Cc -a | cut -d, -f1` + ftos+=`hdr "$m" | ${WORKDIR}/bin/fetchaddr -x To -a | cut -d, -f1` + + # run all filter regexps on the to: and cc: fields + { test "$ftos" = "" } || { + for ft in ${(f)ftos}; do + for exp in ${(k)filter_to}; do + if [[ "$ft" =~ "$exp" ]]; then + act "$c\t\t/ $numm\t\t-> ${filter_to[$exp]}" + cat "$m" | deliver ${filter_to[$exp]} + if [ $? = 0 ]; then + func "to filter match: $exp" + ${=rm} "$m" + match=1 + else + error "Error filtering to maildir ${filter_to[$exp]}" + error "File: $m" + fi + fi + done + done + } + { test "$match" = "1" } && { continue } + + } # own filters + + list="whitelist" + hdr "$m" | isknown + { test $? = 0 } && { print "hit on whitelist" + act "$c\t\t/ $numm\t\t-> known" + cat "$m" | deliver known + { test $? = 0 } && { ${=rm} "$m" } + continue } + + hdr "$m" | awk '/X-Spam-Flag.*YES/ { exit 1 }' + { test $? = 0 } || { + act "$c\t\t/ $numm\t\t-> zz.spam" + cat "$m" | deliver zz.spam + { test $? = 0 } && { ${=rm} "$m" } + continue } + + # if here then file to unsorted + act "$c\t\t/ $numm\t\t-> unsorted" + cat "$m" | deliver unsorted + { test $? = 0 } && { ${=rm} "$m" } + + done + + return 0 +} + +###### +# MUTT + +update_mutt() { + act "updating mutt settings" + func "MUTTDIR: $MUTTDIR" + func "binary: $mutt" func "pgpewrap: $pgpewrap" func "lock: $dotlock" @@ -154,9 +365,27 @@ EOF switch_identity + for f in `cat "$MAILDIRS/Filters.txt" | awk ' + /^#/ {next} + /^./ { print $4 }'`; do + # MUTT (generate mailboxes priority this parser) + print " \\" >> $MUTTDIR/mboxes + print -n " +${f} " >> $MUTTDIR/mboxes + done + print " \\" >> $MUTTDIR/mboxes + print " +unsorted.ml +unsorted" >> $MUTTDIR/mboxes + + uniq $MUTTDIR/mboxes > $TMPDIR/mboxes + mv $TMPDIR/mboxes $MUTTDIR/mboxes + +} + +update_sieve() { + + ####### # SIEVE - act "generating procmail and sieve filter rules" + act "generating sieve filter rules" id=$datestamp.$RANDOM newlock "$MAILDIRS/Filters.sieve" rm -f "$MAILDIRS/Filters.sieve" @@ -190,12 +419,13 @@ cat <<EOF >> "$MAILDIRS/Filters.sieve" { fileinto "zz.blacklist"; stop; } # bounces -if header :contains "Sender" "mailman-bounce" { +if header :contains "Sender" "bounce" { fileinto "zz.bounces"; stop; } -# filters +############# +# own filters EOF # continue later on while we parse filters @@ -287,8 +517,8 @@ EOF if header :contains "To" [ EOF c=${#filter_to} - for f in $filter_to; do - print -n "\"$f\"" >> "$MAILDIRS/Filters.sieve" + for f in ${(k)filter_to}; do + print -n "\"$f\"" >> $sieve c=$(( $c - 1 )) { test $c != 0 } && { print -n "," >> "$MAILDIRS/Filters.sieve" } print >> "$MAILDIRS/Filters.sieve" @@ -306,8 +536,8 @@ EOF if header :contains "From" [ EOF c=${#filter_from} - for f in $filter_from; do - print -n "\"$f\"" >> "$MAILDIRS/Filters.sieve" + for f in ${(k)filter_from}; do + print -n "\"$f\"" >> $sieve c=$(( $c - 1 )) { test $c != 0 } && { print -n "," >> "$MAILDIRS/Filters.sieve" } print >> "$MAILDIRS/Filters.sieve" @@ -362,107 +592,15 @@ if header :is "X-Spam-Flag" "YES" { stop; } +fileinto "unsorted"; EOF -#### PROCMAIL - - cat <<EOF >> "$PROCMAILDIR/rc" -} - -:0 -* PF_DEST ?? . -* ? test \$PMSRC/pf-save.rc -{ INCLUDERC=\$PMSRC/pf-save.rc } - - -# whitelisting filters -:0 w: -* ? \$JARO -l whitelist -q isknown -known/ - -# spam filters -:0 w: -* ^X-Spam-Flag: YES -zz.spam/ - -EOF - - ####### - cat <<EOF >> "$PROCMAILDIR/rc" -# filters generated from Accounts -:0 -* ? test \$PMSRC/pf-chkto.rc -{ -EOF - cat <<EOF >> "$MAILDIRS/Filters.sieve" -# sent to our own address -if header :contains "To" [ -EOF - typeset -alU recv - accts=`${=find} "$MAILDIRS/Accounts/" -type f | grep -v 'smtp'` - for f in ${(f)accts}; do - for addr in `cat "$f" | awk ' - /^email/ { print $2 } - /^alias/ { print $2 } - '`; do func "email $addr in `basename $f`"; recv+=("$addr"); done - done - c=${#recv} - for rr in ${recv}; do \ - - # procmail - print "ADDR=${rr}\tDEST=priv/\tINCLUDERC=\$PMSRC/pf-chkto.rc" \ - >> "$PROCMAILDIR/rc" +unlock $sieve - # sieve - print -n "\"${rr}\"" >> "$MAILDIRS/Filters.sieve" - c=$(( $c - 1 )) - { test $c != 0 } && { print -n "," >> "$MAILDIRS/Filters.sieve" } - print >> "$MAILDIRS/Filters.sieve" - - act "private account: <${rr}>" - done +return 0 - cat <<EOF >> "$MAILDIRS/Filters.sieve" -] -{ fileinto "priv"; stop; } - -EOF - - cat <<EOF >> "$PROCMAILDIR/rc" } -:0 -* PF_DEST ?? . -* ? test \$PMSRC/pf-save.rc -{ INCLUDERC=\$PMSRC/pf-save.rc } - - -# if its an unknown mailinglist, save it into unsorted.ml -:0 -* ^(List-Id|X-(Mailing-)?List): -unsorted.ml/ - -EOF - - # MUTT (generate mailboxes priority this parser) - print " \\" >> "$MUTTDIR/mboxes" - print " +unsorted.ml +unsorted" >> "$MUTTDIR/mboxes" - - uniq "$MUTTDIR/mboxes" > "$TMPDIR/mboxes" - mv "$TMPDIR/mboxes" "$MUTTDIR/mboxes" - rm -f "$TMPDIR/mboxes" - - # conclude procmail - cat <<EOF >> "$PROCMAILDIR/rc" - -# if got here, go to unsorted -:0: -\$DEFAULT - -# -# End of generated procmail rc -# -EOF # conclude sieve diff --git a/src/zlibs/maildirs b/src/zlibs/maildirs @@ -20,6 +20,9 @@ # this source code; if not, write to: # Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# static global variable formail cache (used by rmdupes) +typeset -h formail_cache + # checks if its a maildir # returns 0 (success) if yes @@ -109,7 +112,7 @@ maildirs_lastlog() { unset lasts } -rmdupes() { +rmdupes() { ## special argument lastlog { test "$1" = "lastlog" } && { @@ -120,13 +123,20 @@ rmdupes() { rmdupes ${=lastdirs} notice "Done pruning" # all the prioritization above is so that duplicates are spotted - # across different maildirs and deleted from the filtered source + # across different maildirs and deleted from the filtered source return 0 } ############### tot=0 typeset -al msgs + + { test -r "$formail_cache" } || { + formail_cache="$TMPDIR/filter.rmdupes.$datestamp.$RANDOM" + newlock "$formail_cache" + mycache=1 + } + for folder in ${=@}; do { test -r "$folder" } || { folder="$MAILDIRS/$folder" } { test -r "$folder" } || { @@ -138,13 +148,14 @@ rmdupes() { continue } c=0 - notice "Checking for duplicates in folder: `basename $folder`" - msgs=() - for m in `${=find} "${folder}" -type f`; do - msgs+=("$m") - done - act "${#msgs} messages to check" - for m in ${=msgs}; do + notice "Checking for duplicates in $folder" + msgs=`${=find} "${folder}" -maxdepth 2 -type f` + act "Please wait, this can take a while..." + + + + for m in ${(f)msgs}; do + func "formail < $m" # 128MB should be enough ehre? formail -D 128000000 $formail_cache <"$m" \ && rm "$m" && c=$(( $c + 1 )) @@ -153,6 +164,8 @@ rmdupes() { tot=$(( $tot + $c )) done + { test "$mycache" = "1" } && { unlock "$formail_cache" } + if [ "$tot" = "0" ]; then act "No duplicates found at all" else @@ -272,5 +285,47 @@ filter() { # prunes out all duplicates from last filtered mails, rmdupes lastlog - unlink "$formail_cache" + unlink $formail_cache +} + +# very simple LDA delivery to a maildir +# the delicate part is returning all errors +# so that fetchmail does not deletes mail from server +deliver() { + if [ "$1" = "" ]; then + dest="$MAILDIRS/incoming" + else + dest="$MAILDIRS/$1" + fi + + # create destination maildir if not existing + { test -r "$dest" } || { + act "creating destination maildir: $dest" + maildirmake "$dest" } + + maildircheck "$dest" + { test $? = 0 } || { + error "Invalid maildir destination for delivery, operation aborted." + func "Returning error to caller." + return 1; } + + base="`hostname`-jaro-`date +%Y-%m-%d-%H.%M.%S`-$RANDOM" + + cat > "$dest/new/$base" + { test $? = 0 } || { + error "Could not write email file into maildir, operation aborted." + func "Returning error to caller." + return 1; } + + + { test "$DEBUG" != "0" } && { + func "Delivery successful, log: $MAILDIRS/logs/jaro-deliver.log" + awk ' +BEGIN { print "Delivery to maildir: '"$1"'" } +{ print $0 } +/^$/ { exit } +' "$1/new/$base" >> "$MAILDIRS/logs/jaro-deliver.log" + } + + return 0 } diff --git a/src/zlibs/stats b/src/zlibs/stats @@ -101,7 +101,7 @@ EOF rm -rf $TMPDIR/weekstats.db.mdir cp -r $MAILDIRS/${m} $TMPDIR/weekstats.db.mdir for f in `${=find} $TMPDIR/weekstats.db.mdir -type f`; do - timestamp=`fetchdate "%Y-%U" ${f}` + timestamp=`fetchdate ${f} "%Y-%U"` mdir="${m[(ws:.:)1]}" cat <<EOF | ${SQL} -batch $db > $sql SELECT * FROM stats @@ -200,7 +200,7 @@ EOF for m in ${maildirs}; do for f in `${=find} $MAILDIRS/${m} -type f`; do - timestamp=`fetchdate "%Y-%m-%d" ${f}` + timestamp=`fetchdate ${f} "%Y-%m-%d"` cat <<EOF | ${SQL} -batch $TMPDIR/timecloud.db.$id > $TMPDIR/timecloud.select.$id SELECT * FROM stats WHERE tag IS "${m}" AND date IS "${timestamp}";