filters (18216B)
1 #!/usr/bin/env zsh 2 # 3 # Jaro Mail, your humble and faithful electronic postman 4 # 5 # a tool to easily and privately handle your e-mail communication 6 # 7 # Copyleft (C) 2010-2015 Denis Roio <jaromil@dyne.org> 8 # 9 # This source code is free software; you can redistribute it and/or 10 # modify it under the terms of the GNU Public License as published by 11 # the Free Software Foundation; either version 3 of the License, or 12 # (at your option) any later version. 13 # 14 # This source code is distributed in the hope that it will be useful, 15 # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 17 # Please refer to the GNU Public License for more details. 18 # 19 # You should have received a copy of the GNU Public License along with 20 # this source code; if not, write to: 21 # Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 22 23 ################### 24 # Filters workflow: 25 # 26 # 1. Check if From in blacklist -> zz.blacklist 27 # 2. Check if /Sender.*bounce/ -> zz.bounces 28 # 3. Check if From Filters match -> own.setup 29 # 4. Check if To Filters match -> own.setup 30 # 5. Check if From in whitelist -> known 31 # 6. Check if /X-Spam-Flag.*YES/ -> zz.spam 32 # 7. Check if To own address -> priv 33 # 8. All the rest -> unsorted 34 # 35 36 # load zsh filter cache arrays 37 { test -r "$MAILDIRS/cache/filters" } && { 38 source "$MAILDIRS/cache/filters" } 39 40 init_inbox() { 41 fn init_inbox 42 req=(MAILDIRS) 43 ckreq 44 45 # make sure maildirs where to put mails exist 46 ${=mkdir} "$MAILDIRS" 47 maildirmake "$MAILDIRS/incoming" 48 maildirmake "$MAILDIRS/known" 49 maildirmake "$MAILDIRS/sent" 50 maildirmake "$MAILDIRS/priv" 51 maildirmake "$MAILDIRS/postponed" 52 maildirmake "$MAILDIRS/unsorted" 53 maildirmake "$MAILDIRS/unsorted.ml" 54 maildirmake "$MAILDIRS/remember" 55 maildirmake "$MAILDIRS/outbox" 56 57 if [[ $PASS = 1 ]]; then 58 GPGID=${$(awk '/default-key/ { print $2 }' $HOME/.gnupg/gpg.conf)} 59 [[ -f "$PASSWORD_STORE_DIR/.gpg-id" ]] || pass init ${GPGID} 60 fi 61 ${=mkdir} "$MAILDIRS/cache" 62 ${=mkdir} "$MAILDIRS/logs" 63 ${=mkdir} "$MAILDIRS/tmp" 64 65 { test -d "$MAILDIRS/Accounts" } || { 66 ${=mkdir} "$MAILDIRS/Accounts" 67 cp "$WORKDIR"/Accounts/* "$MAILDIRS/Accounts/" } 68 69 { test -r "$MAILDIRS"/Manual.pdf } || { 70 cp "$WORKDIR"/jaromail-manual.pdf "$MAILDIRS"/Manual.pdf } 71 72 { test -r "$MAILDIRS"/Identity.txt } || { 73 cp "$WORKDIR"/Identity.txt "$MAILDIRS"/Identity.txt } 74 75 { test -r "$MAILDIRS"/Filters.txt } || { 76 cp "$WORKDIR"/Filters.txt "$MAILDIRS"/Filters.txt } 77 78 { test -r "$MAILDIRS"/Aliases.txt } || { 79 cp "$WORKDIR"/Aliases.txt "$MAILDIRS"/Aliases.txt } 80 81 { test -r "$MAILDIRS"/Applications.txt } || { 82 cp "$WORKDIR"/Applications.txt "$MAILDIRS"/Applications.txt } 83 84 return 0 85 } 86 87 # reads all configurations and creates a cache of what is read 88 # the cache consists of array and maps declarations for zsh 89 update_filters() { 90 fn update_filters 91 req=(MAILDIRS) 92 freq=($MAILDIRS/Filters.txt) 93 ckreq 94 95 notice "Updating filters..." 96 97 ff="$MAILDIRS/cache/filters" 98 ${=mkdir} "$MAILDIRS/cache" 99 100 [[ -r "$ff" ]] && { rm -f "$ff" } 101 newlock "$ff" 102 cat <<EOF >> "$ff" 103 # automatically generated by jaromail 104 typeset -Al filter_from 105 typeset -alU filter_own 106 typeset -Al filter_to 107 filter_from=() 108 filter_own=() 109 filter_to=() 110 111 EOF 112 ffilters=`awk ' 113 /^#/ {next} 114 /^./ { print $1 ";" $2 ";" $3 ";" $4 }' "$MAILDIRS/Filters.txt"` 115 116 # 117 func "insert filter rules in the cache" 118 119 for f in ${(f)ffilters}; do 120 header="${f[(ws:;:)1]}" 121 regexp="${f[(ws:;:)2]}" 122 action="${f[(ws:;:)3]}" 123 destination="${f[(ws:;:)4]}" 124 case $header in 125 to) 126 cat <<EOF >> "$ff" 127 filter_to+=("${regexp}" "${destination}") 128 EOF 129 func "from: <${regexp}> -> ${destination}" 130 maildirmake $MAILDIRS/$destination 131 ;; 132 from) 133 cat <<EOF >> "$ff" 134 filter_from+=("${regexp}" "${destination}") 135 EOF 136 func "to: <${regexp}> -> ${destination}" 137 maildirmake $MAILDIRS/$destination 138 ;; 139 140 *) 141 error "invalid filter: $f" 142 ;; 143 esac 144 done 145 146 # # create the notmuch database if not present 147 # [[ -r "$MAILDIRS"/cache/notmuch/rc ]] || { 148 # notice "Indexing emails in the search database" 149 # nm setup 150 # nm new 151 # } 152 153 # 154 func "compile the list of own addresses and aliases" 155 156 for i in `awk ' 157 /^#/ { next } 158 /^$/ { next } 159 /^email/ { print $2 }' \ 160 "$MAILDIRS"/Accounts/*`; do 161 cat <<EOF >> "$ff" 162 filter_own+=($i) 163 EOF 164 done 165 { test -r $MAILDIRS/Aliases.txt } && { 166 for i in `awk ' 167 /^#/ { next } 168 /^$/ { next } 169 { print $1 }' "$MAILDIRS/Aliases.txt"`; do 170 cat <<EOF >> "$ff" 171 filter_own+=($i) 172 EOF 173 done 174 } 175 176 # 177 func "unlocking and compiling the cache" 178 179 unlock "$ff" 180 zcompile "$ff" 181 func "recursive reload" 182 source "$WORKDIR/zlibs/filters" 183 return 0 184 } 185 186 187 filter_maildir() { 188 fn filter_maildir 189 req=(MAILDIRS) 190 ckreq || return 1 191 192 # Makes glob matching case insensitive 193 unsetopt CASE_MATCH 194 195 mdinput="$1" 196 197 # for safety we bail out in case the final fallback 198 # maildir is not existing. unsorted should always 199 # be there. 200 maildircheck "$MAILDIRS/unsorted" 201 [[ $? = 0 ]] || { 202 error "Invalid fallback maildir destination, operation aborted." 203 func "Returning error to caller." 204 return 1 205 } 206 207 # loads up the filter cache (zsh compiled arrays) 208 [[ -r "$MAILDIRS/cache/filters" ]] && { 209 source $MAILDIRS/cache/filters 210 ownfilters=1 211 } 212 213 # default maildir to filter is incoming 214 mdinput=${1:-incoming} 215 216 maildircheck "$MAILDIRS/$mdinput" 217 [[ $? = 0 ]] || { 218 error "Invalid maildir to filter: $mdinput" 219 return 1 220 } 221 222 mails=`${=find} "$MAILDIRS/$mdinput" -maxdepth 2 -type f` 223 numm=`print $mails | wc -l` 224 225 [[ $numm = 0 ]] && { 226 error "Nothing to filter inside maildir $mdinput" 227 return 1 228 } 229 230 notice "Filtering maildir: $mdinput ($numm mails)" 231 c=0 232 233 for m in ${(f)mails}; do 234 235 # clean interrupt 236 [[ $global_quit = 1 ]] && { 237 error "User break requested, interrupting operation" 238 break 239 } 240 241 match=0 242 c=$(($c + 1)) 243 244 # check if its an empty file 245 _fsize=`zstat +size "$m"` 246 [[ $_fsize = 0 ]] && { 247 act "$c\t/ $numm\t(empty)" 248 rm "$m" 249 continue 250 } 251 252 # parse if its a mailinglist 253 _ml=0 254 hdr "$m" | ismailinglist 255 [[ $? = 0 ]] && _ml=1 256 257 list="blacklist" 258 hdr "$m" | sender_isknown 259 [[ $? = 0 ]] && { 260 [[ "$mdinput" = "zz.blacklist" ]] && { 261 act "$c\t/ $numm" 262 continue 263 } 264 265 act "$c\t/ $numm\t-> zz.blacklist" 266 267 [[ $DRYRUN = 1 ]] || { 268 printfile "$m" | deliver zz.blacklist 269 [[ $? = 0 ]] && { rm "$m" } 270 continue 271 } 272 } 273 274 hdr "$m" | awk '/Sender.*mailman-bounce/ { exit 1 }' 275 [[ $? = 0 ]] || { 276 [[ "$mdinput" = "zz.bounces" ]] && { 277 act "$c\t/ $numm" 278 continue 279 } 280 act "$c\t/ $numm\t-> zz.bounces" 281 [[ $DRYRUN = 1 ]] || { 282 printfile "$m" | deliver zz.bounces 283 [[ $? = 0 ]] && { rm "$m" } 284 continue 285 } 286 } 287 288 [[ "$ownfilters" = "1" ]] && { 289 290 func "processing through own filters" 291 292 # run all filter regexps on the from: field 293 _dest="" 294 e_addr=() 295 hdr "$m" | e_parse From 296 [[ $? = 0 ]] && { 297 femail="${(k)e_addr}" # e_parse From hit is always one 298 for exp in ${(k)filter_from}; do 299 300 # fuzzy match on a string (PCRE) 301 if [[ "$femail" =~ "$exp" ]]; then 302 303 # retrieve the filter destination maildir 304 _dest="${filter_from[$exp]}" 305 306 # if destination maildir is same as input, skip 307 [[ "$_dest" = "$mdinput" ]] && { 308 act "$c\t/ $numm" 309 match=1 310 break 311 } 312 313 act "$c\t/ $numm\t-> $_dest\t(from $femail)" 314 315 # tag mailinglists 316 [[ $DRYRUN = 1 ]] || { 317 if [[ $_ml = 1 ]]; then 318 printfile "$m" | deliver "$_dest" "+filtered +mailinglist" 319 else 320 printfile "$m" | deliver "$_dest" "+filtered" 321 fi 322 } 323 324 if [[ $? = 0 ]]; then 325 match=1 326 rm "$m" 327 break 328 else 329 error "Error filtering to maildir $_dest" 330 error "File: $m" 331 continue 332 fi 333 fi 334 done 335 [[ "$match" = "1" ]] && { 336 func "matched filter from: field" 337 continue 338 } 339 } 340 341 _dest="" 342 # recompile the array of destination addresses 343 e_addr=() 344 hdr "$m" | e_parse To 345 hdr "$m" | e_parse Cc 346 # run all filter regexps on the to: and cc: fields 347 [[ $? = 0 ]] && { 348 for ft in ${(k)e_addr}; do 349 for exp in ${(k)filter_to}; do 350 351 # fuzzy match on a string (PCRE) 352 if [[ "$ft" =~ "$exp" ]]; then 353 354 # retrieve the filter destination maildir 355 _dest="${filter_to[$exp]}" 356 357 # if destination maildir is same as input, skip 358 [[ "$_dest" = "$mdinput" ]] && { 359 act "$c\t/ $numm" 360 match=1 361 break 362 } 363 364 act "$c\t/ $numm\t-> $_dest\t(to $ft)" 365 # tag mailinglists 366 [[ $DRYRUN = 1 ]] || { 367 if [[ $_ml = 1 ]]; then 368 printfile "$m" | deliver "$_dest" "+filtered +mailinglist" 369 else 370 printfile "$m" | deliver "$_dest" "+filtered" 371 fi 372 } 373 374 if [[ $? = 0 ]]; then 375 match=1 376 rm "$m" 377 break 378 else 379 error "Error filtering to maildir $_dest" 380 error "File: $m" 381 continue 382 fi 383 fi 384 done 385 [[ "$match" = "1" ]] && { break } 386 done 387 } 388 389 [[ "$match" = "1" ]] && { 390 func "matched filter to:/cc: fields" 391 continue 392 } 393 394 } # own filters 395 396 list="whitelist" 397 hdr "$m" | sender_isknown 398 [[ $? = 0 ]] && { 399 [[ "$mdinput" = "known" ]] && { 400 act "$c\t/ $numm" 401 continue 402 } 403 act "$c\t/ $numm\t-> known" 404 [[ $DRYRUN = 1 ]] || { 405 printfile "$m" | deliver known "+inbox +priv" 406 [[ $? = 0 ]] && { rm "$m" } 407 continue 408 } 409 } 410 411 hdr "$m" | awk '/X-Spam-Flag.*YES/ { exit 1 }' 412 { test $? = 0 } || { 413 [[ "$mdinput" = "zz.spam" ]] && { 414 act "$c\t/ $numm" 415 continue 416 } 417 act "$c\t/ $numm\t-> zz.spam" 418 [[ $DRYRUN = 1 ]] || { 419 printfile "$m" | deliver zz.spam 420 [[ $? = 0 ]] && { rm "$m" } 421 continue 422 } 423 } 424 425 # parse own email and aliases 426 match=0 427 for f in ${(k)e_addr}; do 428 # check if destination address is in filter_own array 429 [[ ${filter_own[(r)$f]} == ${f} ]] && { 430 [[ "$mdinput" = "priv" ]] && { 431 act "$c\t/ $numm" 432 match=1 433 break 434 } 435 act "$c\t/ $numm\t-> priv" 436 [[ $DRYRUN = 1 ]] || { 437 printfile "$m" | deliver priv "+priv" 438 [[ $? = 0 ]] && { 439 rm "$m"; 440 match=1 441 break 442 } 443 } 444 } 445 done 446 [[ $match = 1 ]] && continue 447 448 # its an unkown mailinglist 449 [[ $_ml = 1 ]] && { 450 [[ "$mdinput" = "unsorted.ml" ]] && { 451 act "$c\t/ $numm" 452 continue 453 } 454 act "$c\t/ $numm\t-> unsorted.ml" 455 [[ $DRYRUN = 1 ]] || { 456 printfile "$m" | deliver unsorted.ml "+unsorted +mailinglist" 457 [[ $? = 0 ]] && { 458 rm "$m" 459 continue 460 } 461 } 462 } 463 464 # if here then file to unsorted 465 if [ "$mdinput" = "unsorted" ]; then 466 act "$c\t/ $numm" 467 else 468 act "$c\t/ $numm\t-> unsorted" 469 [[ $DRYRUN = 1 ]] || { 470 printfile "$m" | deliver unsorted "+unsorted" 471 [[ $? = 0 ]] && { rm "$m" } 472 } 473 fi 474 475 done 476 477 return 0 478 } 479 480 481 # sieve_filter() gets an array of patterns to match and builds a long rule 482 # for which if they match the conditional directive they all go in one folder 483 # $1 = conditional directive 484 # $2 = folder to fileinto 485 # sieve_filter_array: array of entries 486 typeset -alU sieve_filter_array 487 sieve_filter() { 488 condition="$1" 489 fileinto="$2" 490 491 cat <<EOF >> "$MAILDIRS/Filters.sieve" 492 493 # $fileinto 494 $condition [ 495 EOF 496 c=${#sieve_filter_array} 497 for i in $sieve_filter_array; do 498 print -n "\"$i\"" >> "$MAILDIRS/Filters.sieve" 499 c=$(( $c - 1 )) 500 { test $c != 0 } && { print -n "," >> "$MAILDIRS/Filters.sieve" } 501 print >> "$MAILDIRS/Filters.sieve" 502 done 503 504 cat <<EOF >> "$MAILDIRS/Filters.sieve" 505 ] 506 { fileinto :create "$fileinto"; stop; } 507 508 EOF 509 return 0 510 } 511 512 typeset -A sieve_filter_map 513 # sieve_complex_filter gets a map of patterns as an argument and builds a 514 # long rule for which any key matching it gets delivered to its value folder 515 # $1 = conditional directive 516 # sieve_filter_map = map of patterns, key is match and value is destination 517 # assign with set -A sieve_filter_map yourmap 518 sieve_complex_filter() { 519 [[ ${#sieve_filter_map} == 0 ]] && { return 1 } 520 condition="$1" 521 func "Sieve complex filter entries: ${#sieve_filter_map}" 522 523 for fil in ${(k)sieve_filter_map}; do 524 print "$condition \"${fil}\" { fileinto :create \"${sieve_filter_map[$fil]}\"; stop; }" \ 525 >> "$MAILDIRS/Filters.sieve" 526 done 527 528 return 0 529 } 530 531 update_sieve() { 532 533 ####### 534 # SIEVE 535 act "generating sieve filter rules" 536 id=`datestamp`.$RANDOM 537 newlock "$MAILDIRS/Filters.sieve" 538 rm -f "$MAILDIRS/Filters.sieve" 539 touch "$MAILDIRS/Filters.sieve" 540 chmod 600 "$MAILDIRS/Filters.sieve" 541 cat <<EOF >> "$MAILDIRS/Filters.sieve" 542 # mailbox supports fileinto :create 543 require ["fileinto","mailbox","variables"]; 544 545 EOF 546 547 # blacklist 548 [[ -r "$MAILDIRS"/blacklist.abook ]] && { 549 sieve_filter_array=() 550 551 for i in `awk -F'=' ' 552 /^email/ { print $2 } 553 ' "$MAILDIRS"/blacklist.abook`; do 554 sieve_filter_array+=("$i") 555 done 556 557 { test "${#sieve_filter_array}" = "0" } || { 558 sieve_filter \ 559 'if header :contains "From"' \ 560 zz.blacklist 561 } 562 } 563 564 # bounces 565 cat <<EOF >> "$MAILDIRS/Filters.sieve" 566 # bounces 567 if header :contains "Sender" "mailman-bounce" { 568 fileinto :create "zz.bounces"; 569 stop; 570 } 571 572 ############# 573 # own filters 574 575 EOF 576 577 set -A sieve_filter_map ${(kv)filter_to} 578 sieve_complex_filter 'if header :contains [ "To","Cc" ] ' 579 580 set -A sieve_filter_map ${(kv)filter_from} 581 sieve_complex_filter 'if header :contains "From"' 582 583 584 585 586 ############################################################## 587 # if the sender is known (whitelist) then put mail in 588 # high priority 'known' maildir or INBOX (sieve) 589 590 act "compiling whitelist rules from addressbook" 591 func "generating whitelist for sieve filters" 592 593 [[ -r "$MAILDIRS"/whitelist.abook ]] && { 594 sieve_filter_array=() 595 for i in `awk -F'=' ' 596 /^email/ { print $2 } 597 ' "$MAILDIRS"/whitelist.abook`; do 598 sieve_filter_array+=("$i") 599 done 600 sieve_filter \ 601 'if header :contains "From"' \ 602 INBOX 603 } 604 605 cat <<EOF >> "$MAILDIRS/Filters.sieve" 606 # spam 607 if header :is "X-Spam-Flag" "YES" { 608 fileinto :create "zz.spam"; stop; 609 } 610 611 EOF 612 613 # own addresses and aliases 614 sieve_filter_array=($filter_own) 615 sieve_filter \ 616 'if header :contains [ "To","Cc" ] ' \ 617 priv 618 619 # unsorted 620 cat <<EOF >> "$MAILDIRS/Filters.sieve" 621 if header :matches "List-Id" "*<*>" { 622 fileinto :create "lists.\${2}"; stop; } 623 elsif header :matches "X-BeenThere" "*@*" { 624 fileinto :create "lists.\${1}.\${2}"; stop; } 625 elsif header :matches "List-Post" "<mailto:*@*" { 626 fileinto :create "lists.\${1}.\${2}"; stop; } 627 628 if anyof (header :contains "X-Mailman-Version" ".", 629 header :contains "Mailing-List" ".") { 630 fileinto :create "lists.unsorted"; stop; } 631 632 fileinto :create "unsorted"; 633 EOF 634 635 unlock "$MAILDIRS/Filters.sieve" 636 637 return 0 638 } # end of update()