commit 85fff4dcda76c804ab3707b095b0b2e68b3d593b
parent 429b71eef4ff09198fbd477d226a592573525d60
Author: Jaromil <jaromil@dyne.org>
Date: Tue, 30 Dec 2014 15:32:29 +0100
several changes for notmuch integration, also gnupg address extraction
Diffstat:
6 files changed, 327 insertions(+), 272 deletions(-)
diff --git a/src/jaro b/src/jaro
@@ -451,7 +451,9 @@ a pipe | in front indicate they take an email body from stdin
list prints to console all the entries in $list
- search search into $list using a string parameter
+ index index fetched email archives for search
+
+ search search using a string parameter
|isknown read e-mail from stdin, return 0 if sender is known
@@ -532,6 +534,7 @@ main()
subcommands_opts[stat]=""
+ subcommands_opts[index]=""
subcommands_opts[search]=""
subcommands_opts[learn]=""
@@ -548,6 +551,8 @@ main()
subcommands_opts[preview]=""
subcommands_opts[later]=""
+ subcommands_opts[remember]=""
+
subcommands_opts[backup]=""
subcommands_opts[rmdupes]=""
subcommands_opts[merge]=""
@@ -675,7 +680,7 @@ main()
send) send ${PARAM} ;; # was checking is_online
peek) peek ${PARAM} ;; # was checking is_online
- later) later ${PARAM} ;;
+ later|remember) cat | deliver remember ;;
update|init)
init_inbox
@@ -686,6 +691,7 @@ main()
help) CLEANEXIT=0; usage ;;
+ index) CLEANEXIT=0; nm_index ${PARAM} ;;
search) CLEANEXIT=0; search ${PARAM} ;;
stat) CLEANEXIT=0; stats ${PARAM} ;;
@@ -783,7 +789,7 @@ main()
;;
list|extract)
- extract ${=PARAM}
+ extract ${PARAM}
exitcode=$?
;;
diff --git a/src/zlibs/addressbook b/src/zlibs/addressbook
@@ -74,13 +74,13 @@ remove_address() {
}
search_addressbook() {
- func "search \"$1\" in $list"
- abook --datafile "$ADDRESSBOOK" --mutt-query "$1"
+ func "search \"$@\" in $list"
+ abook --datafile "$ADDRESSBOOK" --mutt-query "$@"
}
lookup_email() {
- func "lookup email $1 in $list"
+ func "lookup address $1 in $list"
abook --datafile "$ADDRESSBOOK" \
--mutt-query "$1" > /dev/null
return $?
@@ -101,6 +101,7 @@ complete() {
act "Searching for \"$needle\" in mailout groups"
matches=`${=find} "$MAILDIRS/Groups" -type f -name \"*$needle*\"`
fi
+
print "Groups: `print $matches | wc -l` matches"
print
for i in ${(f)matches}; do
@@ -144,7 +145,7 @@ learn() {
case ${what} in
- sender) # simple: one address only on From:
+ sender|from) # simple: one address only on From:
head="`print $buffer | ${WORKDIR}/bin/fetchaddr -x From -a`"
# (Q) eliminates quotes, then word split
email="${(Q)head[(ws:,:)1]}"
@@ -174,7 +175,7 @@ learn() {
return 0
;;
- recipient) # complex: more addresses in To: and Cc:
+ recipient|to) # complex: more addresses in To: and Cc:
head="`print $buffer | ${WORKDIR}/bin/fetchaddr -x To -a`"
for h in ${(f)head}; do
# (Q) eliminates quotes, then word split
@@ -221,13 +222,78 @@ forget() {
# remove_address "${head[(ws:,:)1]}"
}
+# extract all addresses found into a maildir
+extract_maildir() {
+ ## first arg is a directory
+ md="$1"
+ func "extract maildir: $md"
+ ## extract from a maildir
+ maildircheck "$md" && {
+ _action="$2"
+ case $_action in
+ all) ;;
+ recipient) ;;
+ sender) ;;
+ *) _action="all" ;;
+ esac
+
+ # search files
+ _mails=`find $md -type f`
+ # search symlinks
+ _mails+=`find $md -type l`
+
+ # TODO ismailfile() to check if file is a mail?
+
+ # we switch dryrun temporarily off to use learn()
+ # without modifying the addressbook
+ _dryrun=$DRYRUN
+ DRYRUN=1
+
+ notice "Extracting and listing $_action in maildir: $md"
+ act "please wait while scanning `print $_mails | wc -l` mail files..."
+ typeset -a learned
+
+ for i in ${(f)_mails}; do
+ _l=`hdr $i | learn $_action`
+ # handles results on multiple lines (recipients, all)
+ for ii in ${(f)_l}; do
+ learned+=("$ii")
+ done
+ done
+
+ DRYRUN=$_dryrun
+ # eliminates duplicates
+ typeset -A result
+ for i in ${learned}; do
+ _e=${i[(ws:,:)1]}
+ [[ "${result[$_e]}" = "" ]] && {
+ _n=${i[(ws:,:)2]}
+ result+=("$_e" "$_n")
+ print - "$_n <$_e>"
+ }
+ done
+ notice "Unique $_action found: ${#result}"
+ # counts which addresses are known to us
+ _known=0
+ for i in ${(k)result}; do
+ lookup_email ${i}
+ [[ $? = 0 ]] && {
+ _known=$(( $_known + 1 )) }
+ done
+ act "addresses known: $_known"
+ return 0
+ }
+}
+
# extract all entries in addressbook or all addresses in a pgp keyring
# or all signatures on a pgp key (even without importing it)
extract() {
- func "extract() $@"
+ func "calling extract() $PARAM"
+
# without arguments just list all entries in the active list
# default is whitelist
[[ "$1" = "" ]] && {
+ func "extract all from list $list"
awk -F'=' '
/^name/ { printf("%s ",$2) }
/^email/ { printf("<%s>\n",$2) }
@@ -235,140 +301,97 @@ extract() {
return 0
}
- # a map to eliminate duplicates
- typeset -AU result
- [[ -r "$1" ]] || {
- error "file not found: $1"
- error "nothing to extract."
- return 1
- }
+ [[ -r "$1" ]] && { # first arg is a file
- ## arg is a directory
- [[ -d "$1" ]] && {
- func "extract maildir: $1"
- ## extract from a maildir
- maildircheck "$1" && {
- _action="$2"
- case $_action in
- all) ;;
- recipient) ;;
- sender) ;;
- *) _action="all" ;;
- esac
- _mails=`find $1 -type f`
- # TODO ismailfile() to check if file is a mail?
-
- # we switch dryrun temporarily off to use learn()
- # without modifying the addressbook
- _dryrun=$DRYRUN
- DRYRUN=1
-
- notice "Extracting and listing $_action in maildir: $2"
- act "please wait while scanning ${#_mails} mail files..."
- typeset -a learned
-
- for i in ${(f)_mails}; do
- _l=`hdr $i | learn $_action`
- # handles results on multiple lines (recipients, all)
- for i in ${(f)_l}; do
- learned+=("$i")
- done
- done
-
- DRYRUN=$_dryrun
- # eliminates duplicates
- typeset -A result
- for i in ${learned}; do
- _e=${i[(ws:,:)1]}
+ # a map to eliminate duplicates
+ typeset -AU result
+
+ # if first arg is a directory then extract from maildir
+ [[ -d "$1" ]] && {
+ extract_maildir "$1" "$2"
+ return $?
+ }
+
+ func "testing argument with file magic"
+ _magic=`file "$1"`
+
+ ######### GPG
+ # first arg is a GnuPG key ring
+ [[ "$_magic" =~ "GPG key public ring" ]] && {
+
+ notice "Listing addresses found in GPG keyring: $1"
+ _addrs=`gpg --list-keys --with-colons | awk -F: '{print $10}'`
+ for i in ${(f)_addrs}; do
+ _parsed=`print "From: $i" | ${WORKDIR}/bin/fetchaddr -a -x from`
+ _e="${_parsed[(ws:,:)1]}"
+ isemail "$_e"
+ [[ $? = 0 ]] || continue
+ # check if the email is not already parsed
[[ "${result[$_e]}" = "" ]] && {
- _n=${i[(ws:,:)2]}
+ _n="${_parsed[(ws:,:)2]}"
result+=("$_e" "$_n")
print - "$_n <$_e>"
}
done
- notice "Unique $_action found: ${#result}"
+
+ notice "Unique addresses found: ${#result}"
# counts which addresses are known to us
_known=0
for i in ${(k)result}; do
lookup_email ${i}
- [[ $? = 0 ]] && {
- _known=$(( $_known + 1 )) }
+ [[ $? = 0 ]] || {
+ _known=$(( $_known + 1 )) }
done
- act "addresses known: $_known"
+ act "new addresses: $_known"
return 0
}
- }
-
- ######### GPG
- [[ `file "$1"` =~ "GPG key public ring" ]] && {
-
- notice "Listing addresses found in GPG keyring: $1"
- _addrs=`gpg --list-keys --with-colons | awk -F: '{print $10}'`
- for i in ${(f)_addrs}; do
- _parsed=`print "From: $i" | ${WORKDIR}/bin/fetchaddr -a -x from`
- _e="${_parsed[(ws:,:)1]}"
- isemail "$_e"
- [[ $? = 0 ]] || continue
- # check if the email is not already parsed
- [[ "${result[$_e]}" = "" ]] && {
- _n="${_parsed[(ws:,:)2]}"
- result+=("$_e" "$_n")
- print - "$_n <$_e>"
- }
- done
- notice "Unique addresses found: ${#result}"
+ # first arg is a GnuPG public key
+ [[ "$_magic" =~ "PGP public key" ]] && {
+ _gpg="gpg --no-default-keyring --keyring $MAILDIRS/cache/pubkey.gpg --batch --with-colons"
+ rm -f $MAILDIRS/cache/pubkey.gpg
+ ${=_gpg} --import "$1"
+ # first make sure all unknown keys are imported
+ _addrs=`${=_gpg} --list-sigs | awk -F: '{print $5 " " $10}'`
+ for i in ${(f)_addrs}; do
+ [[ "$i" =~ "[User ID not found]" ]] && {
+ act "looking up: $i"
+ ${=_gpg} --recv-key ${i[(w)1]}
+ }
+ done
+
+ _addrs=`${=_gpg} --list-sigs | awk -F: '{print $10}'`
+ for i in ${(f)_addrs}; do
+ _parsed=`print "From: $i" | ${WORKDIR}/bin/fetchaddr -a -x from`
+ _e="${_parsed[(ws:,:)1]}"
+ isemail "$_e"
+ [[ $? = 0 ]] || continue
+ # check if the email is not already parsed
+ [[ "${result[$_e]}" = "" ]] && {
+ _n="${_parsed[(ws:,:)2]}"
+ result+=("$_e" "$_n")
+ print - "$_n <$_e>"
+ }
+ done
+
+ notice "Unique addresses found: ${#result}"
# counts which addresses are known to us
- _known=0
- for i in ${(k)result}; do
- lookup_email ${i}
- [[ $? = 0 ]] || {
- _known=$(( $_known + 1 )) }
- done
- act "new addresses: $_known"
- return 0
+ _known=0
+ for i in ${(k)result}; do
+ lookup_email ${i}
+ [[ $? = 0 ]] || {
+ _known=$(( $_known + 1 )) }
+ done
+ act "new addresses: $_known"
+ return 0
+ }
}
- [[ `file "$1"` =~ "PGP public key" ]] && {
- _gpg="gpg --no-default-keyring --keyring $MAILDIRS/cache/pubkey.gpg --batch --with-colons"
- rm -f $MAILDIRS/cache/pubkey.gpg
- ${=_gpg} --import "$1"
- # first make sure all unknown keys are imported
- _addrs=`${=_gpg} --list-sigs | awk -F: '{print $5 " " $10}'`
- for i in ${(f)_addrs}; do
- [[ "$i" =~ "[User ID not found]" ]] && {
- act "looking up: $i"
- ${=_gpg} --recv-key ${i[(w)1]}
- }
- done
-
- _addrs=`${=_gpg} --list-sigs | awk -F: '{print $10}'`
- for i in ${(f)_addrs}; do
- _parsed=`print "From: $i" | ${WORKDIR}/bin/fetchaddr -a -x from`
- _e="${_parsed[(ws:,:)1]}"
- isemail "$_e"
- [[ $? = 0 ]] || continue
- # check if the email is not already parsed
- [[ "${result[$_e]}" = "" ]] && {
- _n="${_parsed[(ws:,:)2]}"
- result+=("$_e" "$_n")
- print - "$_n <$_e>"
- }
- done
-
- notice "Unique addresses found: ${#result}"
- # counts which addresses are known to us
- _known=0
- for i in ${(k)result}; do
- lookup_email ${i}
- [[ $? = 0 ]] || {
- _known=$(( $_known + 1 )) }
- done
- act "new addresses: $_known"
- return 0
- }
-
+ func "extract from search query"
+ # args are simply strings, then run a search and extract addresses
+ nm_search ${=PARAM}
+ extract_maildir "$MAILDIRS"/cache/notmuch/results all
}
diff --git a/src/zlibs/email b/src/zlibs/email
@@ -168,6 +168,9 @@ fetch() {
return 1
}
+ # updates the notmuch configuration
+ nm_setup
+
notice "Fetching email for account ${account}"
is_online ${imap} ${imap_port}
@@ -506,12 +509,3 @@ EOF
return $?
}
-later() {
- func "Saving message from stdin into remember"
- filename=$USER.${hostname}.$datestamp.$RANDOM
- # hostname was set by the main jaro routine
- func "Filename: $filename"
-
- { maildircheck "${MAILDIRS}/remember" } || { maildirmake "${MAILDIRS}/remember" }
- cat > "${MAILDIRS}/remember/new/$filename"
-}
diff --git a/src/zlibs/filters b/src/zlibs/filters
@@ -457,6 +457,7 @@ EOF
}
+
# sieve_filter() gets an array of patterns to match and builds a long rule
# for which if they match the conditional directive they all go in one folder
# $1 = conditional directive
diff --git a/src/zlibs/maildirs b/src/zlibs/maildirs
@@ -87,30 +87,6 @@ list_maildirs() {
return ${#maildirs}
}
-maildirs_lastlog() {
- # returns an array of destinations maildirs touched by the last filtering operation
- # based on the procmail log format
- typeset -alU dests prio lasts
- _folders=`cat "${MAILDIRS}/logs/procmail.log"|awk '/Folder:/ {print $2}' | cut -d/ -f1`
- for d in ${(f)_folders}; do
- func "maildir touched by last operation: $d"
- # skip procmail glitch
- { test "$d" = "procmail" } && { continue }
- # put filtered to last
- [[ ${PARAM} == *${d}* ]] && { lasts=($lasts $d); continue }
- # always give priority to known, then to priv, then the rest
- { test "$d" = "known" } && { prio=(known $prio); continue }
- { test "$d" = "priv" } && { prio=($prio priv); continue }
- # skip zz. trash
- [[ $d == zz.* ]] && { continue }
- # put them to filter
- dests+=($d)
- done
- print "${=prio} ${=dests} ${=lasts}"
- unset dests
- unset prio
- unset lasts
-}
rmdupes() {
@@ -236,34 +212,29 @@ merge() {
# so that fetchmail does not deletes mail from server
deliver() {
if [ "$1" = "" ]; then
- dest="$MAILDIRS/incoming"
+ dest="$MAILDIRS/incoming"
else
- dest="$MAILDIRS/$1"
- { test -d "$dest" } || { dest="$1"
- { test -d "$dest" } || {
- error "delivery destination path invalid: $1"
- return 1; } }
+ dest="$MAILDIRS/$1"
+ { test -d "$dest" } || { dest="$1"
+ { test -d "$dest" } || {
+ error "delivery destination path invalid: $1"
+ return 1
+ }
+ }
fi
-
+
# create destination maildir if not existing
[[ -r "$dest" ]] || {
act "creating destination maildir: $dest"
- maildirmake "$dest" }
+ maildirmake "$dest"
+ }
maildircheck "$dest"
[[ $? = 0 ]] || {
error "Invalid maildir destination for delivery, operation aborted."
func "Returning error to caller."
- return 1; }
-
- base="`hostname`_jaro_`date +%Y-%m-%d_%H-%M-%S`_$RANDOM"
-
- cat > "$dest/new/$base"
- [[ $? = 0 ]] || {
- error "Could not write email file into maildir, operation aborted."
- func "Returning error to caller."
- return 1; }
-
+ return 1
+ }
[[ $DEBUG = 0 ]] || {
func "Delivery successful, log: $MAILDIRS/logs/jaro-deliver.log"
@@ -273,6 +244,60 @@ BEGIN { print "Delivery to maildir: '"$1"'" }
/^$/ { exit }
' "$MAILDIRS/$1/new/$base" >> "$MAILDIRS/logs/jaro-deliver.log"
}
+
+ # destinations excluded from notmuch indexing
+ [[ "$dest" = "outbox" ]] \
+ || [[ "$dest" =~ "^zz." ]] \
+ || [[ "$dest" = "incoming" ]] && {
+
+ base="`hostname`_jaro_`date +%Y-%m-%d_%H-%M-%S`_$RANDOM"
+
+ cat > "$dest/new/$base"
+ [[ $? = 0 ]] || {
+ error "Could not write email file into maildir $dest."
+ func "Returning error to caller."
+ return 1
+ }
+ return 0
+ }
+ #########
+ # notmuch indexing from here
+ NOTMUCH_CONFIG="$MAILDIRS"/cache/notmuch/rc
+
+ # tag +inbox
+ [[ "$dest" = "known" ]] \
+ || [[ "$dest" = "priv" ]] \
+ || [[ "$dest" = "sent" ]] && {
+
+ cat | notmuch-insert --folder="$dest" +inbox
+ [[ $? = 0 ]] || {
+ error "Could not write email file into maildir $dest using notmuch-insert."
+ func "Returning error to caller."
+ return 1
+ }
+ return 0
+ }
+
+ # tag +unsorted
+ [[ "$dest" = "unsorted" ]] \
+ || [[ "$dest" =~ "^lists." ]] && {
+
+ cat | notmuch-insert --folder="$dest" +unsorted
+ [[ $? = 0 ]] || {
+ error "Could not write email file into maildir $dest using notmuch-insert."
+ func "Returning error to caller."
+ return 1
+ }
+ return 0
+ }
+
+ # anything else +filters
+ cat | notmuch-insert --folder="$dest" +filters
+ [[ $? = 0 ]] || {
+ error "Could not write email file into maildir $dest using notmuch-insert."
+ func "Returning error to caller."
+ return 1
+ }
return 0
}
diff --git a/src/zlibs/search b/src/zlibs/search
@@ -22,108 +22,114 @@
#######################
## Search into maildirs
-# using mairix
-search() {
- # check if the name of a maildir is among params
- # we need at least 2 maildirs, the second is the destination
- typeset -al fold
- typeset -al term
- # intelligent parse of args, position independent
- # check if its a folder, if not is an expression
- for p in ${PARAM}; do
- if [ -r ${p} ]; then
- { maildircheck ${p} } && {
- func "param ${p} is a maildir"
- fold+=(${p}) }
- elif [ -r "${MAILDIRS}/${p}" ]; then
- { maildircheck ${MAILDIRS}/${p} } && {
- func "param ${p} is a jaro maildir"
- fold+=(${MAILDIRS}/${p}) }
- else
- func "param ${p} is a search term"
- term+=(${p})
- fi
+# using notmuch
+
+nm_dir="$MAILDIRS"/cache/notmuch
+
+nm_setup() {
+ mkdir -p "$nm_dir"
+
+ # read if there are other email aliases configured
+ [[ -r "$MAILDIRS"/Aliases.txt ]] && {
+ other_email="other_email"
+ _aliases=`cat "$MAILDIRS"/Aliases.txt`
+ _sep=\=
+ for i in ${(f)_aliases}; do
+ other_email+="${_sep}${i}"
+ _sep=";"
+ done
+ }
+
+ rm -f "$nm_dir"/rc
+ cat <<EOF > "$nm_dir"/rc
+[database]
+path=$MAILDIRS
+
+[user]
+name=$name
+primary_email=$email
+$other_email
+
+[new]
+tags=unread
+ignore=zz.;log;cache;Accounts;Groups;.mutt;webnomad;.abook;.txt;.pdf;.html;.png;.js
+
+[maildir]
+synchronize_flags=true
+EOF
+}
+
+nm_index() {
+ read_account
+ nm_setup
+ func "notmuch --config=${nm_dir}/rc new"
+
+ notmuch --config="${nm_dir}/rc" new
+}
+
+nm_search() {
+ read_account
+ nm_setup
+
+ notice "Searching emails for: $=PARAM"
+ local search_results
+ func "notmuch --config=${nm_dir}/rc search --output=files ${=PARAM}"
+
+ # launch the search with notmuch
+ search_results=`notmuch --config="${nm_dir}/rc" search --output=files ${=PARAM}`
+ act "`print ${search_results} | wc -l` results found"
+ [[ $? = 0 ]] || {
+ error "notmuch search failed with an error"
+ return 1 }
+
+ # populate the maildir with results
+ _resdir="${nm_dir}/results"
+ func "notmuch results in $_resdir"
+ rm -rf "$_resdir"
+ act "populating a maildir with results"
+ maildirmake $_resdir
+ for i in ${(f)search_results}; do
+ ln -s $i "$_resdir/new/`basename $i`"
done
- # now fold is an array of specified folders
- # term is an array of specified search expressions
+}
- { test "${#term}" = "0" } && {
- error "No search terms specified."
- act "Parameters: ${PARAM}"
- return 1
+search() {
+
+ [[ "$PARAM" = "" ]] && {
+ error "No search terms specified."
+ return 1
}
-
- # no folders specified, search into the addressbook
- { test "${#fold}" = "0" } && {
+
+ typeset -al term
typeset -alU results
- notice "Searching addressbook for: ${PARAM}"
- res=""
-
- for t in ${term}; do
- res+=`search_addressbook ${t}`
- done
-
- for rr in ${(f)res}; do
- _email=`print $rr | awk '{ print $1 }'`
- _name=`print $rr | awk '{ for(c=2;c<=NF;c++) printf "%s ", $c }'`
- results+=("$_name <$_email>")
- done
-
- { test "${#results}" = "0" } || {
- act "${#results} matches found:"
- for i in ${results}; do
- print "$i"; done
- return 0
- }
- notice "No matches found."
- return 1
+
+ for p in ${PARAM}; do
+ func "param ${p} is a search term"
+ term+=(${p})
+ done
+
+ [[ "$PARAM" =~ "addr" ]] && {
+ # if addr specified search into the addressbook
+ notice "Searching addressbook for: ${PARAM//addr/}"
+ res=""
+ for t in ${term}; do
+ [[ "$t" =~ "addr" ]] && continue
+ # res+=`search_addressbook ${t}`
+ search_addressbook ${t} | awk '
+/^$/ { next }
+{ for(c=2;c<=NF;c++) printf "%s ", $c
+ print "<" $1 ">" }'
+ done
+ return 0
}
- # base path is the dir of the first folder
- pushd `dirname ${fold[1]}`
- basedir=`pwd`
- popd
-
- notice "Searching ${#fold} folders in $basedir for: ${term}"
- { command -v mairix > /dev/null } || {
- error "Mairix not found, operation aborted."
- return 1 }
- act "Searching through: ${fold}"
- act "Please wait..."
- id=$datestamp.$RANDOM
- rc=$TMPDIR/search.conf.$id
- # forge the folder string for mairix conf
- folders=""; for f in ${fold}; do folders="$folders`basename $f`:"; done
- cat <<EOF > $rc
-base=$basedir
-database=$TMPDIR/search.db.$id
-maildir=${folders}
-mfolder=$TMPDIR/search.result.$id
-mformat=maildir
-EOF
- { test $DEBUG = 1 } && {
- func "Mairix conf (debug output)"
- cat $rc }
-
- exitcode=0
- { test "$DRYRUN" = "1" } || {
- exitcode=1
- mairix -F -f $rc 2>/dev/null
- { test $? = 0 } && {
- found=`mairix -F -f $rc ${=term} 2> /dev/null | awk '{ print $2}'`
- if [ "$found" = "0" ]; then
- error "No matches found."
- else
- ${=mutt} -F $MUTTDIR/rc -R -f $TMPDIR/search.result.$id
- notice "Found $found matches looking for '$term' in $folders"
- exitcode=0
- fi
- }
- } # DRYRUN
- ${=rm} $TMPDIR/search.db.$id
- ${=rm} $TMPDIR/search.conf.$id
- ${=rm} -r $TMPDIR/search.result.$id
- return $exitcode
+ # run search across emails
+ nm_search ${=PARAM}
+
+ # open the results maildir
+ ${=mutt} -F "$MUTTDIR"/rc ${=muttflags} \
+ -f "$MAILDIRS"/cache/notmuch/results
+
}
backup() {
