#!/bin/bash
# or
#!/bin/zsh
##
## Dovecot FTS: Decode text from various files
## Copyright (c) 2014 SATOH Fumiyasu @ OSS Technlogy Corp., Japan
##
## License: GNU General Public License version 3
##
## Required external commands:
##   nkf(1)
##   pdftotext(1) from poppler
##   wvWare(1)
##   xlhtml(1)
##   ppthtml(1) from xlhtml
##   unzip(1)
##   zipinfo(1) from UnZip
##   openssl(1)
##

set -u
set -o pipefail || exit 1 ## Depend on bash 3.0+ or zsh 5.0+
umask 00027

export LANG="C"
export PATH="/opt/osstech/libexec/dovecot:/opt/osstech/bin:$PATH"

conf="${DOVECOT_DECODE2TEXT_CONF:-/opt/osstech/etc/dovecot/decode2text.conf}"
timeout="${DOVECOT_DECODE2TEXT_TIMEOUT:-10}"
tmpdir="${DOVECOT_DECODE2TEXT_TMPDIR:-/tmp}"

## ======================================================================

perr()
{
  echo "$0: ERROR: $1" 1>&2
}

pdie()
{
  perr "$1"
  exit "${2-1}"
}

clean_tempfile() {
  if [[ -n ${file_temps-} ]]; then
    rm -f "${file_temps[@]}"
  fi
}

create_tempfile() {
  mktemp "$tmpdir/${0##*/}${1:+.$1}.XXXXXX"
}

unset file_temps
typeset -a file_temps
trap clean_tempfile EXIT HUP INT PIPE TERM

timeout() {
  command timeout --signal=KILL "$timeout" "$@"
}

utf8ize_text() {
  nkf --oc=UTF-8 -m0 -x
}

html2text() {
  ## FIXME: &#XXXX;
  ## FIXME: &copy; and so on
  sed \
    -e :a \
    -e '/<[^>]*$/{ N; ba; }' \
    -e 's/<[^>]*>/ /g' \
    -e 's/&nbsp;/ /g' \
    -e 's/&quot;/"/g' \
    -e 's/&gt;/>/g' \
    -e 's/&lt;/</g' \
    -e 's/&#[0-9]*;/ /g' \
    -e 's/&[a-z][a-z]*;/ /g' \
    -e 's/&amp;/\&/g' \
    -e '/^[ 	]*$/d' \
    "$@" \
  ;
}

xml2text() {
  ## FIXME &#XXXX;
  sed \
    -e 's/<[^>]*>/ /g' \
    -e 's/&nbsp;/ /g' \
    -e 's/&quot;/"/g' \
    -e "s/&apos;/'/g" \
    -e 's/&gt;/>/g' \
    -e 's/&lt;/</g' \
    -e 's/&#[0-9]*;/ /g' \
    -e 's/&amp;/\&/g' \
    -e '/^[ 	]*$/d' \
    "$@" \
  ;
}

odf2text() {
  unzip -p -- "$1" 'meta.xml' 'content.xml' \
    |xml2text \
  ;
}

openxml2text() {
  unzip -p -- "$@" \
    |sed \
      -e 's/<p:txBody>/ /g' \
      -e 's/<\/t>/ /g' \
    |xml2text \
  ;
}

## ======================================================================

typeset -A exts_by_type=(
[application/zip]="zip"
[application/x-zip]="zip"
[application/x-zip-compressed]="zip"
[application/x-zip-compress]="zip"
[application/pdf]="pdf"
[application/x-pdf]="pdf"
[application/vnd.openxmlformats-officedocument.wordprocessingml.document]="docx docm"
[application/vnd.openxmlformats-officedocument.wordprocessingml.template]="dotx dotm"
[application/vnd.openxmlformats-officedocument.spreadsheetml.sheet]="xlsx xlsm"
[application/vnd.openxmlformats-officedocument.spreadsheetml.template]="xltx xltm"
[application/vnd.openxmlformats-officedocument.presentationml.presentation]="pptx pptm"
[application/vnd.openxmlformats-officedocument.presentationml.slide]="sldx sldm"
[application/vnd.openxmlformats-officedocument.presentationml.slideshow]="ppsx ppsm"
[application/vnd.openxmlformats-officedocument.presentationml.template]="potx potm"
[application/msword]="doc dot"
[application/vnd.ms-word]="doc"
[application/vnd.ms-excel]="xls xla xlb xlt"
[application/msexcel]="xls"
[application/ms-excel]="xls"
[application/x-msexcel]="xls"
[application/x-ms-excel]="xls"
[application/x-excel]="xls"
[application/vnd.ms-powerpoint]="ppt pps"
[application/mspowerpoint]="ppt"
[application/vnd.oasis.opendocument.text]="odt"
[application/vnd.oasis.opendocument.text-template]="ott"
[application/vnd.oasis.opendocument.spreadsheet]="ods"
[application/vnd.oasis.opendocument.spreadsheet-template]="ots"
[application/vnd.oasis.opendocument.presentation]="odp"
[application/vnd.oasis.opendocument.presentation-template]="otp"
[application/vnd.oasis.opendocument.graphics]="odg"
[application/vnd.oasis.opendocument.graphics-template]="otg"
[application/x-rpm]="rpm"
[application/x-tar]="tar"
[application/x-gtar]="tar gtar"
[application/x-gtar-compressed]="tgz taz"
[application/x-x509-ca-cert]="crt"
[text/plain]="txt text asc"
[text/x-sh]="sh bash ksh zsh"
[application/x-sh]="sh"
[text/x-python]="py"
[application/x-python]="py"
[text/x-ruby]="rb"
[application/x-ruby]="rb"
[text/x-perl]="pl pm"
[application/x-perl]="pl"
[text/x-csrc]="c"
[text/x-chdr]="h"
[text/x-c++src]="c++ cpp cxx cc C"
[text/x-c++hdr]="h++ hpp hxx hh"
[text/x-go]="go"
[text/x-java]="java"
[text/x-haskell]="hs"
[text/x-diff]="diff patch"
[text/csv]="csv"
[text/tab-separated-values]="tsv"
[text/tsv]="tsv"
[text/html]="html htm"
[text/css]="css"
[text/xml]="xml"
)

if [ -f "$conf" ]; then
  . "$conf" || exit 1
fi

## ----------------------------------------------------------------------

if [[ $# -eq 0 ]]; then
  for type in "${!exts_by_type[@]}"; do
    echo "$type ${exts_by_type[$type]}"
  done
  echo
  exit 0
fi

if [[ $# != 1 ]]; then
  echo "Usage: $0 <CONTENT-TYPE|FILE>"
  exit 1
fi

file_name="$1"; shift

typeset -l file_type
file_type="$file_name"
typeset -l file_ext
file_ext=""

file_exts="${exts_by_type[$file_type]-}"

if [[ -n $file_exts ]]; then
  file_name="/dev/stdin"
  file_ext="${file_exts%% *}"
  if [[ ! -e $file_name || -S $file_name ]]; then
    file_temp=$(create_tempfile "$file_ext") || pdie "Cannot create a temporary file"
    file_temps+=("$file_temp")
    cat >"$file_temp" || pdie "Cannot copy input to the temprary file"
    file_name="$file_temp"
  fi
else
  if [[ ! -f $file_name ]]; then
    pdie "Unknown type or no such file: $file_name"
  fi
  file_ext="${file_name##*.}"
  if [[ ${#file_ext} -eq ${#file_name} ]]; then
    pdie "Filename has no extension: $file_name"
  fi
fi

case "$file_ext" in
zip)
  ## List archived filenames only
  zipinfo -1z -- "$file_name"
  ;;
pdf)
  if [[ ! -f $file_name ]]; then
    ## pdftotext cannot handle non-seekable input
    file_temp=$(create_tempfile "$file_ext") || pdie "Cannot create a temporary file"
    file_temps+=("$file_temp")
    cat <"$file_name" >"$file_temp" || pdie "Cannot copy input to the temprary file"
    file_name="$file_temp"
  fi
  pdftotext -enc UTF-8 -nopgbrk -- "$file_name" -
  ;;
do[ct][xm])
  openxml2text "$file_name" 'word/document.xml'
  ;;
xl[st][xm])
  openxml2text "$file_name" 'xl/sharedStrings.xml'
  ;;
pp[ts][xm]|pot[xm]|sld[xm])
  openxml2text "$file_name" 'ppt/slides/slide*.xml'
  ;;
do[ct])
  timeout wvWare --charset=UTF-8 --nographics -- "$file_name" \
    |html2text \
  ;
  ;;
xl[sabt])
  timeout xlhtml -a -te -nc -fw "$file_name" \
    |sed \
      -e 's/<TD[^>]*>&nbsp;//g' \
      -e 's/<TD[^>]*>/ /g' \
      -e "s/^<meta .*<\/TITLE>//" \
      -e "s/<I>Spreadsheet's Author:.*xlhtml [^<]*//" \
    |html2text \
  ;
  ;;
pp[ts])
  timeout ppthtml "$file_name" \
    |sed \
      -e 's/^<HTML>.*//' \
      -e 's/^<hr>.*>pptHtml<.*//' \
    |html2text \
  ;
  ;;
o[dt][tspg])
  odf2text "$file_name"
  ;;
rpm)
  rpm -qilv --provides -p - <"$file_name"
  ;;	
tar|gtar)
  tar tf - <"$file_name"
  ;;	
tgz|taz)
  gzip -dc <"$file_name" |tar tf -
  ;;	
crt)
  openssl x509 -text -noout <"$file_name"
  ;;	
txt|text|[chH]|C|[ch]++|[ch]pp|[ch]xx|cc|hh|diff|patch|[ct]sv|css)
  utf8ize_text <"$file_name"
  ;;
asc|sh|bash|[kz]sh|py|rb|p[lm]|go|java|hs)
  cat <"$file_name"
  ;;
html|htm)
  html2text <"$file_name" |utf8ize_text
  ;;
xml)
  xml2text <"$file_name" # |utf8ize_text
  ;;
*)
  pdie "Unknown file extension: $file_ext ($file_name)"
  ;;
esac

status="$?"
if [[ $status -ne 0 ]]; then
  pdie "Decoder returns error status: $status: $file_ext ($file_name)"
fi

exit 0

