#!/usr/bin/ruby -w
##
## SMB Search: Crawler
## Copyright (c) 2007 OSS Technology Co., Japan <http://www.osstech.co.jp/>
##
## Date: 2007-06-10, since 2007-06-18
## Author: SATOH Fumiyasu
##

require 'smbsearch/config'
require 'smbsearch/uri'
require 'smbsearch/smb'
require 'smbsearch/estraierpure'
require 'smbsearch/filter'

## Resolve SID into account info via rpcclient(1)
class LookupSid
  @@rpcclient_cmd = ENV['SMBS_RPCCLIENT_CMD'] || '/usr/bin/rpcclient'
  @@cache = Hash.new

  def self.lookup(server, sid)
    raise "invalid server name: #{server}" unless server =~ /^\w[\w\-\.]+$/
    raise "invalid SID: #{sid}" unless sid =~ /^S(-\d+){3,7}$/

    key = server + '/' + sid
    return @@cache[key] if @@cache.has_key?(key)

    @@cache[key] = nil

    out = `#{@@rpcclient_cmd} -d0 -U% -c 'lookupsids #{sid}' '#{server}'`
    if match = out.match(/^(S(?:-\d+)+)\s([^\\]*)\\(.+)\s+\((\d+)\)$/)
      @@cache[key] = SMBAccount.new(*match[1..4])
    elsif match = out.match(/^result was (NT_STATUS_\w+)$/)
      warn "rpcclient failed: #{$1}"
    else
      warn "rpcclient failed: unknown error: #{out}"
    end

    return @@cache[key]
  end
end

## Verbose output
def v(level, msg)
  puts msg if level == 0 || level <= $c[:verbose_level]
end

## Warning message
def warn(msg)
  STDERR.puts "#{$0}: WARNING: #{msg}"
end

## Error message
def err(msg)
  STDERR.puts "#{$0}: ERROR: #{msg}"
end

## Show error and die
def die(msg)
  err(msg)
  exit 1
end

## Crawler class for SMB share
class Crawler
  include EstraierPure

  SMBS_DOC_VERSION = 1

  @@dir_class = SMB::Dir
  @@file_class = SMB::File
  @@stat_class = SMB
  @@auth_class = SMB

  attr_reader :directory_count, :file_count, :target_file_count
  attr_reader :new_file_count, :old_file_count, :updated_file_count
  attr_reader :processed_file_count, :processed_file_size
  attr_reader :failed_file_count, :failed_file_size

  def initialize(est_uri, filter)
    est_uri = URI.parse(est_uri) unless est_uri.kind_of?(URI)
    @est_user = URI.unescape(est_uri.user)
    @est_pass = URI.unescape(est_uri.password)
    est_uri.user = est_uri.password = nil
    @est_uri = est_uri.to_s
    @filter = filter

    @master = NodeMaster.new
    @master.set_auth(@est_user, @est_pass)
    @master.set_url("#{@est_uri}/master")

    @node = Node.new
    @node.set_auth(@est_user, @est_pass)

    self.reset_stats
  end

  def reset_stats()
    @directory_count = @file_count = @target_file_count = 0
    @new_file_count = @old_file_count = @updated_file_count = 0
    @processed_file_count = @processed_file_size = 0
    @failed_file_count = @failed_file_size = 0
  end

  ## Crawl SMB share
  def crawl(smb_uri, ext)
    smb_uri = URI.parse(smb_uri) unless smb_uri.kind_of?(URI)
    smb_auth = [smb_uri.domain, smb_uri.user, smb_uri.password]
    smb_uri.domain = smb_uri.user = smb_uri.password = nil
    @ext = ext
    @ext_re = Regexp.new("\\.(#{@ext.join('|')})$", Regexp::IGNORECASE)

    @@auth_class.on_authentication {|smb_server, smb_share|
      v 5, "smb auth: smb://#{smb_server}/#{smb_share}/: domain=#{smb_auth[0]}, user=#{smb_auth[1]}, pass=#{smb_auth[2].gsub(/./, '*')}"
      smb_auth
    }

    begin
      smb_uri_stat = @@stat_class.stat(smb_uri)
    rescue Exception => e
      raise "cannot connect to share: #{smb_uri}: #{e}"
    end

    if smb_uri_stat.dir?
      return crawl_dir(smb_uri)
    elsif smb_uri_stat.file?
      return crawl_file(smb_uri)
    end
  end

  ## Crawl SMB directory
  def crawl_dir(dir_uri)
    v 2, "dir: #{dir_uri}: entering"

    begin
      dir = @@dir_class.open(dir_uri)
    rescue Errno::EACCES, Errno::EPERM => e
      v 2, "dir: #{dir_uri}: skipped: permission denied"
      return
    rescue Exception => e
      raise "cannot open directory: #{dir_uri}: #{e}"
    end

    begin
      dir.each_entry do |entry|
	## Skip '.' and '..'
	next if entry.name =~ /^\.\.?$/

	## FIXME: SIGSEGV protector (libsmbclient's bug?)
	if entry.uri.length > 1000
	  v 0, "dir: #{entry.uri}: skipped: URI too long"
	  next
	end

	if entry.dir?
	  @directory_count += 1
	  crawl_dir(entry.uri)
	  next
	elsif entry.file?
	  @file_count += 1
	  crawl_file(entry.uri)
	  next
	end
      end
    ensure
      dir.close
    end
    v 2, "dir: #{dir_uri}: leaving"
  end

  ## Crawl SMB file
  def crawl_file(file_uri)
    unless file_uri.match(@ext_re)
      v 2, "file: #{file_uri}: skipped: not target"
      return
    end

    begin
      stat = @@stat_class.stat(file_uri)
    rescue Errno::EACCES, Errno::EPERM => e
      v 0, "file: #{file_uri}: skipped: permission denied"
      return
    rescue Exception => e
      raise "cannot stat file: #{file_uri}: #{e}"
    end

    @target_file_count += 1

    ## smb://server/share/file -> smb://server/share
    smb_share_uri = file_uri.sub(%r#^([^/:]+://([^/]+)/([^/]+)).*$#, '\1')
    smb_server, smb_share = $2, $3
    ## Set Estraier node URI for share
    @node.set_url_for_uri("#{@est_uri}/node", smb_share_uri)
    ## FIXME: test
    @master.delete_node(Node.name_for_uri(smb_share_uri))

    ## Check if this file is new, modified or not modified
    if doc_id = @node.uri_to_id(file_uri)
      if @node.get_doc_attr_int(doc_id, '_version') != SMBS_DOC_VERSION
	## OK, continue
	v 3, "file: #{file_uri}: updated: database version"
      elsif @node.get_doc_attr_time(doc_id, '@mdate') != stat.mtime
	## OK, continue
	v 3, "file: #{file_uri}: updated: @mdate"
      elsif @node.get_doc_attr_time(doc_id, 'cdate') != stat.ctime
	## OK, continue
	v 3, "file: #{file_uri}: updated: cdate"
      else
	## This document is already registered and is not modified.
	v 2, "file: #{file_uri}: skipped: not modified"
	@old_file_count += 1
	return
      end
      file_is_new = false
      @updated_file_count += 1
    else
      file_is_new = true
      @new_file_count += 1
    end

    begin
      ## NOTE: Samba 3.0.25a and older have crash bug (Samba BUG 4683)
      ## that is triggered by the following line. That is why I use
      ## rpcclient to resolve SID into domain\onwer. -- fumiyas 2007-06-10
      #owner_name = entry.getxattr('system.nt_sec_desc.owner+')
      owner_sid = @@stat_class.getxattr(file_uri, 'system.nt_sec_desc.owner')
      owner = LookupSid.lookup(smb_server, owner_sid)
    rescue Exception => e
      v 0, "file: #{file_uri}: skipped: cannot get owner information: #{e}"
      @failed_file_count += 1
      @failed_file_size += stat.size
      return
    end

    begin
      file = @@file_class.open(file_uri, 'r')
    rescue Errno::EACCES, Errno::EPERM => e
      v 0, "file: #{file_uri}: skipped: permission denied"
      @failed_file_count += 1
      @failed_file_size += stat.size
      return
    end

    v 1, "file: #{file_uri}: registering: " + (file_is_new ? 'new' : 'update')
    begin
      unless doc = make_doc(file_uri, stat, owner, file)
	@failed_file_count += 1
	@failed_file_size += stat.size
	return
      end
    ensure
      file.close
    end

    ## Create node
    ## FIXME: Check if node exists or not
    @master.add_node(Node.name_for_uri(smb_share_uri), smb_share_uri)

    ## Register file to database
    unless @node.put_doc(doc)
      @failed_file_count += 1
      @failed_file_size += stat.size
      raise "cannot register file to database: #{file_uri}: node status: #{@node.status.to_s}"
    end

    v 2, "file: #{file_uri}: registered"
    @processed_file_size += stat.size
    @processed_file_count += 1

    if $c[:verbose_level] >= 3
      doc.attr_names.sort.each do |attr_name|
	v 3, "file attr: #{attr_name}=#{doc.attr(attr_name)}"
      end
      v 10, "file text:\n#{doc.texts}"
    end
  end

  ## Create Estraier document object from file
  def make_doc(file_uri, stat, owner, file)
    doc = Document.new
    ## Required attributes
    doc.add_attr('@uri', file_uri)
    ## SMB-Search attributes
    doc.add_attr_int('_version', SMBS_DOC_VERSION)
    doc.add_attr_time('_mdate', Time.now)
    doc.add_attr('@title', File.basename(file_uri))
    doc.add_attr_int('@size', stat.size)
    doc.add_attr_time('@mdate', stat.mtime)
    doc.add_attr_time('cdate', stat.ctime)

    if owner
      doc.add_attr('owner', owner.name)
      ## FIXME: Needed?
      #doc.add_attr('owner_domain', owner.domain)
      #doc.add_attr('owner_sid', owner.sid)
      ## FIXME: Needed? Local, domain or builtin account?
      #doc.add_attr('owner_type', owner.type)
    end

    ext = file_uri.match(/\.(\w+)$/)[1].downcase
    unless filter = @filter[ext]
      v 0, "file: #{file_uri}: skipped: filter not found"
      return nil
    end

    begin
      rawdoc = filter.process(file)
    rescue Exception => e
      v 0, "file: #{file_uri}: skipped: filter failed: #{e}"
      return nil
    end

    doc.add_attr('@title', rawdoc.title) if rawdoc.title
    doc.add_attr('@author', rawdoc.author) if rawdoc.author
    doc.add_attr('@type', rawdoc.type) if rawdoc.type
    doc.add_text(rawdoc.text)

    return doc
  end
end

## Main
## ======================================================================

if __FILE__ == $0

## Options
## ======================================================================

$stdout.sync = $stderr.sync = true
Signal.trap('INT') { die "interrupted" }
Signal.trap('TERM') { die "terminated" }

$c = SmbSearch::Config.new(
  :verbose_level => {
    :type =>	SmbSearch::Config::TYPE_INTEGER,
    :default =>	ENV['SMBS_VERBOSE_LEVEL'].to_i || 0
  },
  :db_server_uri => {
    :ok_on =>	%r#^https?://.*#
  },
  :target_smb_uris => {
    :type =>	SmbSearch::Config::TYPE_STRING,
    :array? =>	true,
    :default =>	[],
  },
  :target_exts => {
    :type =>	SmbSearch::Config::TYPE_STRING,
    :array? =>	true,
    :default =>	ENV['SMBS_TARGET_EXT'] ?
      ENV['SMBS_TARGET_EXT'].strip.split(/\s+/) :
      %w(odt ods odp doc xls ppt pdf txt htm html xhtml xht)
  },
  :purge => {
    :type =>	SmbSearch::Config::TYPE_BOOL,
    :default =>	false
  },
  :unzip_cmd => {
    :default => ENV['SMBS_UNZIP_CMD'] || '/usr/bin/unzip'
  },
  :wvware_cmd => {
    :default =>	ENV['SMBS_WVWARE_CMD'] || '/usr/bin/wvWare'
  },
  :xlhtml_cmd => {
    :default =>	ENV['SMBS_XLHTML_CMD'] || '/usr/bin/xlhtml'
  },
  :ppthtml_cmd => {
    :default =>	ENV['SMBS_PPTHTML_CMD'] || '/usr/bin/ppthtml'
  },
  :pdftotext_cmd => {
    :default =>	ENV['SMBS_PDFTOTEXT_CMD'] || '/usr/bin/pdftotext'
  }
)

begin
  $c.load(ENV['SMBS_CRAWLER_CONF'] || '/etc/smbsearch/crawler.conf')
rescue Exception => e
  die "cannot load configuration file: #{e}"
end

begin
  $c.load(ARGV)
rescue Exception => e
  die "cannot parse command-line options: #{e}"
end

## Filter
## ======================================================================

filter = Hash.new
filter['txt'] =
  SmbSearch::Filter.new('text/plain', false, false)
filter['html'] = filter['htm'] =
  SmbSearch::Filter.new('text/html', false, true)
filter['xhtml'] = filter['xht'] =
  SmbSearch::Filter.new('text/xhtml+xml', false, true)
## FIXME: Set mime-type in filter proc
filter['odt'] = filter['ods'] = filter['odp'] =
  SmbSearch::FilterCommand.new(nil, true, false,
    $c[:unzip_cmd], '-p', $stdin, 'meta.xml', 'content.xml') {|doc|
      ## FIXME
      #unzip_cmd, '-ca', $stdin, 'mimetype', 'meta.xml', 'content.xml'
      #if (doc.text.sub(/^(application\/[\w\-.]+\.opendocument\.[\w\-.]+)$/, ''))
      #	doc.type = $1
      #end
      doc.text.gsub!(/(<.*?>)+/, "\n")
    }
filter['doc'] =
  SmbSearch::FilterCommand.new('application/msword', true, true,
    $c[:wvware_cmd], '--charset=UTF-8', '--nographics', $stdin)
filter['xls'] =
  SmbSearch::FilterCommand.new('application/vnd.ms-excel', true, true,
    $c[:xlhtml_cmd], $stdin) {|doc|
      ## Remove filename in <title> to ignore it
      doc.text.sub!(/<title>.*<\/title>/im, '')
    }
filter['ppt'] =
  SmbSearch::FilterCommand.new('application/vnd.ms-powerpoint', true, true,
    $c[:ppthtml_cmd], $stdin) {|doc|
      ## Remove filename in <title> to ignore it
      doc.text.sub!(/<title>.*<\/title>/im, '')
    }
filter['pdf'] =
  SmbSearch::FilterCommand.new('application/pdf', true, true,
    $c[:pdftotext_cmd], '-enc', 'UTF-8', '-raw', '-nopgbrk', $stdin)

## Crawling
## ======================================================================

begin
  crawler = Crawler.new($c[:db_server_uri], filter)
  $c[:target_smb_uris].each do |smb_uri|
    start_time = Time.now
    crawler.reset_stats
    crawler.crawl(smb_uri, $c[:target_exts])
    time = Time.now - start_time

    smb_share_uri = smb_uri.sub(%r#^([^/:]+://)([^/]*@)?(([^/]+)/([^/]+)).*$#, '\1\3')
    v 1, "statistics: share: #{smb_share_uri}"
    v 1, "statistics: directory: #{crawler.directory_count}"
    v 1, "statistics: file: #{crawler.file_count}"
    v 1, "statistics: target file (not): #{crawler.target_file_count} (#{crawler.file_count-crawler.target_file_count})"
    v 1, "statistics: old target file: #{crawler.old_file_count}"
    v 1, "statistics: new target file: #{crawler.new_file_count}"
    v 1, "statistics: updated target file: #{crawler.updated_file_count}"
    v 1, "statistics: processed target file: #{crawler.processed_file_count}"
    size_av = crawler.processed_file_count > 0 ?
      (crawler.processed_file_size/crawler.processed_file_count).to_s : '-'
    v 1, "statistics: processed target file size (per file): #{crawler.processed_file_size} (#{size_av})"
    v 1, "statistics: failed target file: #{crawler.failed_file_count}"
    size_av = crawler.failed_file_count > 0 ?
      (crawler.failed_file_size/crawler.failed_file_count).to_s : '-'
    v 1, "statistics: failed target file size (per file): #{crawler.failed_file_size} (#{size_av})"
    time_av = crawler.processed_file_count > 0 ?
      time/crawler.processed_file_count : '-'
    v 1, "statistics: crawled time (per file): #{time} (#{time_av})"
  end
rescue Exception => e
  raise e
end

exit 0 unless $c[:purge]

## Purging
## ======================================================================

begin
rescue Exception => e
  raise e
end

exit 0

## ======================================================================

end