#!/opt/osstech/bin/ruby
##
## Chimera Search: Crawler
## Copyright (c) 2007-2011 SATOH Fumiyasu @ OSS Technology, Inc.
##               <http://www.osstech.co.jp/>
##
## License: GNU General Public License version 2 or later
## Date: 2011-04-11, since 2007-06-18
##

require 'logger'
RAILS_DEFAULT_LOGGER = Logger.new(STDERR)
CHIMERA_APP_TYPE = :crawler
require File.dirname(File.dirname(File.dirname(__FILE__))) + "/config/environment"

require 'chimera/config'
require 'chimera/uri'
require 'chimera/smb'
require 'chimera/estraierpure'
require 'chimera/filter'
require 'chimera/textglob'

require 'cgi'

## Verbose output
def v(level, &msg)
  return unless level == 0 || level <= $c[:verbose_level]
  msg.call.each do |msg|
    puts msg
  end
end

## Warning message
def pwarn(msg)
  STDERR.puts "#{$0}: WARNING: #{msg}"
end

## Error message
def perr(msg)
  STDERR.puts "#{$0}: ERROR: #{msg}"
end

## Show error and die
def pdie(msg)
  perr(msg)
  exit(1)
end

## Crawler class for SMB share
class Crawler
  include EstraierPure

  INDEX_VERSION = 1
  INDEX_LIST_MAX = 1000

  @@dir_class = SMB::Dir
  @@file_class = SMB::File
  @@stat_class = SMB
  @@auth_class = SMB
  @@sidresover_class = SMB::SIDResolver

  attr_accessor :terminate_flag
  attr_accessor :max_file_size, :min_file_size
  attr_accessor :max_text_size, :min_text_size
  attr_accessor :index_sync_per_files
  attr_accessor :file_owner_source_type
  attr_reader :directory_count, :file_count, :target_file_count
  attr_reader :new_file_count, :updated_file_count, :old_file_count, :lamed_file_count
  attr_reader :removed_file_count
  attr_reader :put_file_count, :put_file_size
  attr_reader :put_lamed_file_count, :put_lamed_file_size
  attr_reader :out_file_count, :out_file_size

  def initialize(est_uri, est_user, est_password, est_node_name, est_node_links)
    @est_uri = est_uri.kind_of?(URI) ? est_uri : URI.parse(est_uri)
    @est_user = @est_uri.user ? URI.unescape(@est_uri.user) : est_user
    @est_pass = @est_uri.password ? URI.unescape(@est_uri.password) : est_password
    @est_uri.password = nil
    @est_node_name = est_node_name
    @est_node_links = est_node_links

    @master = NodeMaster.new
    @master.set_url(@est_uri.merge('/master').to_s)
    @master.set_auth(@est_user, @est_pass)
    unless @master.list_node
      raise "index server not available: master node status: " +
	"#{@master.status} (#{@master.status_class})"
    end

    @node = @master.get_node(@est_node_name)
    unless @node.label
      unless @master.create_node(@est_node_name)
	raise "cannot create node: master node status: " +
	  "#{@master.status} (#{@master.status_class})"
      end
      @node = @master.get_node(@est_node_name)
      if @est_node_links > 1
	@est_node_links.times do |node_num|
	  node_name_n = '%s.%02d' % [@est_node_name, node_num]
	  @master.create_node(node_name_n)
	  node_n = @master.get_node(node_name_n)
	  ## hyperestraier-1.4.13/mastermod.h:#define SELFCREDIT 10000
	  @node.set_link(node_n.url, node_name_n, 10000)
	end
      end
    end

    if @node.links.count > 0 || @est_node_links > 1
      @nodes = Array.new(@est_node_links)
      @node.links.each do |node_link|
	(node_uri, node_label, node_credit) = node_link.split(/\t/)
	node_id = node_label.sub(/^.*\D/, '').to_i
	if @nodes[node_id]
	  raise "index link id duplicated: #{node_uri}"
	end
	if node_id >= @est_node_links
	  raise "index link id out of range: #{node_uri}"
	end
	@nodes[node_id] = NodeMaster.new
	@nodes[node_id].set_url(node_uri)
	## FIXME: Do not set_auth if node_uri has auth info
	@nodes[node_id].set_auth(@est_user, @est_pass)
      end
      @nodes.each_with_index do |node, node_id|
	raise "node link id not found: #{node_id}" unless node
      end
    else
      @nodes = []
    end

    @filter = Hash.new

    @max_file_size = @min_file_size = @max_text_size = @min_text_size = -1
    @index_sync_per_files = 0

    self.reset_stats
  end

  def reset_stats()
    @directory_count = @file_count = @target_file_count = 0
    @new_file_count = @updated_file_count = @old_file_count = @lamed_file_count = 0
    @removed_file_count = 0
    @put_file_count = @put_file_size = 0
    @put_lamed_file_count = @put_lamed_file_size = 0
    @out_file_count = @out_file_size = 0
  end

  def index_timeout=(time)
    [@node, *@nodes].each do |node|
      unless node.set_timeout(time)
	raise "cannot set timeout for index: node status: " +
	  "#{node.status} (#{node.status_class})"
      end
    end
  end

  def list_index
    return [@node, *@nodes].map do |node|
      [
	node.name,
	node.label,
	node.size.to_i,
	node.doc_num,
	node.word_num,
	node.cache_usage.to_i,
      ]
    end
  end

  def backup_index
    v(1){"backup index: start"}
    unless @master.backup
      raise "cannot backup index: node status: " +
	"#{@master.status} (#{@master.status_class})"
    end
    v(1){"backup index: end"}
  end

  def rotate_index_log
    v(1){"rotate index log: start"}
    unless @master.rotate_log
      raise "cannot rotate index log: node status: " +
	"#{@master.status} (#{@master.status_class})"
    end
    v(1){"rotate index log: end"}
  end

  def reopen_index_log
    v(1){"reopen index log: start"}
    unless @master.reopen_log
      raise "cannot reopen index log: node status: " +
	"#{@master.status} (#{@master.status_class})"
    end
    v(1){"reopen index log: end"}
  end

  def optimize_index
    v(1){"optimize index: start"}
    [@node, *@nodes].each do |node|
      unless node.optimize
	raise "cannot optimize index: node status: " +
	  "#{node.status} (#{node.status_class})"
      end
    end
    v(1){"optimize index: end"}
  end

  def target_uri=(uri)
    @target_uri = uri.kind_of?(URI) ? uri.dup : URI::SMB.parse(uri)
    @target_uri.path.sub!(/\/*$/, '').gsub!(/\/\/+/, '/')

    domainuser = @target_uri.domainuser
    user = domainuser[:user] ?
      URI.unescape(domainuser[:user]) :
      $c[:target_user]
    domain = domainuser[:domain] ?
      URI.unescape(domainuser[:domain]) :
      user.sub!(/^([^\\]*)\\/, '') ? $1 : nil
    password = @target_uri.password ?
      URI.unescape(@target_uri.password) :
      $c[:target_password]
    @target_uri.user = @target_uri.password = nil

    self.set_target_auth(domain, user, password)
  end

  def set_target_auth(domain, user, password)
    auth = [domain, user, password]
    @@auth_class.on_authentication {|smb_server, smb_share|
      v(5){"auth: smb://%s/%s: domain=%s, user=%s, password=%s" %
	[smb_server, smb_share, auth[0], auth[1], (auth[2] ? auth[2].gsub(/./, '*') : '')]}
      auth
    }
    begin
      @@auth_class.set_credentials(domain, user, password)
    rescue NoMethodError
      ## Could not resolve DFS links by libsmbclient
      ## FIXME: Warn user
    end
  end

  def target_exts=(exts)
    @target_exts = exts
    @target_exts_re = Regexp.new("\\.(#{@target_exts.join('|')})$", Regexp::IGNORECASE)
  end

  def exclude_uris=(uri_globs)
    unless uri_globs && uri_globs.length > 0
      @exclude_uris = @exclude_uris_re_str = @exclude_uris_re = nil
      return
    end
    @exclude_uris = uri_globs
    ## NOTE: Do not use non-POSIX regexp for H.E. that does not support it
    @exclude_uris_re_str = '(%s)' %
      uri_globs.map{|uri_glob|
	Chimera::TextGlob.regexp_string(uri_glob.sub(%r"/$", ''), Chimera::TextGlob::NO_STRICT_LEADING_DOT)
      }.join('|');
    @exclude_uris_re = Regexp.new("^#{@exclude_uris_re_str}$", Regexp::IGNORECASE)
  end

  def exclude_directories=(dir_globs)
    unless dir_globs && dir_globs.length > 0
      @exclude_directories = @exclude_directories_re_str = @exclude_directories_re = nil
      return
    end
    @exclude_directories = dir_globs
    ## NOTE: Do not use non-POSIX regexp for H.E. that does not support it
    @exclude_directories_re_str = '(%s)' %
      dir_globs.map{|dir_glob|
	Chimera::TextGlob.regexp_string(dir_glob, Chimera::TextGlob::NO_STRICT_LEADING_DOT)
      }.join('|');
    @exclude_directories_re = Regexp.new("^#{@exclude_directories_re_str}$", Regexp::IGNORECASE)
  end

  def exclude_files=(file_globs)
    unless file_globs && file_globs.length > 0
      @exclude_files = @exclude_files_re_str = @exclude_files_re = nil
      return
    end
    @exclude_files = file_globs
    ## NOTE: Do not use non-POSIX regexp for H.E. that does not support it
    @exclude_files_re_str = '(%s)' %
      file_globs.map{|file_glob|
	Chimera::TextGlob.regexp_string(file_glob, Chimera::TextGlob::NO_STRICT_LEADING_DOT)
      }.join('|');
    @exclude_files_re = Regexp.new("^#{@exclude_files_re_str}$", Regexp::IGNORECASE)
  end

  def filter=(filter)
    @filter = filter
  end

  def force_update=(flag)
    force_update_old = @force_update
    @force_update = flag

    return force_update_old
  end

  def check_lamed=(flag)
    check_lamed_old = @check_lamed
    @check_lamed = flag

    return check_lamed_old
  end

  def node_by_doc_uri(uri)
    return @node if @nodes.empty?
    ## DJB hash
    return @nodes[(uri.size + uri.unpack("C*").inject{|r,i|((r<<5)+r)^i}) % @nodes.size]
    ## Sum hash
    #return @nodes[(uri.size + uri.unpack("C*").inject{|sum,n|sum+n}) % @nodes.size]
  end

  def put_doc(doc)
    return node_by_doc_uri(doc.attr('@uri')).put_doc(doc)
  end

  ## Crawl on SMB share to process new and updated files
  def crawl_updated_files
    put_file_count_pre = @put_file_count

    begin
      stat = @@stat_class.stat(@target_uri)
    rescue SystemCallError => e
      raise "cannot connect to target URI: #{@target_uri}: #{e}"
    end

    if stat.dir?
      self.crawl_smb_dir(@target_uri)
    elsif stat.file?
      self.crawl_smb_file(@target_uri)
    end

    if @index_sync_per_files > 0 && put_file_count_pre != @put_file_count
      v(1){"check updated: index: syncing"}
      @node.sync
    end
  end

  ## Crawl on SMB directory
  def crawl_smb_dir(dir_uri)
    if @exclude_directories_re
      dir_basename = dir_uri.to_s.sub(%r'^.*/', '')
      if @exclude_directories_re.match(dir_basename)
	v(3){"check updated: dir: #{dir_uri}: skipped: by exclude-directories option"}
	return
      end
    end
    if @exclude_uris_re
      if @exclude_uris_re.match(dir_uri.to_s)
	v(3){"check updated: dir: #{dir_uri}: skipped: by exclude-uris option"}
	return
      end
    end

    v(3){"check updated: dir: #{dir_uri}: entering"}

    begin
      dir = @@dir_class.open(dir_uri)
    rescue Errno::ENOENT
      ## OK: Removed while to open
      return
    rescue Errno::EACCES, Errno::EPERM
      v(2){"check updated: dir: #{dir_uri}: skipped: permission denied to open"}
      return
    rescue SystemCallError => e
      raise "cannot open directory: #{dir_uri}: #{e}"
    end

    begin
      dir.each_entry do |entry|
	break if @terminate_flag

	## Skip '.' and '..'
	next if entry.name =~ /^\.\.?$/

	## FIXME: SIGSEGV protector (libsmbclient's bug?)
	if entry.uri.size > 1000
	  v(0){"check updated: dir: #{entry.uri}: skipped: URI too long"}
	  next
	end

	if entry.dir?
	  @directory_count += 1
	  self.crawl_smb_dir(entry.uri)
	  next
	elsif entry.file?
	  @file_count += 1
	  self.crawl_smb_file(entry.uri)
	  next
	end
      end
    ensure
      dir.close
    end
    v(3){"check updated: dir: #{dir_uri}: leaving"}
  end
  protected :crawl_smb_dir

  ## Crawl SMB file
  def crawl_smb_file(file_uri)
    unless file_uri.match(@target_exts_re)
      v(3){"check updated: file: #{file_uri}: skipped: not target"}
      return
    end

    if @exclude_files_re
      file_basename = file_uri.to_s.sub(%r'^.*/', '')
      if @exclude_files_re.match(file_basename)
	v(3){"check updated: file: #{file_uri}: skipped: by exclude-files option"}
	return
      end
    end
    if @exclude_uris_re
      if @exclude_uris_re.match(file_uri.to_s)
	v(3){"check updated: file: #{file_uri}: skipped: by exclude-uris option"}
	return
      end
    end

    v(4){"check updated: file: #{file_uri}: trying to get file status"}
    begin
      stat = @@stat_class.stat(file_uri)
    rescue Errno::ENOENT
      ## OK: Removed while to stat
      return
    rescue Errno::EACCES, Errno::EPERM
      v(3){"check updated: file: #{file_uri}: skipped: permission denied"}
      return
    rescue SystemCallError => e
      raise "cannot stat file: #{file_uri}: #{e}"
    end

    @target_file_count += 1

    v(5){"check updated: file status: mode=%#o" % stat.mode}
    v(5){
      %w(dev ino nlink uid gid size).map do |attr_name|
	"check updated: file status: #{attr_name}=#{stat.send(attr_name)}"
      end
    }
    v(5){
      %w(atime ctime mtime).map do |attr_name|
	"check updated: file status: #{attr_name}=#{stat.send(attr_name).utc.xmlschema}"
      end
    }

    ## Check filesize
    if @max_file_size >= 0 && stat.size > @max_file_size
      v(1){"check updated: file: #{file_uri}: skipped: max filesize (>#{@max_file_size})"}
      return
    end
    if @min_file_size >= 0 && stat.size < @min_file_size
      v(1){"check updated: file: #{file_uri}: skipped: min filesize (<#{@min_file_size})"}
      return
    end

    file_node = node_by_doc_uri(file_uri)

    ## Check if this file is new, updated or not updated
    add_reason = 'UNKNOWN REASON'
    if @force_update
      add_reason = 'force update'
    elsif doc = file_node.get_doc_by_uri(file_uri, Node::GET_DOC_NOTEXT|Node::GET_DOC_NOKWD)
      v(4){"check updated: file: #{file_uri}: found in index"}
      v(5){
	doc.attr_names.sort.map do |attr_name|
	  "check updated: indexed file attr: #{attr_name}=#{doc.attr(attr_name)}"
	end
      }
      if doc.attr_int('@_version') != INDEX_VERSION
	## OK, continue
	add_reason = 'different index version'
      elsif doc.attr_time('@mdate') != stat.mtime
	## OK, continue
	add_reason = 'modified'
#      elsif doc.attr_time('@cdate') != stat.ctime
#	## OK, continue
#	## FIXME: H.E. defines @cdate as 'create date', not 'status change date'.
#	add_reason = 'status change'
      elsif @check_lamed && status = doc.attr('@_lamed_status')
	## OK, continue
	add_reason = "lamed in index: #{status}"
	@lamed_file_count += 1
      else
	## This document is already indexed and is not modified.
	v(3){"check updated: file: #{file_uri}: skipped: not modified"}
	@old_file_count += 1
	return
      end
      @updated_file_count += 1
    else
      add_reason = 'new'
      @new_file_count += 1
    end

    v(4){"check updated: file: #{file_uri}: generating file info for index"}
    unless doc = self.make_doc(file_uri, stat)
      return
    end

    v(5){
      doc.attr_names.sort.map do |attr_name|
	"check updated: generated file attr: #{attr_name}=#{doc.attr(attr_name)}"
      end
    }
    v(5){"check updated: generated file text size: #{doc.text_size}"}
    v(10){"check updated: generated file text:\n#{doc.texts}"}

    ## Add file to index (or update file info in index)
    v(1){"check updated: file: #{file_uri}: adding to index: #{add_reason}"}
    unless status = file_node.put_doc(doc)
      if file_node.status == 413 ## HTTPRequestEntityTooLarge
	v(0){"filter: file: #{file_uri}: lamed: text size too large"}
	doc.add_attr('@_lamed_status', 'text size too large')
	doc.clear_text
	status = file_node.put_doc(doc)
      end
      unless status
	raise "cannot add file info to index: #{file_uri}: node status: " +
	  "#{file_node.status} (#{file_node.status_class})"
      end
    end

    v(3){"check updated: file: #{file_uri}: added to index: #{add_reason}"}
    @put_file_size += stat.size
    @put_file_count += 1
    if doc.attr('@_lamed_status')
      @put_lamed_file_size += stat.size
      @put_lamed_file_count += 1
    end

    if @index_sync_per_files > 0 && @put_file_count % @index_sync_per_files == 0
      v(1){"check updated: index: syncing"}
      @node.sync
    end
  end
  protected :crawl_smb_file

  ## Create Estraier document object from file
  def make_doc(file_uri, stat)
    doc = Document.new
    ## Required attributes
    doc.add_attr('@uri', file_uri)
    ## Extra attributes
    doc.add_attr_int('@_version', INDEX_VERSION)
    doc.add_attr_time('@_crawled_date', Time.now)
    doc.add_attr_time('@_indexed_date', Time.now)
    file_name = File.basename(file_uri)
    doc.add_attr('@name', file_name)
    doc.add_attr('@title', file_name.sub(/\.\w+$/, ''))
    doc.add_attr_int('@size', stat.size)
    doc.add_attr_time('@mdate', stat.mtime)
    doc.add_attr_time('@cdate', stat.ctime)
    doc.add_hidden_text(file_uri)

    begin
      if !@file_owner_source_type
        owner_name = @@stat_class.getxattr(file_uri, 'system.nt_sec_desc.owner+')
        if owner_name.sub!(/^([^\\]+)\\/, '')
	  ## FIXME: Need?
	  #doc.add_attr('owner_domain', $1)
          ## FIXME: Need SID?
        end
        doc.add_attr('@owner', owner_name)
      elsif owner_sid = @@stat_class.getxattr(file_uri, 'system.nt_sec_desc.owner')
        ## NOTE: Samba 3.0.25a and older have crash bug (Samba Bug 4683)
        ## that is triggered by the following line. That is why I use
        ## rpcclient to resolve SID into domain\onwer. -- fumiyas 2007-06-10
	if owner = @@sidresover_class.lookup(owner_sid, URI.unescape(@target_uri.host))
	  doc.add_attr('@owner', owner.name)
	  ## FIXME: Need?
	  #doc.add_attr('owner_domain', owner.domain)
	  #doc.add_attr('owner_sid', owner.sid)
	  ## FIXME: Need? Local, domain or builtin account?
	  #doc.add_attr('owner_type', owner.type)
	else
	  v(2){"check updated: file: #{file_uri}: cannot resolve owner SID"}
	  doc.add_attr('@_lamed_status', "cannot resolve owner SID")
	  doc.add_attr('@owner', owner_sid)
	end
      else
	v(0){"check updated: file: #{file_uri}: cannot get owner information"}
	doc.add_attr('@_lamed_status', "cannot get owner information")
      end
    rescue SystemCallError => e
      v(0){"check updated: file: #{file_uri}: cannot get owner information: #{e}"}
    end

    ext = file_uri.match(/\.(\w+)$/)[1].downcase
    unless filter = @filter[ext]
      v(0){"filter: file: #{file_uri}: lamed: filter not found"}
      doc.add_attr('@_lamed_status', 'filter not found')
      return doc
    end

    begin
      file = @@file_class.open(file_uri, 'r')
    rescue Errno::ENOENT, Errno::EISDIR
      ## Ignore
      return nil
    rescue Errno::EACCES, Errno::EPERM, Errno::EBUSY => e
      ## EBUSY is returned on NT_STATUS_SHARING_VIOLATION
      v(0){"check updated: file: #{file_uri}: lamed: cannot open to read: #{e}"}
      doc.add_attr('@_lamed_status', "cannot open to read: #{e}")
      return doc
    rescue SystemCallError => e
      raise "cannot open file: #{file_uri}: #{e}"
    end

    begin
      fdoc = filter.process(file_uri, file)
    rescue Chimera::FilterError => e
      v(0){"filter: file: #{file_uri}: lamed: filter failed: #{e}"}
      doc.add_attr('@_lamed_status', "filter failed: #{e}")
      return doc
    ensure
      v(4){"filter: file: #{file_uri}: error message: #{filter.error || 'NONE'}"}
      file.close
    end

    ## Check textsize
    if @max_text_size >= 0 && fdoc.text.size > @max_text_size
      v(2){"check updated: file: #{file_uri}: skipped: max textsize (>#{@max_text_size})"}
      return nil
    end
    if @min_text_size >= 0 && fdoc.text.size < @min_text_size
      v(2){"check updated: file: #{file_uri}: skipped: min textsize (<#{@min_text_size})"}
      return nil
    end

    if fdoc.title
      doc.add_attr('@title', fdoc.title)
      doc.add_hidden_text(fdoc.title)
    end
    doc.add_attr('@author', fdoc.author) if fdoc.author
    doc.add_attr('@type', fdoc.mime_type) if fdoc.mime_type
    doc.add_text(fdoc.text)

    return doc
  end
  protected :make_doc

  ## Crawl on index to process removed files from SMB share
  def crawl_removed_files()
    out_file_count_pre = @out_file_count

    loop_count = 0
    prev_file_uri = nil
    loop do
      break if @terminate_flag

      loop_count += 1

      v(3){"check removed: #{@target_uri}: #{loop_count}: listing files to check"}
      unless docs = @node.list(INDEX_LIST_MAX, prev_file_uri)
	raise "cannot list files from index to check removed: node status: " +
	  "#{@node.status} (#{@node.status_class})"
      end
      v(2){"check removed: #{@target_uri}: #{loop_count}: files to check: #{docs.size}"}

      break if docs.size == 0

      docs.each do |doc|
	break if @terminate_flag

	file_uri = doc.attr('@uri')
	file_node = self.node_by_doc_uri(file_uri)

	@removed_file_count += 1
	v(3){"check removed: file: #{file_uri}: checking if file exists"}
	v(5){
	  doc.attr_names.sort.map do |attr_name|
	    "check removed: file attr: #{attr_name}=#{doc.attr(attr_name)}"
	  end
	}

	v(4){"check removed: file: #{file_uri}: trying to get file status"}
	begin
	  file_uri_stat = @@stat_class.stat(file_uri)
	rescue Errno::ENOENT
	  v(3){"check removed: file: #{file_uri}: continue: removed"}
	rescue Errno::EACCES, Errno::EPERM
	  v(3){"check removed: file: #{file_uri}: continue: permission denied to stat"}
	rescue SystemCallError => e
	  raise "cannot stat file: #{file_uri}: #{e}"
	end
	if file_uri_stat
	  v(4){"check removed: file: #{file_uri}: skipped: existing"}
	  prev_file_uri = file_uri
	  next
	end

	v(1){"check removed: file: #{file_uri}: removing from index"}
	unless file_node.out_doc_by_uri(file_uri)
	  raise "cannot remove file from index: #{file_uri}: node status: " +
	    "#{file_node.status} (#{file_node.status_class})"
	end

	v(2){"check removed: file: #{file_uri}: removed from index"}
	@out_file_size += doc.attr_int('@size')
	@out_file_count += 1
	if @index_sync_per_files > 0 && @out_file_count % @index_sync_per_files == 0
	  v(1){"check removed: index: syncing"}
	  @node.sync
	end
      end
    end

    if @index_sync_per_files > 0 && out_file_count_pre != @out_file_count
      v(1){"check removed: index: syncing"}
      @node.sync
    end
  end

  def remove_target_uri
    target_uri = @target_uri.to_s.sub(/([^\/])$/, '\1/')
    self.remove_files_by_uri("remove uri: #{@target_uri}", target_uri)
  end

  def remove_excluded
    if @exclude_uris_re_str
      uris_re = "^#{@exclude_uris_re_str}(/|$)"
      self.remove_files_by_uri('remove excluded: uris', uris_re, 'ISTRRX')
    end
    return if @terminate_flag

    if @exclude_directories_re_str
      dirs_re = "/#{@exclude_directories_re_str}/"
      self.remove_files_by_uri('remove excluded: dirs', dirs_re, 'ISTRRX')
    end
    return if @terminate_flag

    if @exclude_files_re_str
      files_re = "/#{@exclude_files_re_str}$"
      self.remove_files_by_uri('remove excluded: files', files_re, 'ISTRRX')
    end
  end

  def remove_files_by_uri(task, uri, op='ISTRBW')
    out_file_count_pre = @out_file_count
    exp = "@uri #{op} #{uri}"
    v(5){"#{task}: expression: #{exp}"}

    @node.set_snippet_width(0, 0, 0)
    cond = Condition.new
    cond.add_attr(exp)
    cond.set_max(INDEX_LIST_MAX)
    loop_count = 0
    loop do
      break if @terminate_flag

      loop_count += 1

      v(3){"#{task}: #{loop_count}: searching files to remove"}
      unless nres = @node.search(cond, 1)
	raise "cannot search files from index to remove: node status: " +
	  "#{@node.status} (#{@node.status_class})"
      end
      v(2){"#{task}: #{loop_count}: files to remove: #{nres.doc_num}"}

      break if nres.doc_num == 0

      nres.doc_num.times do |i|
	break if @terminate_flag

	doc = nres.get_doc(i)
	file_uri = doc.attr('@uri')
	file_node = self.node_by_doc_uri(file_uri)

	v(1){"#{task}: file: #{file_uri}: removing from index"}
	v(5){
	  doc.attr_names.sort.map do |attr_name|
	    "#{task}: file attr: #{attr_name}=#{doc.attr(attr_name)}"
	  end
	}

	unless file_node.out_doc_by_uri(file_uri)
	  raise "cannot remove file from index: #{file_uri}: node status: " +
	    "#{file_node.status} (#{file_node.status_class})"
	end

	v(2){"#{task}: file: #{file_uri}: removed from index"}
	@out_file_size += doc.attr_int('@size')
	@out_file_count += 1
	if @index_sync_per_files > 0 && @out_file_count % @index_sync_per_files == 0
	  v(1){"#{task}: index: syncing"}
	  @node.sync
	end
      end

      break if nres.doc_num < INDEX_LIST_MAX
    end

    if @index_sync_per_files > 0 && out_file_count_pre != @out_file_count
      v(1){"#{task}: index: syncing"}
      @node.sync
    end
  end
end

## Main
## ======================================================================

if __FILE__ == $0

$stdout.sync = $stderr.sync = true
Signal.trap('INT') { pdie "interrupted" }
Signal.trap('TERM') { pdie "terminated" }
Signal.trap('USR1') {} ## NOP

## Command-line options
## ======================================================================

begin
  $c.load(ARGV)
rescue SystemCallError, Chimera::ConfigError => e
  pdie "cannot parse command-line options: #{e}"
end

SMB::SIDResolver.rpcclient_cmd = $c[:rpcclient_command]

## Filter definition
## ======================================================================

Chimera::Filter.timeout = $c[:filter_timeout]
Chimera::Filter.tmpdir = $c[:filter_tmpdir]
Chimera::FilterCommand.command_timeout = $c[:filter_command_timeout]
Chimera::FilterCommand.path = $c[:filter_command_path]

html_filter = Proc.new {|fdoc|
  unless fdoc.title
    ## NOTE: wvWare outputs NULL byte from broken(?) MS-Word document
    if m = fdoc.text.match(/<title(?:\s[^>]*)?>(.+?)<\/title\s*>/im) and !m[1].match(/\x00/)
      fdoc.title = CGI.unescapeHTML(m[1].gsub(/<[^>]*>/m, '')).strip
    elsif m = fdoc.text.match(/<h1(?:\s[^>]*)?>(.+?)<\/h1\s*>/im) and !m[1].match(/\x00/)
      fdoc.title = CGI.unescapeHTML(m[1].gsub(/<[^>]*>/m, '')).strip
    end
  end
  if fdoc.text.match(/<meta\s+name="author"\s+content="([^"]+)"/im)
    fdoc.author = CGI.unescapeHTML($1).strip
  end

  fdoc.text.gsub!(/(?:\s*<!--.*?-->\s*)+/m, "\n")
  fdoc.text.gsub!(/(?:\s*<[^>]*>\s*)+/m, "\n")
  fdoc.text.gsub!(/(?:\s*&nbsp;\s*)+/, "\n")
  fdoc.text = CGI.unescapeHTML(fdoc.text)
}
xml_filter = Proc.new {|fdoc|
  fdoc.text = CGI.unescapeHTML(fdoc.text.gsub(/(<[^>]*>)+/m, "\n").gsub(/&apos;/, "'"))
}
openxml_filter = Proc.new {|fdoc|
  fdoc.text.gsub!(%r!<p:txBody>(?:.*?)</p:txBody>!) do |txbody|
    txbody.gsub(/<a:p>/, "\n").gsub(/<[^>]*>/, '')
  end
  xml_filter.call(fdoc)
}

filter = Hash.new
filter['txt'] = filter['text'] = filter['csv'] = filter['tsv'] = Chimera::Filter.new(
  :name =>	'Chimera standard plain text filter',
  :version =>	1
)
filter['html'] = filter['htm'] =
filter['xhtml'] = filter['xht'] =
filter['shtml'] = filter['shtm'] = filter['sht'] = Chimera::Filter.new(
  :name =>	'Chimera standard HTML filter',
  :version =>	1,
  :filter =>	html_filter
)
## OpenOffice.org, Sun StarOffice/StarSuite 6+
filter['odt'] = filter['ods'] = filter['odp'] = filter['odg'] =
filter['ott'] = filter['ots'] = filter['otp'] = filter['otg'] =
filter['sxw'] = filter['sxc'] = filter['sxi'] = filter['sxd'] =
filter['stw'] = filter['stc'] = filter['sti'] = filter['std'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard ODF filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:unzip_command],
  :args =>	['-p', '/dev/stdin', 'meta.xml', 'content.xml'],
  :filter =>	Proc.new {|fdoc|
    fdoc.text = CGI.unescapeHTML(fdoc.text.gsub(/(<[^>]*>)+/m, "\n").gsub(/&apos;/, "'"))
  }
)
filter['doc'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard MS Word filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:wvware_command],
  :args =>	['--charset=UTF-8', '--nographics', '/dev/stdin'],
  :filter =>	html_filter
)
filter['xls'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard MS Excel filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:xlhtml_command],
  :args =>	['-a', '-te', '-nc', '-fw', '/dev/stdin'],
  :read_filter => Proc.new {|chunk|
    ## Remove many "<TD>&nbsp</TD>"s to reduce memory usage
    chunk.gsub(/(?:<TD(?:\s[^>]*)?>&nbsp;)+/, '')
  },
  :filter =>	Proc.new {|fdoc|
    sheet_names = Array.new
    ## Remove "<title>/dev/stdin</title>"
    fdoc.text.sub!(/<title>.*?<\/title>/im, '')
    ## Adjust footer
    fdoc.text.sub!(/Spreadsheet's Author:([^<]*)(?:(?:(?!<br>).)*<br>){4}(?=<\/body>)/im, ' \1 ')
    ## Scan sheet names to use as the document title
    fdoc.text.gsub!(/<h1(?:\s[^>]*)?>(.*?)<\/h1\s*>/im) do |chunk|
      sheet_name = CGI.unescapeHTML($1.gsub(/<[^>]*>/m, '')).strip
      case sheet_name
      ## FIXME: I18N
      when /^(?:Sheet|表)\d+$/
	## Ignore default sheet titles
      when ''
	## Ignore empty sheet titles
      else
	sheet_names << sheet_name
      end

      next nil
    end
    if sheet_names.size > 0
      fdoc.title = sheet_names.join(', ')
    end
    html_filter.call(fdoc)
  }
)
filter['ppt'] = filter['pps'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard MS PowerPoint filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:ppthtml_command],
  :args =>	['/dev/stdin'],
  :filter =>	Proc.new {|fdoc|
    ## Remove "<title>/dev/stdin</title>"
    fdoc.text.sub!(/<title>.*?<\/title>/im, '')
    ## Remove footer
    fdoc.text.sub!(/.*\n(?=.*\n\z)/i, '')
    html_filter.call(fdoc)
  }
)
## MS Word 2007
filter['docx'] = filter['docm'] =	## Document
filter['dotx'] = filter['dotm'] =	## Document Template
Chimera::FilterCommand.new(
  :name =>	'Chimera standard Office Open XML WordprocessingML filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:unzip_command],
  :args =>	['-p', '/dev/stdin', 'word/document.xml'],
  :filter =>	openxml_filter
)
## MS Excel 2007
#filter['xlsb'] =			## Binary Book (non-XML) (FIXME)
#filter['xlam'] =			## Add-in (FIXME)
filter['xlsx'] = filter['xlsm'] =	## Book
filter['xltx'] = filter['xltm'] =	## Book Template
Chimera::FilterCommand.new(
  :name =>	'Chimera standard Office Open XML SpreadsheetML filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:unzip_command],
  :args =>	['-p', '/dev/stdin', 'xl/workbook.xml', 'xl/sharedStrings.xml'],
  :filter =>	Proc.new {|fdoc|
    sheet_names = Array.new
    fdoc.text.sub!(%r!<workbook(?:\s[^>]*)?>.*?</workbook>!) do |workbook|
      workbook.scan(%r!<sheet(?:\s[^>]*)?>!) do |sheet|
        if m = sheet.match(/^.*\sname="([^"]*)/) and !m[1].match(/^Sheet\d+$/)
          sheet_names << m[1]
        end
      end
      next nil
    end
    if sheet_names.size > 0
      fdoc.title = sheet_names.join(', ')
    end

    openxml_filter.call(fdoc)
  }
)
## MS PowerPoint 2007
#filter['ppam'] =			## Add-in (FIXME)
#filter['thmx'] =			## Office Theme (FIXME)
filter['pptx'] = filter['pptm'] =	## Presentation
filter['potx'] = filter['potm'] =	## Presentation Template
filter['ppsx'] = filter['ppsm'] =	## Slide show
filter['sldx'] = filter['sldm'] =	## Slide
Chimera::FilterCommand.new(
  :name =>	'Chimera standard Office Open XML PresentationML filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:unzip_command],
  :args =>	['-p', '/dev/stdin', 'ppt/slides/slide*.xml'],
  :filter =>	openxml_filter
)
filter['pdf'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard PDF filter',
  :version =>	1,
  :encoding =>	'UTF-8',
  :command =>	$c[:pdftotext_command],
  :args =>	['-enc', 'UTF-8', '-raw', '-nopgbrk', '/dev/stdin', '-'],
  :filter =>	html_filter
)
filter['xdw'] = filter['xbd'] = Chimera::FilterCommand.new(
  :name =>	'Chimera standard Fuji Xerox DocuWorks filter',
  :version =>	1,
  :encoding =>	'CP932',
  :command =>	$c[:xdw2text_command],
  ## Note: xdw2text with -p option uses the last command-line argument
  ##       as a temporary filename that must NOT be an existing file.
  :args =>	[
    '-p',
    '/dev/stdin',
    Proc.new {"#{$c[:filter_tmpdir]}/chimera.xdw2text.#{$$}.tmp"}
  ]
)

begin
  require 'exifr'
  filter['jpg'] = filter['jpeg'] = Chimera::Filter.new(
    :name =>	'Chimera standard JPEG (EXIF) filter',
    :version =>	1,
    :reader => Proc.new {|input|
      EXIFR::JPEG.new(input).to_hash.map do |attr, value|
	"#{attr}: #{value}"
      end.join("\n")
    }
  )
  filter['tif'] = filter['tiff'] = Chimera::Filter.new(
    :name =>	'Chimera standard TIFF (EXIF) filter',
    :version =>	1,
    :reader => Proc.new {|input|
      EXIFR::TIFF.new(input).to_hash.map do |attr, value|
	"#{attr}: #{value}"
      end.join("\n")
    }
  )
rescue LoadError
  ## Ignore
end

if doccat_cmd = $c[:doccat_command]
  doccat_filter = Chimera::FilterCommand.new(
    :name =>		'Chimera standard Dehenken DocCat filter',
    :version =>		1,
    :encoding =>	'UTF-8',
    :command =>		$c[:doccat_command],
    :args => [
      '-p',		## Print document property information
      '-o', '8',	## Output is UTF-8
      '-T',		## Trim extra newlines from PDF output
      '/dev/stdin'
    ]
  )

  ## FIXME: DocCat filter always overrides OSS filters

  doccat_h = `'#{doccat_cmd.gsub(/'/, %q{'"'"'})}' -h`
  if m = doccat_h.match(/\sLicense flag: (\d+)(.*)$/) and m[1] == "0"
    ## MS Office
    filter['doc'] = filter['xls'] = filter['ppt'] = filter['pps'] =
    ## MS Office 2007
    filter['docx'] = filter['xlsx'] = filter['pptx'] = filter['ppsx'] =
    filter['docm'] = filter['xlsm'] = filter['pptm'] = filter['ppsm'] =
    ## JustSystems Ichitaro 5
    filter['jaw'] = filter['jtw'] =
    ## JustSystems Ichitaro 6
    filter['jbw'] = filter['juw'] =
    ## JustSystems Ichitaro 7
    filter['jfw'] = filter['jvw'] =
    ## JustSystems Ichitaro 8 and later
    filter['jtd'] = filter['jtt'] =
    ## Lotus WordPro
    filter['lwp'] =
    ## Fujitsu OASYS
    filter['oa2'] = filter['oa3'] =
    ## RTF
    filter['rtf'] =
    ## XML Paper Specification (XPS)
    filter['xps'] =
    ## XML
    filter['xml'] =
      doccat_filter
  else
    pdie "invalid DocCat license: #{m[1]}#{m[2]}"
  end
  if m = doccat_h.match(/\sPDF Lic flag: (\d+)(.*)$/) and m[1] == "0"
    filter['pdf'] = doccat_filter
  else
    ## Ignore
    #pdie "invalid DocCat PDF option license: #{m[1]}#{m[2]}"
  end
end

$c[:target_exts] = filter.keys if $c[:target_exts].size == 0

## ======================================================================

crawler = Crawler.new(
  $c[:index_uri],
  $c[:index_user],
  $c[:index_password],
  $c[:index_name],
  $c[:index_links]
)

Signal.trap('USR1') { crawler.terminate_flag = true }

case
when $c[:list_index]
  [%w(NAME LABEL SIZE FILES WORDS CACHE%), %w(---- ----- ---- ----- ----- ------), *crawler.list_index].map do |line|
    puts '%-16s %-16s %12s %10s %10s %6s' % line
  end
  exit(0)
when $c[:backup_index]
  crawler.backup_index
  exit(0)
when $c[:rotate_index_log]
  crawler.rotate_index_log
  exit(0)
when $c[:reopen_index_log]
  crawler.reopen_index_log
  exit(0)
when $c[:optimize_index]
  crawler.optimize_index
  exit(0)
end

unless $c[:no_lock]
  ## FIXME: On Solaris, a file cannot be lock through a read-only file handle.
  ##        That is, we must use a writeable lock file. I think Solaris is
  ##        correct, Linux is bad. -- fumiyas, 2007-09-09
  lock = File.new($c[:lock_file] || CHIMERA_CONF, 'r')
  unless lock.flock(File::LOCK_EX | File::LOCK_NB)
    pdie "cannot obtain lock: another crawler process is running?"
  end
end

if $c[:remove_uri]
  target_uris = $c[:remove_uri] == 'ALL' ?
    $c[:target_uris] : [$c[:remove_uri]]
else
  target_uris = $c[:target_uris]
end

crawler.index_timeout = $c[:index_timeout] if $c[:index_timeout]
crawler.index_sync_per_files = $c[:index_sync_per_files]
crawler.target_exts = $c[:target_exts]
crawler.filter = filter
crawler.file_owner_source_type = $c[:file_owner_source_type]
crawler.force_update = $c[:force_update]
crawler.check_lamed = $c[:check_lamed]

crawler.exclude_uris = $c[:exclude_uris] if $c[:exclude_uris]
crawler.exclude_directories = $c[:exclude_directories] if $c[:exclude_directories]
crawler.exclude_files = $c[:exclude_files] if $c[:exclude_files]

crawler.max_file_size = $c[:max_file_size]
crawler.min_file_size = $c[:min_file_size]
crawler.max_text_size = $c[:max_text_size]
crawler.min_text_size = $c[:min_text_size]

## :remove_excluded option is exclusive option with other options
if $c[:remove_excluded]
  v(1){"remove excluded: start"}
  crawler.reset_stats
  time_start = Time.now
  crawler.remove_excluded
  time_diff = Time.now - time_start
  v(1){"remove excluded: statistics: removed file: %d" %
    [crawler.out_file_count]}
  size_av = crawler.out_file_count > 0 ?
    (crawler.out_file_size/crawler.out_file_count).to_s : '-'
  v(1){"remove excluded: statistics: " +
    "removed file total size (per file): %d (%s)" %
    [crawler.out_file_size, size_av]}
  time_av = crawler.out_file_count > 0 ?
    time_diff/crawler.out_file_count : '-'
  v(1){"remove excluded: statistics: time (per file): %s (%s)" %
    [time_diff, time_av]}
  v(1){"remove excluded: end"}
  exit(0);
end

target_uris.each do |target_uri|
  crawler.target_uri = target_uri = URI::SMB.parse(target_uri)

  ## Removing files from the index
  ## ======================================================================

  ## :remove_uri option is exclusive option with other options

  if $c[:remove_uri]
    v(1){"remove uri: start: #{target_uri}"}
    crawler.reset_stats
    time_start = Time.now
    crawler.remove_target_uri
    time_diff = Time.now - time_start
    v(1){"remove uri: statistics: URI: %s" %
      [target_uri]}
    v(1){"remove uri: statistics: removed file: %d" %
      [crawler.out_file_count]}
    size_av = crawler.out_file_count > 0 ?
      (crawler.out_file_size/crawler.out_file_count).to_s : '-'
    v(1){"remove uri: statistics: " +
      "removed file total size (per file): %d (%s)" %
      [crawler.out_file_size, size_av]}
    time_av = crawler.out_file_count > 0 ?
      time_diff/crawler.out_file_count : '-'
    v(1){"remove uri: statistics: time (per file): %s (%s)" %
      [time_diff, time_av]}
    v(1){"remove uri: end: #{target_uri}"}
    next
  end

  ## Crawling
  ## ======================================================================

  if $c[:check_updated]
    v(1){"check updated: start: #{target_uri}"}
    crawler.reset_stats
    time_start = Time.now
    begin
      crawler.crawl_updated_files
    rescue SystemCallError => e
      perr e
    end
    time_diff = Time.now - time_start
    v(1){"check updated: statistics: URI: %s" %
      [target_uri]}
    v(1){"check updated: statistics: directory: %d" %
      [crawler.directory_count]}
    v(1){"check updated: statistics: file: %d" %
      [crawler.file_count]}
    v(1){"check updated: statistics: target file: %d" %
      [crawler.target_file_count]}
    v(1){"check updated: statistics: new target file: %d" %
      [crawler.new_file_count]}
    v(1){"check updated: statistics: updated target file: %d" %
      [crawler.updated_file_count]}
    v(1){"check updated: statistics: lamed target file: %d" %
      [crawler.lamed_file_count]} if $c[:check_Lamed]
    v(1){"check updated: statistics: old target file: %d" %
      [crawler.old_file_count]}
    v(1){"check updated: statistics: indexed target file (lamed): %d (%d)" %
      [crawler.put_file_count, crawler.put_lamed_file_count]}
    size_av = crawler.put_file_count > 0 ?
      (crawler.put_file_size/crawler.put_file_count).to_s : '-'
    v(1){"check updated: statistics: " +
      "indexed target file total size (per file): %d (%s)" %
      [crawler.put_file_size, size_av]}
    time_av = crawler.put_file_count > 0 ?
      time_diff/crawler.put_file_count : '-'
    v(1){"check updated: statistics: time (per file): %s (%s)" %
      [time_diff, time_av]}
    v(1){"check updated: end: #{target_uri}"}
  end

  if $c[:check_removed]
    v(1){"check removed: start: #{target_uri}"}
    crawler.reset_stats
    time_start = Time.now
    begin
      crawler.crawl_removed_files
    rescue SystemCallError => e
      perr e
    end
    time_diff = Time.now - time_start
    v(1){"check removed: statistics: URI: %s" %
      [target_uri]}
    v(1){"check removed: statistics: checked file: %d" %
      [crawler.removed_file_count]}
    v(1){"check removed: statistics: removed file: %d" %
      [crawler.out_file_count]}
    size_av = crawler.out_file_count > 0 ?
      (crawler.out_file_size/crawler.out_file_count).to_s : '-'
    v(1){"check removed: statistics: " +
      "removed file total size (per file): %d (%s)" %
      [crawler.out_file_size, size_av]}
    time_av = crawler.out_file_count > 0 ?
      time_diff/crawler.out_file_count : '-'
    v(1){"check removed: statistics: time (per file): %s (%s)" %
      [time_diff, time_av]}
    v(1){"check removed: end: #{target_uri}"}
  end
end

## ======================================================================

exit(0)

end

