#!/usr/bin/env python

import sys, getopt, imaplib, tempfile, signal
import urllib2
import traceback
import os, os.path, time, popen2, select, ConfigParser, fcntl
import logging
from email.Parser import Parser

verbose = False
config = None
configfile = "/usr/etc/fred/transproc.conf"

logfile = None

def usage():
    print '%s [-c config] [-v]' % sys.argv[0]

def debug(msg):
    logging.debug(msg)
    #global verbose
    #if verbose:
    #    sys.stdout.write(msg + '\n')

def error(msg):
    logging.error(msg)
    #sys.stderr.write(msg + '\n')

class Processor(object):
    '''Class representing a processor of transcript.'''
    def __init__(self, name):
        self.name = name
        self.type = None
        self.data_type = None
        self.data_mimetype = None
        self.input_encoding = 'utf8'
        self.fromhead = None
        self.script = None
        self.error_seen = False

    def init_source(self):
        pass

    def finalize_source(self):
        pass

    def get_statements(self):
        pass

    def process_statement_file(self, statement_file):
        self.error_seen = False

        #debug('[%s]: processing message %s' % (self.name, msgid))
        # run the processor:
        #    stdin:  raw source data
        #    stdout: result of processing (XML)
        #    stderr: errors
        try:
            (status, outxml, err) = runCommand(self.script, statement_file['octets'])
        except Exception, e:
            error('[%s]: execution of processor failed: %s' % (self.name, e))
            error(traceback.format_exc())
            self.error_seen = True
            return
        if status == 2:
            # we will avoid processing nexttime ($not true right now$)
            error('[%s]: message %s rejected: %s' % (self.name, statement_file['msgid'], err))
        elif status != 0:
            if status != 1:
                error('[%s]: unknown return code (%s): %s' %
                        (self.name, status, err))
            else:
                error('[%s]: processing of data failed: %s' %
                        (self.name, err))
            # we will reprocess the email nexttime
            self.error_seen = True
        else:
            debug('[%s]: message processed by processor' % (self.name))
            statement_file['outxml'] = outxml

    def upload_xml(self, statement_file):
        # update database
        outxml = statement_file.get('outxml')
        if outxml == '':
            debug('[%s]: Statement file have empty outxml (propably no payments in input source).' % self.name)
            return
        if outxml is None:
            error('[%s]: Statement file have no outxml, you must first procoss it!' % self.name)
            return

        save_statement = (self.data_type == 'statement'
                          and statement_file.get('octets'))
        if save_statement:
            tmp_file, tmp_filename = tempfile.mkstemp(prefix='transproc_source_bankstatement')
            os.write(tmp_file, statement_file['octets'])
            os.close(tmp_file)

        try:
            cmd = '%s --bank_import_xml' % config.get('general', 'backendcmd')

            if save_statement:
                cmd += ' --bank_statement_file=%s --bank_statement_file_mimetype=%s' % (tmp_filename, self.data_mimetype)
            if self.input_encoding != 'utf8':
                outxml = outxml.decode(self.input_encoding).encode('utf8')

            #open('/tmp/outxml-%s.xml' % (self.name + statement_file.get('msgid', '')) , 'w').write(outxml)
            (status, output, err) = runCommand(cmd, outxml, read_iterations=config.getint('general', 'backendcmd_max_read_iterations')) # this needs higher read_iterations as it does a lot of ouput to stdout @UnusedVariable
            if status != 0:
                error('[%s]: Error when executing command: %s\n%s' % (self.name, cmd, err))
                # restore the UNSEEN flag - something in script failed
                # we will reprocess the email nexttime
                self.error_seen = True
            else:
                #debug('[%s]: Backend\'s output:' % self.name)
                #debug(output)
                if err:
                    error('[%s]: Error from backend:\n' % self.name)
                    error(err)
                else:
                    debug('[%s]: Output XML sent to backend by processor' % (self.name))
                self.mark_as_processed(statement_file.get('msgid'))
        finally:
            if save_statement:
                os.remove(tmp_filename)

    def mark_as_processed(self, ident):
        ''' Mappes processed data source (e.g. particular email) (if possible),
            so that it will not be processed again.
        '''
        pass


class IMAPProcessor(Processor):
    def __init__(self, name):
        super(IMAPProcessor, self).__init__(name)
        self.type = 'imap'
        self.imaphost = None
        self.imapuser = None
        self.imappass = None
        self.imapfolder = None
        self.imapssl = None
        self.part = 2

        self.subject_filter = ''

        self.server = None

    def init_source(self):
        if self.imapssl:
            self.server = imaplib.IMAP4_SSL(self.imaphost)
        else:
            self.server = imaplib.IMAP4(self.imaphost)
        debug('[%s]: Connected to IMAP %s' % (self.name, self.imaphost))
        self.server.login(self.imapuser, self.imappass)
        debug('[%s]: IMAP logged in as %s' % (self.name, self.imapuser))
        if self.imapfolder:
            self.server.select(self.imapfolder)
        else:
            self.server.select() # preserve default argument "mailbox" of select() method

    def finalize_source(self):
        self.server.close()
        self.server.logout()
        debug('[%s]: IMAP logged out' % self.name)

    def get_statements(self):
        # get unseen messages from a bank
        filters = ['(UNSEEN)', '(FROM %s)' % self.fromhead]
        if self.subject_filter:
            filters.append('(SUBJECT "%s")' % self.subject_filter)
        query = '(%s)' % ' '.join(filters)


        r, data = self.server.search(None, query) #@UnusedVariable
        msgids = data[0].split(' ')
        if not msgids[0]:
            debug('[%s]: No new messages in mailbox' % self.name)
            return []
        # fetch new messages
        messages = {}
        statement_files = []
        for msgid in msgids:
            (r, data) = self.server.fetch(msgid, '(BODY.PEEK[])') # ".PEEK" means that FLAG \Seen is not set! @UnusedVariable

            messages[msgid] = data[0][1]

        # process new messages by processor

        for msgid in messages:
            # separate attachment in which we are interested
            parser = Parser()
            mail = parser.parsestr(messages[msgid])


            if self.part > 1 and not mail.is_multipart():
                debug('[%s]: skipping message %s, it is not multipart' %
                        (self.name, msgid))
                self.mark_as_processed(msgid)
                continue
            try:
                part = mail.get_payload(self.part - 1) # parts are counted from 0
            except IndexError:
                debug('[%s]: skipping message %s, it has not %s parts as needed' %
                        (self.name, msgid, self.part))
                self.mark_as_processed(msgid)
                continue

            statement_file = {}
            statement_file['msgid'] = msgid
            statement_file['filename'] = part.get_filename()
            statement_file['mimetype'] = self.data_mimetype
            statement_file['octets'] = part.get_payload(decode=True)
            statement_files.append(statement_file)
        debug('[%s]: %d new messages' % (self.name, len(statement_files)))
        return statement_files

    def mark_as_processed(self, ident):
        self.server.store(ident, '+FLAGS', '\\Seen')


class HTTPProcessor(Processor):
    def __init__(self, name):
        super(HTTPProcessor, self).__init__(name)
        self.type = 'http'
        self.input_file = None
        self.url = None

    def init_source(self):
        debug('[%s]: Opening HTTP source' % self.name)
        opener = urllib2.build_opener()
        try:
            page = opener.open(self.url)
            if page.code == 200:
                self.input_file = page.read()
                debug('[%s]: HTTP source successfully obtained' % self.name)
            else:
                self.error_seen = True
                error('[%s]: HTTP source returned code %s' % (self.name, page.code))
        except (IOError, urllib2.URLError), e:
            self.error_seen = True
            error('[%s]: Failed to connect to HTTP source:\n%s' % (self.name, e))

    def get_statements(self):
        statement_file = {}
        statement_file['octets'] = self.input_file
        return [statement_file]

def makeNonBlocking(fd):
    """
    Set non-blocking attribute on file.
    """
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY) #@UndefinedVariable

def runCommand(cmd, stdin, read_iterations=8):
    """
    Run command in non-blocking manner.
    read_iterations parametr is rather number of iteration in read stdout+err cycle.
    """
    # run the command
    child = popen2.Popen3(cmd, True)
    try:
        if (stdin):
            child.tochild.write(stdin)
        child.tochild.close()
    except IOError, e:
        error('Writing/closing stdin of child process coused IOERRor number %s' % e.errno)
        # try to read stdin & stderr
        debug('stdout of child:')
        debug(child.fromchild.read())
        debug('stderr of child:')
        debug(child.childerr.read())
        raise


    outfile = child.fromchild
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)
    makeNonBlocking(errfd)
    outdata = errdata = ''
    outeof = erreof = 0
    for round in range(int(read_iterations)):
        # wait for input at most 1 second
        ready = select.select([outfd, errfd], [], [], 1.0)
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '':
                outeof = 1
            else:
                outdata += outchunk
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '':
                erreof = 1
            else:
                errdata += errchunk
        if outeof and erreof: break
        time.sleep(0.3) # give a little time for buffers to fill

    child.fromchild.close()
    child.childerr.close()

    status = os.waitpid(child.pid, os.WNOHANG)

    if status[0] == 0:
        time.sleep(1)
        error("Child doesn't want to exit, TERM signal sent.")
        os.kill(child.pid, signal.SIGTERM)
        time.sleep(1.2) # time to exit
        status = os.waitpid(child.pid, os.WNOHANG)

        if status[0] == 0:
            error("Child doesn't want to die, KILL signal sent.")
            os.kill(child.pid, signal.SIGKILL)
            time.sleep(1.2) # time to exit
            status = os.waitpid(child.pid, os.WNOHANG)

    stat = 9 # by default assume error
    if outeof and erreof and (status[0] == child.pid) and os.WIFEXITED(status[1]):
        stat = os.WEXITSTATUS(status[1])
    return stat, outdata, errdata


def main():
    global verbose
    global config
    global configfile
    error_seen = False
    # parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:v', []) #@UnusedVariable
    except getopt.GetoptError:
        usage()
        sys.exit(2)
    # parameter values
    for o, a in opts:
        if o == '-c':
            configfile = a
        elif o == '-v':
            verbose = True
        else:
            usage()
            sys.exit(2)
    # read configuration file
    config = ConfigParser.ConfigParser(
        {'procdir':'/usr/libexec/transproc',
         'logfile':'/var/log/transproc.log',
         'backendcmd_max_read_iterations': '30'}
    )
    config.read(configfile)

    if not config.has_option('general', 'backendcmd'):
        error('Backend command in configuration file must be set')
        sys.exit(2)

    if not config.has_option('general', 'logfile'):
        error('logfile must be set')
        sys.exit(2)

    if not config.has_option('general', 'loglevel'):
        error('loglevel must be set')
        sys.exit(2)

    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    console_handler = logging.StreamHandler()
    console_handler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
    if verbose:
        console_handler.setLevel(logging.DEBUG)
    else:
        console_handler.setLevel(logging.INFO)
    logger.addHandler(console_handler)

    loglevel = getattr(logging, config.get('general', 'loglevel').upper(), logging.DEBUG)
    file_handler = logging.FileHandler(config.get('general', 'logfile'), 'a')
    file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    file_handler.setLevel(loglevel)
    logger.addHandler(file_handler)



    # iterate over other sections in config
    processors = []
    for section in config.sections():
        if section == 'general':
            continue

        if config.has_option(section, 'type'):
            if config.get(section, 'type') == 'imap':
                proc = IMAPProcessor(section)
            elif config.get(section, 'type') == 'http':
                proc = HTTPProcessor(section)
            else:
                error('type directive must be set in %s section' % section)
                sys.exit(2)
        else:
            error('type directive must be set in %s section' % section)
            sys.exit(2)

        if config.has_option(section, 'data_type'):
            data_type = config.get(section, 'data_type')
            if data_type not in ('statement', 'payments'):
                error('data_type directive must "statement" or "payments" in section %s' % section)
                sys.exit(2)
            proc.data_type = data_type
        else:
            error('data_type directive must be set in %s section' % section)
            sys.exit(2)

        if proc.data_type == 'statement':
            proc.data_mimetype = config.get(section, 'data_mimetype')

        if not config.has_option(section, 'processor'):
            error('processor directive must be set in %s section' % section)
            sys.exit(2)
        proc.script = os.path.join(config.get('general', 'procdir'), config.get(section, 'processor'))
        if not os.path.isfile(proc.script):
            error('processor script "%s" does not exist' % proc.script)
            sys.exit(2)
        if config.has_option(section, 'input_encoding'):
            proc.input_encoding = config.get(section, 'input_encoding')

        if proc.type == 'imap':
            if not config.has_option(section, 'imaphost'):
                error('imaphost directive must be set in %s section' % section)
                sys.exit(2)
            proc.imaphost = config.get(section, 'imaphost')
            if not config.has_option(section, 'imappass'):
                error('imappass directive must be set in %s section' % section)
                sys.exit(2)
            proc.imappass = config.get(section, 'imappass')
            if not config.has_option(section, 'imapuser'):
                error('imapuser directive must be set in %s section' % section)
                sys.exit(2)
            proc.imapuser = config.get(section, 'imapuser')
            if not config.has_option(section, 'imapfolder'):
                error('imapfolder directive must be set in %s section' % section)
                sys.exit(2)
            proc.imapfolder = config.get(section, 'imapfolder')
            if not config.has_option(section, 'imapssl'):
                error('imapssl directive must be set in %s section' % section)
                sys.exit(2)
            proc.imapssl = config.getboolean(section, 'imapssl')
            if not config.has_option(section, 'fromhead'):
                error('fromhead directive must be set in %s section' % section)
                sys.exit(2)
            proc.fromhead = config.get(section, 'fromhead')
            if config.has_option(section, 'part'):
                proc.part = config.getint(section, 'part')
            if config.has_option(section, 'subject_filter'):
                proc.subject_filter = config.get(section, 'subject_filter')
        elif proc.type == 'http':
            if not config.has_option(section, 'processor'):
                error('processor directive must be set in %s section' % section)
                sys.exit(2)

            if config.has_option(section, 'url'):
                proc.url = config.get(section, 'url')
            else:
                error('url directive must be set in %s section' % section)
                sys.exit(2)

        processors.append(proc)
    # end of configuration
    # action begins now
    for proc in processors:
        proc.init_source()
        if proc.error_seen:
            error_seen = True
            continue
        statement_files = proc.get_statements()
        if proc.error_seen:
            error_seen = True
            continue
        for statement_file in statement_files:
            proc.process_statement_file(statement_file)
            #if not proc.error_seen and proc.data_type == 'statement':
            #    proc.save_statement_file(statement_file)
            if not proc.error_seen:
                proc.upload_xml(statement_file)
            else:
                error_seen = True

        proc.finalize_source()

    if error_seen:
        sys.exit(1)


if __name__ == '__main__':
    main()
