#!/usr/bin/env python

import sys, getopt, imaplib, tempfile, signal
import urllib2
import traceback
import os, os.path, time, datetime, popen2, select, ConfigParser, fcntl
import logging
from email.Parser import Parser
from dateutil.relativedelta import relativedelta
import zeep
import requests
from enum import Enum

verbose = False
config = None
configfile = "/usr/etc/fred/transproc.conf"
iso_date_format = '%Y-%m-%d'
logfile = None


def usage():
    print('%s [-c config] [-v]' % sys.argv[0])


def debug(msg):
    logging.debug(msg)
    # global verbose
    # if verbose:
    #     sys.stdout.write(msg + '\n')


def error(msg):
    logging.error(msg)
    # sys.stderr.write(msg + '\n')


class Processor(object):
    '''Class representing a processor of transcript.'''
    def __init__(self, name):
        self.name = name
        self.type = None
        self.data_type = None
        self.data_mimetype = None
        self.input_encoding = 'utf8'
        self.fromhead = None
        self.script = None
        self.error_seen = False

    def init_source(self):
        pass

    def finalize_source(self):
        pass

    def get_statements(self):
        pass

    def process_statement_file(self, statement_file):
        self.error_seen = False

        # debug('[%s]: processing message %s' % (self.name, msgid))
        # run the processor:
        #    stdin:  raw source data
        #    stdout: result of processing (XML)
        #    stderr: errors
        try:
            (status, outxml, err) = runCommand(self.script, statement_file['octets'])
        except Exception as e:
            error('[%s]: execution of processor failed: %s' % (self.name, e))
            error(traceback.format_exc())
            self.error_seen = True
            return
        if status == 2:
            # we will avoid processing nexttime ($not true right now$)
            error('[%s]: message %s rejected: %s' % (self.name, statement_file['msgid'], err))
        elif status != 0:
            if status != 1:
                error('[%s]: unknown return code (%s): %s' % (self.name, status, err))
            else:
                error('[%s]: processing of data failed: %s' % (self.name, err))
            # we will reprocess the email nexttime
            self.error_seen = True
        else:
            debug('[%s]: message processed by processor' % (self.name))
            statement_file['outxml'] = outxml

    def run_backend_command_on_output_xml(self, statement_file):
        outxml = statement_file.get('outxml')
        if outxml == '':
            debug('[%s]: Statement file have empty outxml (propably no payments in input source).' % self.name)
            return
        if outxml is None:
            error('[%s]: Statement file have no outxml, you must first procoss it!' % self.name)
            return

        save_statement = (self.data_type == 'statement' and statement_file.get('octets'))
        if save_statement:
            tmp_file, tmp_filename = tempfile.mkstemp(prefix='transproc_source_bankstatement')
            os.write(tmp_file, statement_file['octets'])
            os.close(tmp_file)

        try:
            cmd = config.get('general', 'backend_payments_cmd')
            if save_statement:
                try:
                    cmd = config.get('general', 'backend_statement_cmd')
                except ConfigParser.NoOptionError:
                    cmd = config.get('general', 'backend_payments_cmd')
                try:
                    cmd = cmd.format(STATEMENT_FILE=tmp_filename, STATEMENT_FILE_MIMETYPE=self.data_mimetype)
                except KeyError:
                    error('[%s] Error when composing backend command arguments (%s)' % (self.name, str(e)))
                    return
            if self.input_encoding != 'utf8':
                outxml = outxml.decode(self.input_encoding).encode('utf8')

            # open('/tmp/outxml-%s.xml' % (self.name + statement_file.get('msgid', '')) , 'w').write(outxml)
            # this needs higher read_iterations as it does a lot of ouput to stdout @UnusedVariable
            (status, output, err) = runCommand(cmd, outxml, read_iterations=config.getint('general', 'backendcmd_max_read_iterations'))
            if status != 0:
                error('[%s]: Error when executing command: %s\n%s' % (self.name, cmd, err))
                # restore the UNSEEN flag - something in script failed
                # we will reprocess the email nexttime
                self.error_seen = True
            else:
                # debug('[%s]: Backend\'s output:' % self.name)
                # debug(output)
                if err:
                    error('[%s]: Error from backend:\n' % self.name)
                    error(err)
                else:
                    debug('[%s]: Output XML sent to backend by processor' % (self.name))
                self.mark_as_processed(statement_file.get('msgid'))
        finally:
            if save_statement:
                os.remove(tmp_filename)

    def mark_as_processed(self, ident):
        ''' Mappes processed data source (e.g. particular email) (if possible),
            so that it will not be processed again.
        '''
        pass


class IMAPProcessor(Processor):
    def __init__(self, name):
        super(IMAPProcessor, self).__init__(name)
        self.type = 'imap'
        self.imaphost = None
        self.imapuser = None
        self.imappass = None
        self.imapfolder = None
        self.imapssl = None
        self.part = 2

        self.subject_filter = ''

        self.server = None

    def init_source(self):
        if self.imapssl:
            self.server = imaplib.IMAP4_SSL(self.imaphost)
        else:
            self.server = imaplib.IMAP4(self.imaphost)
        debug('[%s]: Connected to IMAP %s' % (self.name, self.imaphost))
        self.server.login(self.imapuser, self.imappass)
        debug('[%s]: IMAP logged in as %s' % (self.name, self.imapuser))
        if self.imapfolder:
            self.server.select(self.imapfolder)
        else:
            self.server.select()  # preserve default argument "mailbox" of select() method

    def finalize_source(self):
        self.server.close()
        self.server.logout()
        debug('[%s]: IMAP logged out' % self.name)

    def get_statements(self):
        # get unseen messages from a bank
        filters = ['(UNSEEN)', '(FROM %s)' % self.fromhead]
        if self.subject_filter:
            filters.append('(SUBJECT "%s")' % self.subject_filter)
        query = '(%s)' % ' '.join(filters)

        r, data = self.server.search(None, query)  # @UnusedVariable
        msgids = data[0].split(' ')
        if not msgids[0]:
            debug('[%s]: No new messages in mailbox' % self.name)
            return []
        # fetch new messages
        messages = {}
        statement_files = []
        for msgid in msgids:
            (r, data) = self.server.fetch(msgid, '(BODY.PEEK[])')  # ".PEEK" means that FLAG \Seen is not set! @UnusedVariable

            messages[msgid] = data[0][1]

        # process new messages by processor

        for msgid in messages:
            # separate attachment in which we are interested
            parser = Parser()
            mail = parser.parsestr(messages[msgid])

            if self.part > 1 and not mail.is_multipart():
                debug('[%s]: skipping message %s, it is not multipart' % (self.name, msgid))
                self.mark_as_processed(msgid)
                continue
            try:
                part = mail.get_payload(self.part - 1)  # parts are counted from 0
            except IndexError:
                debug('[%s]: skipping message %s, it has not %s parts as needed' % (self.name, msgid, self.part))
                self.mark_as_processed(msgid)
                continue

            statement_file = {}
            statement_file['msgid'] = msgid
            statement_file['filename'] = part.get_filename()
            statement_file['mimetype'] = self.data_mimetype
            statement_file['octets'] = part.get_payload(decode=True)
            statement_files.append(statement_file)
        debug('[%s]: %d new messages' % (self.name, len(statement_files)))
        return statement_files

    def mark_as_processed(self, ident):
        self.server.store(ident, '+FLAGS', '\\Seen')


class HTTPProcessor(Processor):
    def __init__(self, name):
        super(HTTPProcessor, self).__init__(name)
        self.type = 'http'
        self.input_file = None
        self.url = None

    def init_source(self):
        debug('[%s]: Opening HTTP source' % self.name)
        opener = urllib2.build_opener()
        try:
            page = opener.open(self.url)
            if page.code == 200:
                self.input_file = page.read()
                debug('[%s]: HTTP source successfully obtained' % self.name)
            else:
                self.error_seen = True
                error('[%s]: HTTP source returned code %s' % (self.name, page.code))
        except (IOError, urllib2.URLError) as e:
            self.error_seen = True
            error('[%s]: Failed to connect to HTTP source:\n%s' % (self.name, e))

    def get_statements(self):
        statement_file = {}
        statement_file['octets'] = self.input_file
        return [statement_file]


class CSOBCEBProcessor(Processor):

    class FileStatus(str, Enum):
        not_ready = 'R'
        ready = 'D'
        error = 'F'

    def __init__(self, name):
        super(CSOBCEBProcessor, self).__init__(name)
        self.type = 'csob_ceb'
        self.api_endpoint = 'https://ceb-bc.csob.cz/cebbc/api'
        self.interval = None
        self.wsdl_file = None
        self.cert_file = None
        self.key_file = None
        self.contract_number = None
        self.client = None
        self.client_history = None
        self.service = None

    def init_source(self):
        self.client_history = zeep.plugins.HistoryPlugin()
        session = requests.Session()
        session.cert = (self.cert_file, self.key_file)
        transport = zeep.transports.Transport(session=session)
        self.client = zeep.Client(self.wsdl_file, transport=transport, plugins=[self.client_history])
        self.service = self.client.create_service('{http://ceb-bc.csob.cz/CEBBCWS}CEBBCWSBinding', self.api_endpoint)

    def get_statements(self):
        statement_files = []

        from_date = None
        if self.interval:
            from_date = datetime.datetime.today() - datetime.timedelta(days=int(self.interval))
        files = self.service.GetDownloadFileList_v2(self.contract_number, from_date)
        files_details = files.FileList.FileDetail if files.FileList else []
        for file_info in files_details:
            if file_info.Type.lower() == 'vypis' and file_info.Filename.lower().endswith('.xml'):
                debug('[{}]: file to download: filename={f.Filename} crdate={f.CreationDateTime} size={f.Size} status={f.Status} url={f.Url}'
                    .format(self.name, f=file_info))
                if file_info.Status == CSOBCEBProcessor.FileStatus.ready:
                    rsp = requests.get(file_info.Url, cert=(self.cert_file, self.key_file))
                    if rsp.ok:
                        debug('[{}]: {}: download ok'.format(self.name, file_info.Filename))
                        statement_file = {}
                        statement_file['filename'] = file_info.Filename
                        statement_file['mimetype'] = self.data_mimetype
                        statement_file['octets'] = rsp.content
                        statement_files.append(statement_file)
                    else:
                        error('[{}] file ({f.Filename}) download error (http-status={})'.format(self.name, rsp.status_code, f=file_info))
                elif file_info.Status == CSOBCEBProcessor.FileStatus.error:
                    error('[{}] file preparation error on csob side'.format(self.name))

        debug('[{}]: {} new files'.format(self.name, len(statement_files)))
        return statement_files



def makeNonBlocking(fd):
    """
    Set non-blocking attribute on file.
    """
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)  # @UndefinedVariable


def runCommand(cmd, stdin, read_iterations=8):
    """
    Run command in non-blocking manner.
    read_iterations parametr is rather number of iteration in read stdout+err cycle.
    """
    # run the command
    child = popen2.Popen3(cmd, True)
    try:
        if (stdin):
            child.tochild.write(stdin)
        child.tochild.close()
    except IOError as e:
        error('Writing/closing stdin of child process coused IOERRor number %s' % e.errno)
        # try to read stdin & stderr
        debug('stdout of child:')
        debug(child.fromchild.read())
        debug('stderr of child:')
        debug(child.childerr.read())
        raise

    outfile = child.fromchild
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)
    makeNonBlocking(errfd)
    outdata = errdata = ''
    outeof = erreof = 0
    for round in range(int(read_iterations)):
        # wait for input at most 1 second
        ready = select.select([outfd, errfd], [], [], 1.0)
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '':
                outeof = 1
            else:
                outdata += outchunk
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '':
                erreof = 1
            else:
                errdata += errchunk
        if outeof and erreof:
            break
        time.sleep(0.3)  # give a little time for buffers to fill

    child.fromchild.close()
    child.childerr.close()

    status = os.waitpid(child.pid, os.WNOHANG)

    if status[0] == 0:
        time.sleep(1)
        error("Child doesn't want to exit, TERM signal sent.")
        os.kill(child.pid, signal.SIGTERM)
        time.sleep(1.2)  # time to exit
        status = os.waitpid(child.pid, os.WNOHANG)

        if status[0] == 0:
            error("Child doesn't want to die, KILL signal sent.")
            os.kill(child.pid, signal.SIGKILL)
            time.sleep(1.2)  # time to exit
            status = os.waitpid(child.pid, os.WNOHANG)

    stat = 9  # by default assume error
    if outeof and erreof and (status[0] == child.pid) and os.WIFEXITED(status[1]):
        stat = os.WEXITSTATUS(status[1])
    return stat, outdata, errdata


def configure_processor_common_parameters(config, section):
    if config.has_option(section, 'type'):
        if config.get(section, 'type') == 'imap':
            proc = IMAPProcessor(section)
        elif config.get(section, 'type') == 'http':
            proc = HTTPProcessor(section)
        elif config.get(section, 'type') == 'csob_ceb':
            proc = CSOBCEBProcessor(section)
        else:
            error('type directive must be set in %s section' % section)
            sys.exit(2)
    else:
        error('type directive must be set in %s section' % section)
        sys.exit(2)

    if config.has_option(section, 'data_type'):
        data_type = config.get(section, 'data_type')
        if data_type not in ('statement', 'payments'):
            error('data_type directive must "statement" or "payments" in section %s' % section)
            sys.exit(2)
        proc.data_type = data_type
    else:
        error('data_type directive must be set in %s section' % section)
        sys.exit(2)

    if proc.data_type == 'statement':
        proc.data_mimetype = config.get(section, 'data_mimetype')

    if not config.has_option(section, 'processor'):
        error('processor directive must be set in %s section' % section)
        sys.exit(2)
    proc.script = os.path.join(config.get('general', 'procdir'), config.get(section, 'processor'))
    if not os.path.isfile(proc.script):
        error('processor script "%s" does not exist' % proc.script)
        sys.exit(2)
    if config.has_option(section, 'input_encoding'):
        proc.input_encoding = config.get(section, 'input_encoding')

    return proc


def configure_processor_imap_parameters(processor, config, section):
    if processor.type == 'imap':
        if not config.has_option(section, 'imaphost'):
            error('imaphost directive must be set in %s section' % section)
            sys.exit(2)
        processor.imaphost = config.get(section, 'imaphost')
        if not config.has_option(section, 'imappass'):
            error('imappass directive must be set in %s section' % section)
            sys.exit(2)
        processor.imappass = config.get(section, 'imappass')
        if not config.has_option(section, 'imapuser'):
            error('imapuser directive must be set in %s section' % section)
            sys.exit(2)
        processor.imapuser = config.get(section, 'imapuser')
        if not config.has_option(section, 'imapfolder'):
            error('imapfolder directive must be set in %s section' % section)
            sys.exit(2)
        processor.imapfolder = config.get(section, 'imapfolder')
        if not config.has_option(section, 'imapssl'):
            error('imapssl directive must be set in %s section' % section)
            sys.exit(2)
        processor.imapssl = config.getboolean(section, 'imapssl')
        if not config.has_option(section, 'fromhead'):
            error('fromhead directive must be set in %s section' % section)
            sys.exit(2)
        processor.fromhead = config.get(section, 'fromhead')
        if config.has_option(section, 'part'):
            processor.part = config.getint(section, 'part')
        if config.has_option(section, 'subject_filter'):
            processor.subject_filter = config.get(section, 'subject_filter')
    else:
        error('wrong processor type')
        sys.exit(2)


def configure_processor_http_parameters(processor, config, section):
    if processor.type == 'http':
        today = 'TODAY'
        if config.has_option(section, 'end_date'):
            end_date = config.get(section, 'end_date')
        else:
            end_date = today
        if config.has_option(section, 'url'):
            processor.url = config.get(section, 'url')
            if end_date == today:
                end_date = datetime.datetime.today()
            else:
                try:
                    end_date = datetime.datetime.strptime(end_date, iso_date_format)
                except ValueError:
                    error('end_date directive must be date in ISO format %s in %s section' % iso_date_format, section)
                    sys.exit(2)
            start_date_placeholder = '{START_DATE}'
            if start_date_placeholder in processor.url:
                if not config.has_option(section, 'start_date_format'):
                    error('start_date_format directive must be set for url with placeholder in %s section' % section)
                    sys.exit(2)
                start_date_format = config.get(section, 'start_date_format')
                if not config.has_option(section, 'interval'):
                            error('interval directive must be set for url with placeholder in %s section' % section)
                            sys.exit(2)
                else:
                    try:
                        interval = int(config.get(section, 'interval'))
                    except ValueError:
                        error('interval directive must be integer in %s section' % section)
                        sys.exit(2)
                    if interval not in range(25):
                        error('interval directive must be from interval (0-25) in %s section' % section)
                        sys.exit(2)
                    interval = relativedelta(months=interval)
                start_date = end_date - interval
                start_date_string = start_date.strftime(start_date_format)
                if start_date_string == start_date_format:
                    error('value of start_date_format could not be used for setting of date in %s section' % section)
                    sys.exit(2)
                processor.url = processor.url.replace(start_date_placeholder, start_date_string)
            end_date_placeholder = '{END_DATE}'
            if end_date_placeholder in processor.url:
                if not config.has_option(section, 'end_date_format'):
                    error('end_date_format directive must be set for url with placeholder in %s section' % section)
                    sys.exit(2)
                end_date_format = config.get(section, 'end_date_format')
                end_date_string = end_date.strftime(end_date_format)
                if end_date_string == end_date_format:
                    error('value of end_date_format could not be used for setting of date in %s section' % section)
                    sys.exit(2)
                processor.url = processor.url.replace(end_date_placeholder, end_date_string)
        else:
            error('url directive must be set in %s section' % section)
            sys.exit(2)
    else:
        error('wrong processor type')
        sys.exit(2)


def configure_processor_csob_ceb_parameters(processor, config, section):

    class ConfigWrapper(object):

        def __init__(self, config, section_name, error_handler=None):
            self.config = config
            self.section_name = section_name
            self.error_handler = error_handler

        def get_mandatory(self, option_name):
            try:
                return self.config.get(self.section_name, option_name)
            except ConfigParser.NoOptionError:
                if self.error_handler:
                    self.error_handler(self.section_name, option_name)
                else:
                    raise

        def get_optional(self, option_name, default=None):
            try:
                return self.config.get(self.section_name, option_name)
            except ConfigParser.NoOptionError:
                return default

    def log_and_exit_on_error_handler(section_name, option_name):
        error('{} option must be set in {} section'.format(option_name, section_name))
        sys.exit(2)

    if processor.type == 'csob_ceb':
        config_wrapper = ConfigWrapper(config, section, error_handler=log_and_exit_on_error_handler)
        processor.api_endpoint = config_wrapper.get_optional('api_endpoint', default=processor.api_endpoint)
        processor.interval = config_wrapper.get_optional('interval', default=processor.interval)
        processor.wsdl_file = config_wrapper.get_mandatory('wsdl_file')
        processor.cert_file = config_wrapper.get_mandatory('cert_file')
        processor.key_file = config_wrapper.get_mandatory('key_file')
        processor.contract_number = int(config_wrapper.get_mandatory('contract_number'))
    else:
        error('wrong processor type')
        sys.exit(2)


def parse_config_section(config, section):
    proc = configure_processor_common_parameters(config, section)
    if proc.type == 'imap':
        configure_processor_imap_parameters(proc, config, section)
    elif proc.type == 'http':
        configure_processor_http_parameters(proc, config, section)
    elif proc.type == 'csob_ceb':
        configure_processor_csob_ceb_parameters(proc, config, section)
    return proc

def main():
    global verbose
    global config
    global configfile
    global iso_date_format

    error_seen = False
    # parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:v', [])  # @UnusedVariable
    except getopt.GetoptError:
        usage()
        sys.exit(2)
    # parameter values
    for o, a in opts:
        if o == '-c':
            configfile = a
        elif o == '-v':
            verbose = True
        else:
            usage()
            sys.exit(2)
    # read configuration file
    config = ConfigParser.ConfigParser(
        {'procdir': '/usr/libexec/transproc',
         'logfile': '/var/log/transproc.log',
         'backendcmd_max_read_iterations': '30'}
    )
    config.read(configfile)

    if not config.has_option('general', 'backend_payments_cmd'):
        error('Backend payments processing command in configuration file must be set')
        sys.exit(2)

    if not config.has_option('general', 'logfile'):
        error('logfile must be set')
        sys.exit(2)

    if not config.has_option('general', 'loglevel'):
        error('loglevel must be set')
        sys.exit(2)

    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    console_handler = logging.StreamHandler()
    console_handler.setFormatter(logging.Formatter('%(levelname)s %(message)s'))
    if verbose:
        console_handler.setLevel(logging.DEBUG)
    else:
        console_handler.setLevel(logging.ERROR)
    logger.addHandler(console_handler)

    loglevel = getattr(logging, config.get('general', 'loglevel').upper(), logging.DEBUG)
    file_handler = logging.FileHandler(config.get('general', 'logfile'), 'a')
    file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
    file_handler.setLevel(loglevel)
    logger.addHandler(file_handler)

    # iterate over other sections in config
    processors = []
    for section in config.sections():
        if section == 'general':
            continue
        proc = parse_config_section(config, section)
        processors.append(proc)
    # end of configuration
    # action begins now
    for proc in processors:
        proc.init_source()
        if proc.error_seen:
            error_seen = True
            continue
        statement_files = proc.get_statements()
        if proc.error_seen:
            error_seen = True
            continue
        for statement_file in statement_files:
            proc.process_statement_file(statement_file)
            # if not proc.error_seen and proc.data_type == 'statement':
            #    proc.save_statement_file(statement_file)
            if not proc.error_seen:
                proc.run_backend_command_on_output_xml(statement_file)
            else:
                error_seen = True

        proc.finalize_source()

    if error_seen:
        sys.exit(1)


if __name__ == '__main__':
    main()
