ody_migr_executor.py

#!/usr/bin/env python
"""Main module to execute Mammoth to Odyssey Data Migration.

usage: ody_migr_executor.py [-h] [--secret SECRET] [--reset_and_migrate]
                            [--verbose] [--disable_logging]
                            {www4,www3,www5,www6} {clean,migrate,summary} cu
                            {settings,memdata,memhist,admin} username password

Argument Parser for Mammoth to Odyssey Data Migration.

positional arguments:
  {www4,www3,www5,www6}
                        server code for Mammoth endpoint
  {clean,migrate,summary}
                        migration action
  cu                    target Credit Union <CUCODE> to migrate (upper case
                        required)
  {settings,memdata,memhist,admin,loanapp}
                        data category to migrate
  username              mammoth monitor username
  password              mammoth monitor password (base64 encoded)

optional arguments:
  -h, --help            show this help message and exit
  --secret SECRET, -s SECRET
                        pass secret key as optional argument instead
                        (alternative to environment variable)
  --reset_and_migrate, -reset
                        clean db tables and commit before migration)
  --verbose, -v         output verbosity
  --disable_logging, -dl
                        disable logging, logs only critical messages


Requirements:
    Python>=3.4
    Main packages:
        psycopg2>=2.7.4
        requests

Note (06/14/2018):
All the data migration scripts(app/tools/bin/ody_migr_*.py) have a shelf life.
Their usage is limited to one-time data migration of CU and its members from
Mammoth to Odyssey. We do not intend to maintain these scripts as-is after
migrations are complete.
"""


import sys
import os
import grp
import pwd
import argparse
import logging
import base64

from logging import FileHandler, StreamHandler, Formatter
from ody_migr_settings import migrate_settings
from ody_migr_members import migrate_members
from ody_migr_admin import migrate_admin
from ody_migr_loans import migrate_loanapps
from ody_migr_utils import file_exc_decorator
from ody_migr_mmth_endpoint import MammothMigration

import ody_migr_db_handler as pg_handler
from psycopg2.extras import RealDictCursor
from ody_migr_transaction import pg_crsr_hndlr_decrtr

from ody_migr_config import (ENV_MIGR_SECRET_KEY,
                             SERVER_CHOICES,
                             ACTION_CHOICES,
                             DATA_CHOICES,
                             DATA_OPT_SETTINGS,
                             DATA_OPT_MEMDATA,
                             DATA_OPT_SWITCH_ACCOUNTS,
                             DATA_OPT_MEMHIST,
                             DATA_OPT_ADMIN,
                             DATA_OPT_LOANAPP,
                             DATA_OPT_GETCULIST,
                             SUPPORTED_DEV_CU,
                             LOG_FILE_NAME,
                             SYS_TYPE_BATCH,
                             SYS_TYPE_CLOSED,
                             SYS_TYPE_LIVE,
                             SYS_TYPE_WEBONLY)

LOGGER = logging.getLogger(__name__)

# dictionary of main entrypoints for major data categories
MIGRATION_ENTRYPOINTS = {
    DATA_OPT_SETTINGS: migrate_settings,
    DATA_OPT_MEMDATA: migrate_members,
    DATA_OPT_SWITCH_ACCOUNTS: migrate_members,
    DATA_OPT_MEMHIST: migrate_members,
    DATA_OPT_ADMIN: migrate_admin,
    DATA_OPT_LOANAPP: migrate_loanapps
}


@file_exc_decorator
def get_ownership_detail(_directory):
    """get file/directory user/group ownership detail"""
    stat_info = os.stat(_directory)
    user = pwd.getpwuid(stat_info.st_uid)[0]
    group = grp.getgrgid(stat_info.st_gid)[0]
    return user, group


@file_exc_decorator
def owned_file_handler(_file_name, mode='w', owner=None):
    """Return a logging file handler with specified user/group ownership"""
    if owner:
        import pwd
        import grp
        # convert user and group names to uid and gid
        uid = pwd.getpwnam(owner[0]).pw_uid
        gid = grp.getgrnam(owner[1]).gr_gid
        owner = (uid, gid)
        if not os.path.exists(_file_name):
            open(_file_name, 'a').close()
        os.chown(_file_name, *owner)
    return FileHandler(_file_name, mode)


@file_exc_decorator
def setup_logging(_verbose,
                  _disable_logging,
                  formatter,
                  _cu,
                  _server,
                  _log_file):
    """Setup logging configuration and handlers."""
    log_file_dir = os.path.dirname(_log_file)

    # if log directory/file does not exist
    if not os.path.exists(log_file_dir):
        err_msg = ("Log directory `{}` does not exist. "
                   "MAKE SURE OF THE FOLLOWING: "
                   "(i) `/home/{}/*` directories are in place (perhaps, `{}` "
                   "is not created in Odyssey?). "
                   "(ii) CUCODE `{}` is a valid Mammoth target for migration "
                   "in `{}`."
                   "(iii) CUCODE `{}` is an intended Odyssey source "
                   "for migration.").format(
            log_file_dir,
            _cu.lower(),
            _cu.upper(),
            _cu.upper(),
            _server,
            _cu.upper())
        raise Exception(err_msg)

    log_dir_ownership = get_ownership_detail(log_file_dir)

    fh = owned_file_handler(_log_file, mode="w", owner=log_dir_ownership)
    fh.setFormatter(formatter)
    LOGGER.addHandler(fh)
    handlers = [fh]

    if(_disable_logging):
        logging.basicConfig(level=logging.CRITICAL, handlers=handlers)

    else:
        if _verbose:
            logging.basicConfig(level=logging.DEBUG, handlers=handlers)
        else:
            logging.basicConfig(level=logging.INFO, handlers=handlers)


def get_parser():
    """Argument parser for the main executor"""
    parser = argparse.ArgumentParser(
        description="Argument Parser for Mammoth to Odyssey Data Migration.")

    parser.add_argument(
        "server",
        type=str,
        choices=SERVER_CHOICES,
        help="server code for Mammoth endpoint"
    )

    parser.add_argument(
        "action",
        type=str,
        choices=ACTION_CHOICES,
        help="migration action"
    )

    parser.add_argument(
        "cu",
        type=str,
        help="target Credit Union <CUCODE> to migrate (upper case required)"
    )

    parser.add_argument(
        "data_category",
        type=str,
        choices=DATA_CHOICES,
        help="data category to migrate"
    )

    parser.add_argument(
        "username",
        type=str,
        help="mammoth monitor username"
    )

    parser.add_argument(
        "password",
        type=str,
        help="mammoth monitor password (base64 encoded)"
    )

    parser.add_argument(
        "--secret",
        "-s",
        type=str,
        help=("pass secret key as optional argument instead "
              "(alternative to environment variable)")
    )

    parser.add_argument(
        "--reset_and_migrate",
        "-reset",
        help="clean db tables and commit before migration)",
        action="store_true"
    )

    parser.add_argument(
        "--verbose",
        "-v",
        help="output verbosity",
        action="store_true"
    )

    parser.add_argument(
        "--disable_logging",
        "-dl",
        help="disable logging, logs only critical messages",
        action="store_true"
    )

    return parser


@pg_crsr_hndlr_decrtr
def get_valid_cu_list():
    """Return a list of valid CUs from cuinfo"""
    sql = ("select user_name, www_server, case "
           " when (coalesce(system_options, 0) & {}) <> 0 then 'Live'"
           " when (coalesce(system_options, 0) & {}) <> 0 then 'Batch'"
           " when (coalesce(system_options, 0) & {}) <> 0 then 'Web Only'"
           " else 'Other' end as cutype from cuinfo"
           " where (coalesce(system_options, 0) & {}) = 0"
           " order by user_name".format(SYS_TYPE_LIVE,
                                       SYS_TYPE_BATCH,
                                       SYS_TYPE_WEBONLY,
                                       SYS_TYPE_CLOSED))
    valid_cu_list = []
    with pg_handler.PGSession() as conn:
        with conn.cursor(cursor_factory=RealDictCursor) as cur:
            cur.execute(sql)
            valid_cu_list = cur.fetchall()
    return valid_cu_list


def run():
    """Main entrypoint for all the migration operations."""
    argv = sys.argv[1:]
    args = get_parser().parse_args(argv)

    # verify that necessary env variables are exported
    if os.getenv(ENV_MIGR_SECRET_KEY) is None:
        if args.secret is None:
            error_msg = ("SECRET KEY is required (either supply as --secret"
                         " argument or set `{}` env variable.)".format(
                             ENV_MIGR_SECRET_KEY))
            LOGGER.error(error_msg)
            raise SystemExit(error_msg)
        os.environ[ENV_MIGR_SECRET_KEY] = args.secret.strip()

    if os.getenv(ENV_MIGR_SECRET_KEY) is None:
        error_msg = "Environment variable: `{}` is not set!".format(
            ENV_MIGR_SECRET_KEY)
        LOGGER.error(error_msg)
        raise SystemExit(error_msg)

    requested_cu = args.cu.strip()
    # log formatter
    log_formatter = Formatter('%(asctime)s '
                              '%(levelname)-4s [{}-{}-{}-{}] '
                              '(%(name)s|%(funcName)s:%(lineno)d) :: '
                              '%(message)s'
                              .format(args.server.strip().upper(),
                                      args.data_category.strip().upper(),
                                      args.action.strip().upper(),
                                      requested_cu))

    # fill up LOG_FILE_NAME structure
    log_file = LOG_FILE_NAME.format(
        requested_cu.lower(),
        args.action.strip().lower(),
        args.data_category.strip().lower())

    # setup logging
    setup_logging(args.verbose,
                  args.disable_logging,
                  log_formatter,
                  requested_cu,
                  args.server.strip(),
                  log_file)

    # log file exists (does not necessarily mean that requested_cu has been
    # set up appropriatedly in Odyssey for migration)
    LOGGER.info("MIGRATION STARTED: [Data Category: {}, "
                "Operation: {}, CU: {}]".format(
                    args.data_category.strip().upper(),
                    args.action.strip().upper(),
                    requested_cu))

    # validate CU and servers for action=migrate only
    if args.action.strip() == "migrate":
        # Check if the requested_cu and servers are valid
        if requested_cu in SUPPORTED_DEV_CU and args.server.strip() == "www4":
            LOGGER.warning("Relaxing validation for DEV CUs. "
                           "Allowed DEV CUs are: {}".format(SUPPORTED_DEV_CU))
        else:
            allow_cu_migration = False
            valid_cu_info = {}

            all_cu_list_monitor = get_valid_cu_list()

            for cu_info in all_cu_list_monitor:
                # initially, we are only allowing "Live" CU's for migration
                if cu_info['cutype'].strip() in ["Live"] and cu_info[
                        'user_name'].strip().upper() == requested_cu and cu_info[
                        'www_server'].strip() == args.server.strip():
                    allow_cu_migration = True
                    valid_cu_info = cu_info
                    break
                else:
                    continue

            if(allow_cu_migration):
                LOGGER.info("CUCODE `{}` [{}] is valid for "
                            "server `{}` in production."
                            .format(
                                valid_cu_info['user_name'].strip(),
                                valid_cu_info['cutype'].strip(),
                                valid_cu_info['www_server'].strip())
                            )
            else:
                error_msg = ("CUCODE `{}` is not valid for "
                             "server `{}` in production."
                             .format(requested_cu, args.server.strip()))
                LOGGER.error(error_msg)
                raise SystemExit(error_msg)

    # when running the scripts individually in the web-container,
    # passwords are being displayed as raw text; this is just a
    # simple attempt to mask the exact strings
    args.password = base64.b64decode(args.password.strip()).decode()

    # prepare arguments for migration
    args_migration = [
        args.cu.strip(),
        args.server.strip(),
        args.action.strip(),
        args.username.strip(),
        args.password,
        args.verbose
    ]

    # adjust arguments for memdata and memhist
    if args.data_category in [DATA_OPT_MEMHIST, DATA_OPT_MEMDATA,
                              DATA_OPT_SWITCH_ACCOUNTS]:
        args_migration.append(args.data_category)
        args_migration.append(args.reset_and_migrate)

    # adjust arguments for admin
    elif args.data_category in [DATA_OPT_ADMIN, DATA_OPT_LOANAPP]:
        args_migration.append(args.reset_and_migrate)

    # start migration
    migration_exec_resp = MIGRATION_ENTRYPOINTS[args.data_category](*args_migration)

    LOGGER.info("MIGRATION DONE: [Data Category: {}, "
                "Operation: {}, CU: {}]".format(
                    args.data_category.strip().upper(),
                    args.action.strip().upper(),
                    args.cu.strip()))
    return migration_exec_resp


if __name__ == "__main__":
    sys.exit(run())