ody_migr_config.py

#!/usr/bin/env python
"""Common config file shared accross all the modules."""


import os

# GENERAL VARIABLES

# common log formatter
LOG_FORMATTER = ('%(asctime)s %(name)-4s %(process)-6s '
                 '%(levelname)-6s %(message)s')

# log file name structure
LOG_FILE_NAME = "/home/{}/logs/ody_migr_{}_{}.log"

# server url based on DEVMODE and REQUIRE_ENCRYPTION  env variables
http_protocol = "https" if os.getenv("REQUIRE_ENCRYPTION") == "1" else "http"

if os.getenv("DEVMODE") == "1":
    server_host = "localhost:8000"
else:
    # this line should obtain appropriate domain url if DEVMODE=0
    server_host = os.getenv("TICKET_DOMAIN", "localhost:8000")
    # if TICKET_DOMAIN is still "", then somehow DEVMODE is set to "0"
    # (eg. from inside web-shell during debugging), then we set the server host
    # to be localhost:8000
    if server_host.strip() == "":
        # we should not
        server_host = "localhost:8000"

# System variables re-defined from app/web/monitor/library/cu_top.i
# Need to match values from cu_top.i script
SYS_TYPE_BATCH = 1
SYS_TYPE_LIVE = 2
SYS_TYPE_WEBONLY = 16
SYS_TYPE_CLOSED = 64

SERVER_URL = "{}://{}".format(http_protocol, server_host)

# root directory of the migration related scripts
SCRIPT_ROOT = "/project/app/tools/bin" if os.getenv(
    "DEVMODE") == "1" else "/opt/odyssey/tools/bin"


# keyword for secret key env variable
ENV_MIGR_SECRET_KEY = "ODY_MIGR_SECRET_KEY"

# REQUIRED FOR TESTING
TEST_USERNAME = ""
TEST_PASSWORD = ""
TESTING_SERVER = "www4"

# lists of supported CUs (argparse) -- only associated with `www4` and used
# for development testing purpose
# Even if we add other invalid CU's here for migration from www4 during
# development, Samuel's code re-validates existence of the CU in www4 and
# the migration will not proceed
SUPPORTED_DEV_CU = ["SCRUBCU", "CRUISECU"]

# supported end servers (argparse)
# "www" is used only with DATA_OPT_GETCULIST option
SERVER_CHOICES = ["www4", "www3", "www5", "www6"]

# all action keywords
ACTION_OPT_CLEAN = "clean"
ACTION_OPT_MIGRATE = "migrate"
ACTION_OPT_SUMMARY = "summary"

# action choices (argparse)
ACTION_CHOICES = [ACTION_OPT_CLEAN, ACTION_OPT_MIGRATE, ACTION_OPT_SUMMARY]

# all data category keywords
DATA_OPT_SETTINGS = "settings"
DATA_OPT_MEMBERS = "members"
DATA_OPT_MEMDATA = "memdata"
DATA_OPT_SWITCH_ACCT_MEMBERS = "switchaccountmembers"
DATA_OPT_SWITCH_ACCOUNTS = "switchaccounts"
DATA_OPT_MEMHIST = "memhist"
DATA_OPT_CREATE_HIST = "createhistoryfile"
DATA_OPT_GET_HIST = "gethistoryfile"
DATA_OPT_ADMIN = "admin"
DATA_OPT_LOANAPP = "loanapp"
DATA_OPT_GETCULIST = "getculist"  # get a list of CU's from www for validation
DATA_OPT_CLEANUP = "cleanup"  # memhist cleanup after the migration is complete

# wait for this many SECONDS after requesting for createhistoryfile and
# before requesting for gethistoryfile
HIST_CREATE_WAIT_TIME = 10
# number of members to import at a time for memdata
MEMBERS_PAGE_SIZE = 50
SWITCH_ACC_BATCH_SIZE = 400

# data categories for testing MAMMOTH endpoint
DATA_CHOICES_FOR_TEST = [
    DATA_OPT_SETTINGS,
    DATA_OPT_MEMBERS,
    DATA_OPT_MEMDATA,
    DATA_OPT_CREATE_HIST,
    DATA_OPT_GET_HIST,
    DATA_OPT_ADMIN
]

# list of CU's that you may want to use for testing
# do not use SOCU in the list - one test case will fail if included
SUPPORTED_CU_TEST = ["SCRUBCU", "CRUISECU"]

# data category choices (argparse)
DATA_CHOICES = [
    DATA_OPT_SETTINGS,
    DATA_OPT_MEMDATA,
    DATA_OPT_SWITCH_ACCOUNTS,
    DATA_OPT_MEMHIST,
    DATA_OPT_ADMIN,
    DATA_OPT_LOANAPP

]

# dictionary of pre-genererated expected authentication
# pass-phrases based on the chosen hashing algorithm (SHA1)
# generated based on CU, action (settings, members, etc) and secret key
EXPECTED_HASH_DICT = {
    DATA_OPT_SETTINGS: {
        "SCRUBCU": "0fd5f28efe6f1c8858ab251738eda5555a38bd9e",
        "CRUISECU": "ae74dc467ffacdaaafaa1a457d73704df8e6dbb2",
    },
    "members": {
        "SCRUBCU": "64ee2be7ed12c75fbe03f761685dad4386cfff97",
        "CRUISECU": "e603fbfca999c4b02bc831bd5ab49037b788736e",
    },
    DATA_OPT_MEMDATA: {
        "SCRUBCU": "72a82e4b1b29eb2e1ab9b7eed2094bcd9d4bc520",
        "CRUISECU": "63067f1c4f8d8c7abec7df6a3cad1ee920ef6630",
    },
    DATA_OPT_CREATE_HIST: {
        "SCRUBCU": "f0e7befce9d7cd29b05a85787438a8a553aca290",
        "CRUISECU": "12a7944ca58b49bd3af001145d4b6877326eceb9",
    },
    DATA_OPT_GET_HIST: {
        "SCRUBCU": "c2d6f276212bfcfc7c68fe080806648d2bbd3910",
        "CRUISECU": "af5eb1dd2862f9433d8d85dd21e45aee09b2043f",
        # to test history file does not exist as SOCU is not listed in
        # SUPPORTED_CU_TEST
        "SOCU": "d38f8bb69acaef2fe1c5a1055fe0abab723f313c"
    },
    DATA_OPT_ADMIN: {
        "SCRUBCU": "39ca9d4bc5a81fbf4dd1bf64b56c3f0e5b71d323",
        "CRUISECU": "666b7bcb47937454acf4eaebf537ae4eb26cebf6",
    }
}

# key parameter to `memdata` in HTTP GET parameter
KEY_MEMBERS = "members"
# Default error string in the MAMMOTH response
DEFAULT_ERROR_STR = ''
# Keys in the response dictionary from MAMMOTH
RESPONSE_KEYS = ["data", "action", "error"]
RESPONSE_HIST_KEYS = ["file", "action", "error"]
# Keys in the response dictionary for different
# data category type received from MAMMOTH
EXPECTED_DATA_KEYS = {
    DATA_OPT_SETTINGS: [
        "cuadmin",
        "custom-content",
        "menu",
        "branding"
    ],
    DATA_OPT_MEMBERS: [
        "count",
        "list"
    ],
    DATA_OPT_MEMDATA: {
        "cuusers",
        "cuquestselect",
        "accountbalance",
        "loanbalance",
        "holds",
        "cualertbal",
        "cualertcheck",
        "cualertloan",
        "cualerttrans",
        "cuadmeco",
        "culogtrack",
        "userlogins",
        "cusurveysays",
        "curepeattx",
        "cucmsresponse",
        "cuauditusers",
        "cuovermicr"
    },
    DATA_OPT_CREATE_HIST: [],
    DATA_OPT_GET_HIST: [],
    DATA_OPT_ADMIN: [
        'cualertmsgs',
        'cusurveydetail',
        'cusurveymaster',
        'extkey',
        'cuadmnotify',
        'cutrusteddetail',
        'adminusers',
        'cusurveyintro',
        'cucontact',
        'cuhavetrans',
        'cusurveyquest',
        'cualtroute'
    ]
}

# All database SQL operation keywords
CREATE = "create"
INSERT = "insert_all"
INSERT_ONE = "insert_one"
SELECT_RETURN = "select_and_return"
SELECT = "select_and_print"
SELECT_COUNT = "select_count"
DELETE = "delete"
TABLE_EXISTS = "table_exists"
SUMMARY = "tables_summary"

# MENU CONFIG

# default entry for menu item platform for cu_featuremenu table
DEF_MENU_ITEM_PLATFORM = ["D", "M", "A"]
# code to indicate menu feature that has no mapping
FEATURE_NOT_FOUND = "CD_NT_FND"
# expected empty feature code
EXPECTED_EMPTY_FEATURE_CODE = ""

# hrefs that do not have feature code
NO_FEATURE_CODE_HREF = [
    "hcuLogout",
    "hcuContactUs"
]

#  hcuConnect mode=`XXX` that do not map to menu link
NO_MENU_LINK_MODE = [
    "DREAMPOINTS",
    "HcuDMI",
    "DMI",
    "PSCU_ACCPT",
    "PSCU_INFO",
    "FICS",
    "FIS_EZCARD",
    "HcuEZCARD",
    "GOTOMYCARD",
    "FISERV_CPS"
]

# Menu feature code mapping dictionaries
# Note: comments are left as they are intentionally
# as these mappings were the result of continuous
# discussions and this might help with any surprise
# cases in future when dealing with data from other CUs
HCUCONNECT_MODE_CODE_MAP = {
    # "DREAMPOINTS": "",  # no menu link
    # "HcuDMI": "",  # no menu link
    # "DMI": "",  # no menu link
    "ENSENTA_BRDC": "RDC",
    "CHKFREE": "BILLPAY",
    "PAYVERIS": "BILLPAY",
    "IPAY_V3": "BILLPAY",
    "PSCUPAY_SSO": "BILLPAY",
    # "PSCU_ACCPT": "",   # no menu link
    # "PSCU_INFO": "",   # no menu link
    "MemberPay": "BILLPAY",
    # "FICS": "",  # no menu link
    # "FIS_EZCARD": "",  # no menu link
    # "HcuEZCARD": "",  # no menu link
    # "GOTOMYCARD": "", # no menu link
    "LASERTEC_ES": "ESTMT",
    "IDS_ES": "ESTMT",
    "LASERPRINT_ES": "ESTMT",
    "MWI_LOANPAY": "BILLPAY",
    "Bit": "ESTMT",
    "CERTEGY": "BILLPAY",
    "LKCS": "ESTMT",
    "MVANTE": "BILLPAY",
    "MidIPAY": "BILLPAY",  # no implementation
    "IPAY": "BILLPAY",
    "MDesk": "PFM",
    "MDesk3": "PFM",
    "VSOFT": "RDC",
    "Digital": "ESTMT",
    "BDI": "ESTMT",
    "DELUXE_OP": "BASIC",  # implies hcuOrderChecks
    "ThinkDigital": "ESTMT",
    "hcuLoanApp": "ONLINEAPP",
    # "FISERV_CPS": "",  # never implemented
    "MeridianLink": "ONLINEAPP",
    "WebShare": "ESTMT"
}

FEATURE_CODE_HREF_MAP = {
    "hcuAccounts": "BASIC",
    "hcuTxtBanking": "SMS",
    "hcuTransfer": "TRN",
    "hcuAlerts": "ALERT",
    # seen in www4 (doesn't have equivalent .prg file in Odyssey)
    "hcuEalerts": "ALERT",
    "hcuMobilePay": "MBILLPAY",
    "hcuStatement": "ESTMT",
    # seen in www4 (doesn't have equivalent .prg file in Odyssey)
    "hcuStatements": "ESTMT",
    "hcuSecureMail": "SECUREMSG",
    "hcuDownload": "TRNDOWN",
    "hcuTransferSchedule": "TRNSCHED",
    "hcuArchiveCheck": "BASIC",
    "hcuForms": "BASIC",
    "hcuProfileAlias": "BASIC",
    "hcuProfileEmail": "BASIC",
    "hcuProfilePwd": "BASIC",
    "hcuProfileSecurity": "BASIC",
    "hcuProfileDesc": "BASIC",
    "hcuOrderChecks": "BASIC",  # hcuConnect + mode=DELUXE_OP
    "hcuDisclosures": "BASIC",
    "hcuHistory": "BASIC"
}

# fa-icon mapping for S and H type menus - does not relate to D type menus
# Maps from menu_item["display"]["en_US"] to fa-icon name
FA_ICON_MAP = {
    "Sign Out": "power-off",
    "Logout": "power-off",
    "Transfers": "exchange",
    "Transfer": "exchange",
    "Account": "bank",
    "Accounts": "bank",
    "Commercial": "sitemap",
    "Communications": "sliders",
    "Disclosures": "file-text-o",
    "Profile": "cogs",
    "Pay Bills": "money",
    "PayBills": "money",
    "Mobile PayBills": "money",
    "Mobile Pay": "money",
    "Mobile Deposit": "anchor"
}

# BRANDING CONFIG

# main <cu> specific brand.<cu>.css file path (needs to append cu code)
BRANDING_TARGET_FILE = ("/home/{}/public_html/bankingIncludes/production"
                        "/css/brand.{}.css")

# .brand.<cu>.css backup file path (needs to append cu code)
BRANDING_BCK_FILE = os.path.join(
    os.path.dirname(BRANDING_TARGET_FILE), ".brand.{}.css")

# recently imported mammoth version of brand.css file (needs to append cu code)
BRANDING_MMTH_FILE = os.path.join(
    os.path.dirname(BRANDING_TARGET_FILE), "brand.css")

# ODY brand.css template file path
BRANDING_TMPL_FILE = ("/home/homecu/public_html/bankingIncludes"
                      "/css/brand.css")

BRANDING_DEFAULTS = {
    "fontFamily":
    "Helvetica, \"Helvetica Neue\", Arial, sans-serif",
    "headerBackground": "#24417A",
    "headerForeground": "#FFFFFF",
    "headerBorder": "#2A2D33",
    "footerBackground": "#E7E8EA",
    "footerForeground": "#777777",
    "menuGroupBackground": "#1B8DD6",
    "menuGroupBackgroundLighter": "#2198E3",
    "menuGroupForeground": "#FFFFFF",
    "menuForeground": "#337AB7",
    "menuForegroundHover": "#23527C",
    "menuBackgroundHover": "#EEEEEE",
    "menuBackground": "#F8F8F8",
    "slideinBackground": "#808080",
    "slideinForeground": "#FFFFFF",
    "contentBackground": "#FFFFFF",
    "contentForeground": "#333333",
    "logoHeightDesktop": "75px",
    "logoHeightMobile": "35px",
    "logoBackground": "transparent",
    "contentBorderWidth": "0px",
    "contentBorderStyle": "solid",
    "contentBorderColor": "#000000",
    "navBadgeColor": "#777777",
    "navIconColor": "#9D9D9D",
    "navIconColorHover": "#FFFFFF",
    "navBackground": "transparent",
    "navForeground": "#FFFFFF",
    "navActiveBorder": "#FF0000",
    "bannerBackground": "#161620",
    "bannerHeight": "60px"
}

# memberdata cu_scheduledtxn interval mapping
SCHEDULE_INTERVAL_MAPPING = {
    "OneTime": "OneTime",
    "EveryWeek": "Weekly",
    "Every2Weeks": "BiWeekly",
    "Every2Months": "Every2Months",
    "SemiMonthly": "FifteenthLast",
    "OnceMonthly": "Monthly",
    "Every3Months": "Quarterly",
    "Every6Months": "SemiAnnually",
    "Annually": "Annually"
}

# cuadminprogs mapping from Mammoth to Odyssey to be used in cuadminallow
# and cuadminexclude
CUADMINPROGS_PERMISSIONS_MAPPING = {
    "LockedRpt": "lockedRpt",
    "cu_notify": "notificationEmails",
    "Index": "secureForms",
    "ContMaint": "contMaint",
    "SetupMail": "setupMail",
    "SendMail": "sendMail",
    "cu_reports": "cuReports",
    "EmailSearch": "emailSearch",
    "SurveyMntc": "surveyMntc",
    "AdmECO": "secureMessages",
    "AddChange": "userSupport"
}

# Member account rights
MEMBER_ACC_RIGHTS = ["ACCESS", "ES", "RDC", "BP"]

# Default member account rights options
DEF_MEMACCT_RIGHTS_OPTIONS = ["D", "A"]

# Referred from cu_flagconst.i -> 'CU2_SPEC18' variable
# Special handling for loan type 18 Credit Cards
CU2_SPEC18 = 4

# User id cross reference csv file
USERID_CROSS_REF_CSV_FILE = "/home/{}/tmp/ody_migr_user_id_cross_reference.csv"

# File to record a list of orphaned users identified during migration
MAMMOTH_ORPHANED_MEMLIST_CSV_FILE =\
    "/home/{}/tmp/ody_migr_mammoth_orphaned_members.csv"

# File to record a list of members who never logged in in Mammoth
MAMMOTH_NEVER_LOGGEDIN_MEMLIST_CSV_FILE =\
    "/home/{}/tmp/ody_migr_mammoth_never_loggedin_members.csv"

# File to store mammoth to odyssey cusurvemaster.surveyid mappings
ADMIN_CUSURVEY_ID_MAP_CSV_FILE =\
    "/home/{}/tmp/ody_migr_admin_cusurveyid_map.csv"

# LOANAPP related csv files
# File to store mapping to mammoth lnappschemamaster.loanid
# to odyssey lnappschemamaster.loanid
LOANAPP_SCHEMAMASTER_LOANID_MAP_CSV_FILE =\
    "/home/{}/tmp/ody_migr_loanapp_loanid_map.csv"

# File to store mapping to mammoth lnappuser.userid
# to odyssey lnappuser.userid
LOANAPP_USER_USERID_MAP_CSV_FILE =\
    "/home/{}/tmp/ody_migr_loanapp_userid_map.csv"

# File to log curepeattx outliers during member data migration
SCHEDULED_TRANSFER_OUTLIERS_CSV_FILE =\
    "/home/{}/tmp/ody_migr_curepeattx_outliers.csv"

# downloaded copy of member history files
MEMHIST_DOWNLOADED_FILE = "/home/{}/tmp/memhist"
MEMHIST_ACCOUNT_HISTORY_FILENAME = "memhist_accounthistory"
MEMHIST_LOAN_HISTORY_FILENAME = "memhist_loanhistory"

KEY_CU_PREFIXED_TBLS = "cu_prefixed"
KEY_COMMON_TBLS = "common"

# tables that are REQUIRED to exist before running any migration operation
ODY_MUST_EXIST_TABLES = {
    DATA_OPT_SETTINGS: {
        KEY_CU_PREFIXED_TBLS: [],
        KEY_COMMON_TBLS: ["cuadmin", "cucmsfrags",
                          "cu_featuremenu", "lnappconfig"]
    },
    DATA_OPT_SWITCH_ACCOUNTS: {
        KEY_CU_PREFIXED_TBLS: [
            "{}memberacctrights",
            "{}useraccounts"
        ],
        KEY_COMMON_TBLS: []
    },
    DATA_OPT_MEMDATA: {
        KEY_CU_PREFIXED_TBLS: [
            "{}user",
            "{}group",
            "{}usercontact",
            "{}memberacct",
            "{}memberacctrights",
            "{}accountbalance",
            "{}loanbalance",
            "{}useraccounts",
            "{}holds",
            "{}audituser",
            "{}crossaccounts",
            "{}transdtl",
            "{}transhdr"
        ],
        KEY_COMMON_TBLS: [
            "cu_profile",
            "cusurveysays",
            "curepeattx",
            "cucmsresponse",
            "cu_alerts",
            "culogtrack",
            "cuadmeco",
            "cu_scheduledtxn",
            "cuovermicr"
        ]
    },
    DATA_OPT_MEMHIST: {
        KEY_CU_PREFIXED_TBLS: ["{}accounthistory", "{}loanhistory"],
        KEY_COMMON_TBLS: []
    },
    DATA_OPT_ADMIN: {
        KEY_CU_PREFIXED_TBLS: ["{}extkey"],
        KEY_COMMON_TBLS: [
            "cuauditadmin",
            "cuadminallow",
            "cuadminexclude",
            "cucontact",
            "cuadminusers",
            "cusurveyquest",
            "cusurveydetail",
            "cusurveymaster",
            "cualertmsgs",
            "cuadmnotify",
            "cusurveyintro",
            "cuhavetrans",
            "cutrusteddetail",
            "cualtroute"
        ]
    },
    DATA_OPT_LOANAPP: {
        KEY_CU_PREFIXED_TBLS: [],
        KEY_COMMON_TBLS: [
            "lnappschemamaster",
            "lnappschemadetail",
            "lnappuser",
            "lnappuser_questselect",
            "lnappuserresponse"
        ]
    }
}

# Used for drop down lists in admin.  auditaction => auditfulldesc
AUDITUSER_DESCMAP = {
    "ADD": "Add New Account",
    "DEL_A": "Delete Account",
    "DEL_H": "Delete History",
    "RESET": "Forgot Password",
    "UPD_F": "Update Account"
}