2 """Main module to migrate admin data. 4 Usage: export ODY_MIGR_SECRET_KEY=n3c3s1t@m0$Mud@rT0d@$L@$C0$@$! && \ 5 python3 app/tools/bin/cudrop_test.py 12 from datetime
import datetime, timedelta
16 from ody_migr_config
import (INSERT,
23 from ody_migr_transaction
import PgTransaction
24 from ody_migr_mmth_endpoint
import MammothMigration
25 from psycopg2.extras
import DictCursor
26 from ody_migr_utils
import format_error_permission
27 import ody_migr_db_handler
as pg_handler
29 LOGGER = logging.getLogger(__name__)
36 NUM_OF_NEW_RECORDS = 2
40 FIXTURE_READY_TIME = 5
42 TEST_DATE_NOW = datetime.now().strftime(
"%Y%m%d")
44 TEST_DATE_NOW_YEARMO = datetime.now().strftime(
"%Y%m")
47 PERL_SCRIPTS_ROOT =
"/opt/odyssey/tools/bin" 50 table_to_perlscript_map = {
51 "curdcstatus":
"cudropcurdcstatus.pl",
52 "cu_vendorlog":
"cudropcu_vendorlog.pl",
53 "cu_vendorcache":
"cudropcu_vendorcache.pl",
54 "lnappuserresponse":
"cudroplnapp.pl",
55 "culogtrack":
"cudropculogtrack.pl",
56 "history":
"cudrophist.pl",
57 "cuaudituser":
"cudropcuaudituser.pl",
58 "cuauditadmin":
"cudropcuauditadmin.pl",
59 "trans_hdr_dtl":
"cudropcutrans.pl",
60 "cuuser":
"cudropcuusers.pl",
74 "scrubcuaccounthistory",
79 def run_cmd_subprocess(_cmd):
80 """Utility to execute command: run _cmd as a sub-process and wait until 84 _cmd: command to run as a sub-process (either sequence of commands 85 or a whitespace separated string, in which case it is splited 86 using shell-like syntax 88 Catch IOError, OSError, handler error and raise SystemExit 91 if not isinstance(_cmd, list):
92 _cmd = shlex.split(_cmd)
93 subprocess.Popen(_cmd, shell=
False).wait()
94 except (IOError, OSError)
as e:
95 perm_err_msg = format_error_permission(e,
" ".join(_cmd))
96 LOGGER.error(perm_err_msg)
101 """Load content for tables""" 103 hdr_tablename =
"{}transhdr".format(cu)
104 dtl_tablename =
"{}transdtl".format(cu)
106 with pg_handler.PGSession()
as conn:
107 with conn.cursor()
as cur:
108 setup_transaction = PgTransaction(conn, cur)
110 misc_migration = MammothMigration(
116 endpoint_script=
"hcuadm/mOdysseyMigrMiscTables.prg" 119 misc_migration_response = misc_migration.run()
120 mmth_misc_data = misc_migration_response[
"data"]
121 for tbl, records
in mmth_misc_data.items():
122 if tbl
in [
"curdcstatus"]:
123 for record
in records:
124 record[
"user_id"] = 100
133 delete_hdr_1 = (datetime.now() -
134 timedelta(days=367)).strftime(
"%Y%m%d")
136 delete_hdr_2 = (datetime.now() -
137 timedelta(days=500)).strftime(
"%Y%m%d")
139 donot_delete_hdr_1 = (
141 timedelta(days=365)).strftime(
"%Y%m%d")
143 donot_delete_hdr_2 = datetime.now().strftime(
"%Y%m%d")
147 {
"id": 1,
"processed_date": delete_hdr_1},
148 {
"id": 2,
"processed_date": delete_hdr_2},
149 {
"id": 3,
"processed_date": donot_delete_hdr_1},
150 {
"id": 4,
"processed_date": donot_delete_hdr_2}
154 {
"id": 1,
"transhdr_id": 1},
155 {
"id": 2,
"transhdr_id": 2},
156 {
"id": 3,
"transhdr_id": 2},
157 {
"id": 4,
"transhdr_id": 3},
158 {
"id": 5,
"transhdr_id": 4}
165 collection=hdr_records
172 collection=dtl_records
175 setup_transaction.commit()
178 def fixture_cleanup():
179 """Delete fixture content from tables""" 181 hdr_tablename =
"{}transhdr".format(cu)
182 dtl_tablename =
"{}transdtl".format(cu)
183 with pg_handler.PGSession()
as conn:
184 with conn.cursor()
as cur:
185 cleanup_transaction = PgTransaction(conn, cur)
187 for tbl
in MISC_TABLES + HISTORY_TABLES:
204 cleanup_transaction.commit()
208 """Test suite for database cleanup scripts""" 216 """Validate expected counts for following tables: 224 with pg_handler.PGSession()
as conn:
225 with conn.cursor()
as cur:
228 curdcstatus_count = select_transaction(
231 return_only_count=
True 233 self.assertEqual(curdcstatus_count, LIMIT_SELECT)
235 cuvendorlog_count = select_transaction(
238 return_only_count=
True 240 self.assertEqual(cuvendorlog_count, LIMIT_SELECT)
242 cuvendorcache_count = select_transaction(
245 return_only_count=
True 247 self.assertEqual(cuvendorcache_count, min(LIMIT_SELECT, 23))
249 lnappuserresponse_count = select_transaction(
252 return_only_count=
True 254 self.assertEqual(lnappuserresponse_count,
255 min(LIMIT_SELECT, 44))
257 culogtrack_count = select_transaction(
260 return_only_count=
True 262 self.assertEqual(culogtrack_count, LIMIT_SELECT)
265 """Test cleanupscripts for history tables: 268 - scrubcuaccounthistory 273 with pg_handler.PGSession()
as conn:
274 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
277 for tbl_account, tbl_loan
in [(
"scrubcuaccounthistory",
278 "scrubculoanhistory")]:
281 initial_records_loan = hist_transaction(
286 initial_records_account = hist_transaction(
291 assert len(initial_records_account) >= NUM_OF_NEW_RECORDS
292 assert len(initial_records_loan) >= NUM_OF_NEW_RECORDS
295 print(
"\ninitial records [{}]\n".format(
"history"))
296 for rec
in initial_records_loan +\
297 initial_records_account:
304 latest_records_account = []
305 latest_records_loan = []
307 for i, record
in enumerate(initial_records_account[
308 :NUM_OF_NEW_RECORDS]):
309 latest_rec = record.copy()
310 latest_rec[
"date"] = TEST_DATE_NOW
312 latest_rec[
"accountnumber"] = int(
313 latest_rec[
"accountnumber"]) + SERIAL_OFFSET + i
315 latest_rec[
"tracenumber"] = int(
316 latest_rec[
"tracenumber"]) + SERIAL_OFFSET + i
318 latest_rec[
"certnumber"] += SERIAL_OFFSET + i
320 latest_records_account.append(latest_rec)
322 for record
in initial_records_loan[:NUM_OF_NEW_RECORDS]:
323 latest_rec = record.copy()
325 latest_rec[
"date"] = TEST_DATE_NOW
327 latest_rec[
"accountnumber"] = int(
328 latest_rec[
"accountnumber"]) + SERIAL_OFFSET + i
330 latest_rec[
"tracenumber"] = int(
331 latest_rec[
"tracenumber"]) + SERIAL_OFFSET + i
333 latest_records_loan.append(latest_rec)
336 print((
"\nnewly added records with " 337 "today's date [{}]\n").format(
"history"))
338 for rec
in latest_records_loan +\
339 latest_records_account:
345 collection=latest_records_account
351 collection=latest_records_loan
354 loan_records_after_insertion = hist_transaction(
359 account_records_after_insertion = hist_transaction(
364 assert len(loan_records_after_insertion)\
365 == len(initial_records_loan) + NUM_OF_NEW_RECORDS
367 assert len(account_records_after_insertion)\
368 == len(initial_records_account) + NUM_OF_NEW_RECORDS
372 run_cmd_subprocess(
"{}/{} SCRUBCU".format(
374 table_to_perlscript_map[
"history"]))
377 for tbl_account, tbl_loan
in [(
"scrubcuaccounthistory",
378 "scrubculoanhistory")]:
379 loan_records_after_cleanup = hist_transaction(
384 account_records_after_cleanup = hist_transaction(
390 loan_records_after_cleanup) == NUM_OF_NEW_RECORDS
392 account_records_after_cleanup) == NUM_OF_NEW_RECORDS
396 "\nrecords after cleanup [{}]\n".format(
"history"))
397 for rec
in loan_records_after_cleanup +\
398 account_records_after_cleanup:
402 """Test cleanupscripts for trans{hdr,dtl} tables: 411 hdr_tablename =
"{}transhdr".format(cu)
412 dtl_tablename =
"{}transdtl".format(cu)
413 with pg_handler.PGSession()
as conn:
414 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
417 initial_hdr_records_count = trans_transaction(
420 return_only_count=
True 422 self.assertEqual(initial_hdr_records_count, 4)
424 initial_dtl_records_count = trans_transaction(
427 return_only_count=
True 429 self.assertEqual(initial_dtl_records_count, 5)
432 run_cmd_subprocess(
"{}/{} SCRUBCU".format(
434 table_to_perlscript_map[
"trans_hdr_dtl"]))
436 hdr_records_count_after_cleanup = trans_transaction(
439 return_only_count=
True 441 self.assertEqual(hdr_records_count_after_cleanup, 2)
443 dtl_records_count_after_cleanup = trans_transaction(
446 return_only_count=
True 448 self.assertEqual(dtl_records_count_after_cleanup, 2)
451 """Validate expected counts for following tables: 460 - cudropcurdcstatus.pl 461 - cudropcu_vendorlog.pl 462 - cudropcu_vendorcache.pl 464 - cudropculogtrack.pl 466 with pg_handler.PGSession()
as conn:
467 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
469 for table_name
in MISC_TABLES:
474 initial_records = curdcstatus_transaction(
480 print(
"\ninitial records [{}]\n".format(table_name))
481 for rec
in initial_records:
482 if table_name ==
"lnappuserresponse":
483 print(rec[
"respsubmiton"], rec[
"respstarton"])
491 assert len(initial_records) >= NUM_OF_NEW_RECORDS
493 for i, record
in enumerate(initial_records[
494 :NUM_OF_NEW_RECORDS]):
495 latest_rec = record.copy()
497 if table_name ==
"curdcstatus":
498 latest_rec[
"lastupdate"] = TEST_DATE_NOW
499 latest_rec[
"depositid"] += SERIAL_OFFSET + i
501 elif table_name ==
"lnappuserresponse":
502 latest_rec[
"respid"] += SERIAL_OFFSET + i
503 latest_rec[
"respsubmiton"] = TEST_DATE_NOW
504 latest_rec[
"respstarton"] = TEST_DATE_NOW
506 elif table_name ==
"culogtrack":
507 latest_rec[
"accountnumber"] = int(
508 latest_rec[
"accountnumber"])\
511 latest_rec[
"yearmo"] = TEST_DATE_NOW_YEARMO
513 elif table_name
in [
"cu_vendorlog",
"cu_vendorcache"]:
514 latest_rec[
"entry"] = TEST_DATE_NOW
516 latest_records.append(latest_rec)
519 print((
"\nnewly added records with " 520 "today's date [{}]\n").format(table_name))
521 for rec
in latest_records:
522 if table_name ==
"lnappuserresponse":
523 print(rec[
"respsubmiton"], rec[
"respstarton"])
527 curdcstatus_transaction(
530 collection=latest_records
532 curdcstatus_transaction.commit()
534 records_after_insertion = curdcstatus_transaction(
539 assert len(records_after_insertion)\
540 == len(initial_records) + NUM_OF_NEW_RECORDS
543 run_cmd_subprocess(
"{}/{}".format(
545 table_to_perlscript_map[table_name]))
548 records_after_cleanup = curdcstatus_transaction(
553 assert len(records_after_cleanup) == NUM_OF_NEW_RECORDS
557 "\nrecords after cleanup [{}]\n".format(
559 for rec
in records_after_cleanup:
560 if table_name ==
"lnappuserresponse":
561 print(rec[
"respsubmiton"], rec[
"respstarton"])
566 """Test cuauditadmin cleanup script 569 - cudropcuauditadmin.pl 571 run_cmd_subprocess(
"{} {} {} clean SCRUBCU admin {} {}".format(
573 "/opt/odyssey/tools/bin/ody_migr_executor.py",
579 run_cmd_subprocess(
"{} {} {} migrate SCRUBCU admin {} {}".format(
581 "/opt/odyssey/tools/bin/ody_migr_executor.py",
587 time.sleep(FIXTURE_READY_TIME)
589 with pg_handler.PGSession()
as conn:
590 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
593 dict_cur.execute(
"SELECT * from cuauditadmin where " 594 "auditdate<'20170801' limit {}".format(
597 to_be_deleted_records = dict_cur.fetchall()
598 to_be_deleted_records = [dict(rec)
599 for rec
in to_be_deleted_records]
603 dict_cur.execute(
"delete from cuauditadmin")
604 cuauditadmin_transaction.commit()
608 for i, rec
in enumerate(to_be_deleted_records):
609 rec_recent = rec.copy()
610 rec_recent[
"auditdate"] = TEST_DATE_NOW
611 rec_recent[
"user_name"] = rec_recent[
"user_name"] + str(i)
612 recent_records.append(rec_recent)
614 cuauditadmin_transaction(
617 collection=to_be_deleted_records + recent_records
619 cuauditadmin_transaction.commit()
621 run_cmd_subprocess(
"{}/{}".format(
623 table_to_perlscript_map[
"cuauditadmin"]))
625 after_cleanup_count = cuauditadmin_transaction(
628 return_only_count=
True 631 self.assertEqual(after_cleanup_count, len(recent_records))
634 """Verify deletion of <cu>user (only) table 639 run_cmd_subprocess(
"{} {} {} clean SCRUBCU memdata {} {}".format(
641 "/opt/odyssey/tools/bin/ody_migr_executor.py",
646 run_cmd_subprocess(
"{} {} {} migrate SCRUBCU memdata {} {}".format(
648 "/opt/odyssey/tools/bin/ody_migr_executor.py",
654 time.sleep(FIXTURE_READY_TIME)
656 with pg_handler.PGSession()
as conn:
657 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
661 dict_cur.execute(
"SELECT * from scrubcuuser " 662 "limit {}".format(LIMIT_SELECT))
664 initial_records = dict_cur.fetchall()
665 initial_records = [dict(rec)
666 for rec
in initial_records]
669 if len(initial_records) != 0:
676 cuuser_transaction.commit()
680 record_not_to_delete = initial_records[0].copy()
681 record_not_to_delete[
"pwchange"] = TEST_DATE_NOW
682 record_not_to_delete[
"passwd"] =
's0m3^@l!dPassW0rD' 683 if "user_id" in record_not_to_delete:
684 del record_not_to_delete[
"user_id"]
685 record_not_to_delete[
"user_name"] += str(SERIAL_OFFSET)
686 new_records.append(record_not_to_delete)
689 for i, rec
in enumerate(initial_records):
690 updated_rec = rec.copy()
691 updated_rec[
"pwchange"] =
'20160801' 692 updated_rec[
"passwd"] =
'NULL PASSWORD' 694 if "user_id" in updated_rec:
695 del updated_rec[
"user_id"]
696 updated_rec[
"user_name"] += str(
697 SERIAL_OFFSET + i * 5 + 1)
698 new_records.append(updated_rec)
703 collection=new_records
705 cuuser_transaction.commit()
707 run_cmd_subprocess(
"{}/{} SCRUBCU".format(
709 table_to_perlscript_map[
"cuuser"]))
711 count_after_cleanup = cuuser_transaction(
714 return_only_count=
True 716 self.assertEqual(count_after_cleanup, 1)
719 """Test <cu>audituser cleanup script 722 - cudropcuaudituser.pl 724 run_cmd_subprocess(
"{} {} {} clean SCRUBCU memdata {} {}".format(
726 "/opt/odyssey/tools/bin/ody_migr_executor.py",
732 run_cmd_subprocess(
"{} {} {} migrate SCRUBCU memdata {} {}".format(
734 "/opt/odyssey/tools/bin/ody_migr_executor.py",
740 time.sleep(FIXTURE_READY_TIME)
742 with pg_handler.PGSession()
as conn:
743 with conn.cursor(cursor_factory=DictCursor)
as dict_cur:
746 dict_cur.execute(
"SELECT * from scrubcuaudituser " 747 "limit {}".format(LIMIT_SELECT))
749 initial_records = dict_cur.fetchall()
750 initial_records = [dict(rec)
751 for rec
in initial_records]
753 cuauditadmin_transaction(
757 cuauditadmin_transaction.commit()
761 for i, rec
in enumerate(initial_records):
762 rec_recent = rec.copy()
763 rec_recent[
"auditdate"] = TEST_DATE_NOW
764 rec_recent[
"accountnumber"] += chr(i * 2 + 97)
765 recent_records.append(rec_recent)
767 rec_recent_del = rec.copy()
768 rec_recent_del[
"auditdate"] =
'20140801' 769 rec_recent_del[
"accountnumber"] += chr(i * 2 + 1 + 97)
770 recent_records.append(rec_recent_del)
772 cuauditadmin_transaction(
775 collection=recent_records
777 cuauditadmin_transaction.commit()
779 run_cmd_subprocess(
"{}/{} SCRUBCU".format(
781 table_to_perlscript_map[
"cuaudituser"]))
783 after_cleanup_count = cuauditadmin_transaction(
786 return_only_count=
True 788 self.assertEqual(after_cleanup_count, len(recent_records) / 2)
791 """Cleanup fixture""" 795 if __name__ ==
"__main__":
def test_cuauditadmin(self)
def test_misc_counts(self)
def test_drop_cu_hist(self)
def test_trans_hdr_dtl(self)
def test_drop_cu_misc(self)