stack.py

"""
AWS stack provisioning.
"""

import collections
import boto3
import getpass
import humanize
import json
import os
import re
import sys
import shellish
import shlex
import time
import uuid
from pwgen import pwgen
from . import s3, base, migrations, cloudformation, cumanage
from . import REGISTRIES
from . import get_client_oauth_data


RDS_CONFIG_BUCKET = 'homecu.odyssey.rds'
RDS_CONFIG_PATH_FMT = '%(region)s/%(ecs_cluster)s/%(rds_config)s.json'


class StackCommand(base.HostingCommand):
    """ Manage entire odyssey stacks in AWS. """

    name = 'stack'

    def setup_args(self, parser):
        self.add_subcommand(ListCommand, default=True)
        self.add_subcommand(ShowCommand)
        self.add_subcommand(MonitorCommand)
        self.add_subcommand(SetEnvCommand)
        self.add_subcommand(GetEnvCommand)
        self.add_subcommand(SetStackStateCommand)
        self.add_subcommand(ScaleCommand)
        self.add_subcommand(StartMaintCommand)
        self.add_subcommand(EndMaintCommand)
        self.add_subcommand(MigrationCommand)
        self.add_subcommand(CreateCommand)
        self.add_subcommand(UpgradeCommand)
        self.add_subcommand(RemoveCommand)


class ListCommand(s3.S3BucketMixin, base.HostingCommand):
    """ List Odyssey stacks used in AWS. """

    name = 'ls'
    state_style = {
        'init': 'blue',
        'ready': 'green',
        'active': 'green',
        'partial_deploy': 'red',
        'corrupt': 'red',
        'deploying': 'b',
        'removing': 'b'
    }

    def __init__(self, *args, **kwargs):
        '''List Command Initialization'''
        super().__init__(*args, **kwargs)
        self.terse_headers = [
            'Stack',
            'Status',
            'Version',
            'URL(s)',
            'Region',
            'Last Deploy'
        ]
        self.verbose_headers = [
            'Stack',
            'Stack Version',
            'Status',
            'Version',
            'Creator',
            'URL(s)',
            'Region',
            'Created',
            'Last Deploy'
        ]

    def setup_args(self, parser):
        self.table_args_parser = self.add_table_arguments()
        self.add_argument('--verbose', '-v', action='store_true',
                          help='Enable verbose output')

    def run(self, args):
        headers = self.verbose_headers if args.verbose else self.terse_headers
        table_options = self.table_args_parser(args)
        with shellish.Table(headers=headers, **table_options) as t:
            # We get smarter table formatting if we get all the data first.
            output = []
            for summary in self.stacks.objects.all():
                stack = self._get_stack_data(s3.StackTracker(summary))
                output.append([y for (x, y) in stack.items() if x in headers])
            t.print(output)

    def _get_stack_data(self, stack):
        now = base.localnow()
        if stack.deploy.get('updated'):
            delta = now - base.parse_ts(stack.deploy.get('updated'))
            deployed = '%s ago' % humanize.naturaldelta(delta)
        else:
            deployed = ''
        style = self.state_style.get(stack.state, 'reverse')
        status = '<%s>%s</%s>' % (style, stack.state, style)
        if stack.env.get('MAINTENANCE_MODE') == '1':
            status = '%s (<red>MAINTENANCE</red>)' % status
        return collections.OrderedDict((
            ('Stack', stack.name),
            ('Stack Version', stack.stack_version),
            ('Status', status),
            ('Version', stack.version),
            ('Creator', stack.creator.split('@', 1)[0]),
            ('URL(s)', ' '.join('https://%s/' % x for x in stack.dns)),
            ('Region', stack.region),
            ('Created', humanize.naturaldate(base.parse_ts(stack.created))),
            ('Last Deploy', deployed)
        ))


class ShowCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Show full detail of a stack. """

    name = 'show'
    min_version = '0.8.0'
    use_pager = True
    allowed_states = {'init', 'ready', 'active', 'partial_deploy', 'corrupt'}

    def setup_args(self, parser):
        self.add_stack_argument()

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        self.check_requirements(stack)

        t = shellish.Table(headers=['Field', 'Value'],
                           title='Stack: %s' % args.stack)
        t.print((
            ('Name', stack.name),
            ('State', stack.state),
            ('DNS', ', '.join(stack.dns)),
            ('Stack Version', stack.stack_version),
            ('Creator', stack.creator),
            ('Region', stack.region),
            ('Cluster', stack.cluster),
            ('Created', base.formatdatetime(base.parse_ts(stack.created))),
            ('Deploy Count', stack.deploy.get('count', 0)),
        ))
        if hasattr(stack, 'db'):  # >= 0.7 stacks only
            t.print_row(('Database', '%s@%s:%d' % (stack.db['name'],
                         stack.db['host'], stack.db['port'])))
        t.print_footer('')
        t.close()

        if stack.deploy:
            dep = stack.deploy
            t = shellish.Table(headers=['Field', 'Value'],
                               title='Current Deployment')
            t.print((
                ('Updated', base.formatdatetime(base.parse_ts(
                                                dep['updated']))),
                ('Project', dep['project']),
                ('User/Machine', dep['build_ident']),
                ('GIT Rev', dep['git_rev']),
                ('GIT Branch', dep['git_branch']),
                ('GIT Repo', dep['git_repo']),
                ('GitHub', '%s/commit/%s' % (
                 dep['git_repo'].rstrip('.git'), dep['git_rev'][:10])),
                ('Docker Version', dep['docker_version']),
            ))
            t.print_footer('')
            t.close()

            cfn_stack = cloudformation.describe_stack(stack)
            services = [x['OutputValue'] for x in cfn_stack['Outputs']
                        if 'Service' in x['OutputKey']]

            ecs_api = boto3.client('ecs', region_name=stack.region)
            services = ecs_api.describe_services(
                cluster=stack.cluster,
                services=services
            )['services']
            t = shellish.Table(headers=[
                'Service',
                'Status',
                'Task Def',
                'Desired Count',
                'Pending Count',
                'Running Count',
                'Created',
                'Updated'
            ], title="Container Services")
            rows = []
            for service in services:
                for deploy in service['deployments']:
                    rows.append([
                        service['serviceName'].rsplit('-')[-2],
                        service['status'],
                        deploy['taskDefinition'].rsplit('/', 1)[-1],
                        deploy['desiredCount'],
                        deploy['pendingCount'],
                        deploy['runningCount'],
                        humanize.naturaldelta(base.localnow() -
                                              deploy['createdAt']),
                        humanize.naturaldelta(base.localnow() -
                                              deploy['updatedAt']),
                    ])
            t.print(rows)
            t.print_footer('')
            t.close()

        if stack.env:
            t = shellish.Table(headers=['Variable', 'Value'],
                               title='Environment')
            t.print(sorted(stack.env.items()))
            t.print_footer('')
            t.close()

        tbl_config = {
            "title": 'Change Log',
            "headers": ['Date', 'Change', 'Operator'],
            "accessors": [
                lambda x: base.formatdatetime(base.parse_ts(x['ts'])),
                'msg',
                'operator'
            ]
        }
        with shellish.Table(**tbl_config) as t:
            t.print(reversed(stack.get_changelog()))


class ScaleCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Set the number of application containers to run in the cluster. """

    name = 'scale'
    allowed_states = {'active', 'partial_deploy'}
    min_version = '0.8'

    def setup_args(self, parser):
        self.add_argument('container_count', type=int, help='Set the service '
                          '<i>desired</i> count to this value.')
        self.add_stack_argument()

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        self.check_requirements(stack)
        stack.check_passphrase()
        stack.set('app_service_count', args.container_count)
        stack_cf_params = create_cloudformation_stack_params(stack)

        cs_result = cloudformation.deploy_changeset(
            stack,
            os.environ.get('GIT_REV'),
            stack_cf_params,
        )

        if cs_result:
            stack._save('Changed container count of `app` to %d' %
                        args.container_count)
        else:
            stack._save('Scaling app container failed')
            stack.update_stack('corrupt')


class MaintCommand(s3.S3BucketMixin, base.HostingCommand):

    allowed_states = {'ready', 'active', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()

    def prerun(self, args):
        super().prerun(args)
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        self.check_requirements(stack)
        stack.check_passphrase()
        self.ecs = boto3.client('ecs', region_name=stack.region)
        self.stack = stack

    def task_env_to_dict(self, task_env):
        """ Convert task definition environment section to a dict. """
        return dict((x['name'], x['value']) for x in task_env)

    def dict_to_task_env(self, env):
        """ Convert task definition environment section to a dict. """
        return [{
            "name": key,
            "value": value
        } for key, value in env.items()]

    def get_task_definition(self, ident):
        """ Get cleaned up task definition suitable for
        register_task_definition. """
        family = 'odyssey-%s-%s' % (self.stack.name, ident)
        result = self.ecs.describe_task_definition(taskDefinition=family)
        fulldef = result['taskDefinition']
        whitelist = (
            "family",
            "networkMode",
            "containerDefinitions",
            "volumes",
            "placementConstraints"
        )
        return dict((key, value) for key, value in fulldef.items()
                    if key in whitelist)

    def restart_app_service(self):
        """ Update app service by pointing it to the latest task def. """
        common_name = 'odyssey-%s-app' % self.stack.name
        self.ecs.update_service(cluster=self.stack.cluster,
                                service=common_name,
                                taskDefinition=common_name)

    def adj_sched_service_count(self, count):
        sched_service = 'odyssey-%s-sched' % self.stack.name
        self.ecs.update_service(cluster=self.stack.cluster,
                                service=sched_service, desiredCount=count)

    def taskdef_env_override(self, taskdef, key, value=None):
        """ Patch the ENV for all containers in the app task def. """
        for cdef in taskdef['containerDefinitions']:
            env = self.task_env_to_dict(cdef['environment'])
            if value is None:
                try:
                    del env['MAINTENANCE_MODE']
                except KeyError:
                    pass
            else:
                env['MAINTENANCE_MODE'] = value
            cdef['environment'] = self.dict_to_task_env(env)


class StartMaintCommand(MaintCommand):
    """ Put the stack into maintenance mode.

    <i>Note, the full meaning and extent of maintenance mode is not covered
    here.</i>

    <b>Action Summary:</b>
        1. Disable public access to web container.
        2. Shutdown scheduler service.
    """

    name = 'start-maint'
    max_version = '0.7.1'

    def run(self, args):
        if self.stack.env.get('MAINTENANCE_MODE') == '1':
            raise SystemExit("Already in maintenance mode")
        # Prevent new deploys from reactivating the stack by saving the maint
        # mode in the stacks ENV config.
        self.stack.env['MAINTENANCE_MODE'] = '1'
        self.stack._save("Starting Maintenance")
        self.adj_sched_service_count(0)
        appdef = self.get_task_definition('app')
        self.taskdef_env_override(appdef, 'MAINTENANCE_MODE', '1')
        self.ecs.register_task_definition(**appdef)
        self.restart_app_service()


class EndMaintCommand(MaintCommand):
    """ Resume normal operation of stack currently in maintenance mode. """

    name = 'end-maint'
    max_version = '0.7.1'

    def setup_args(self, parser):
        self.add_stack_argument()

    def run(self, args):
        if self.stack.env.get('MAINTENANCE_MODE') != '1':
            raise SystemExit("Maintenance mode is not active")
        del self.stack.env['MAINTENANCE_MODE']
        self.stack._save("Ending Maintenance")
        self.adj_sched_service_count(1)
        appdef = self.get_task_definition('app')
        self.taskdef_env_override(appdef, 'MAINTENANCE_MODE')
        self.ecs.register_task_definition(**appdef)
        self.restart_app_service()


class MigrationCommand(MaintCommand):
    """Migrate stack versions

    Usage: hosting stack migrate {stack-name} --to {target-version}

    All parameters are required.

    Backwards migration isn't necessarily supported, however, it's not
    expressly prohibited either.  Such a migration would simply be a
    reversed expression.

    """

    name = 'migrate'
    min_version = '0.8.0'

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('--passphrase', env='ODYSSEY_STACK_PASSPHRASE',
                          help='To avoid being prompted for a passphrase on '
                          'protected stacks, you can set it on the command '
                          'line or via the env.')
        self.add_argument('stack_version', help='Target version of migration')
        self.add_argument('--verbose', '-v', action='store_true',
                          help='Show verbose output')

    def confirm_action(self, message):
        confirm = input(message)
        if confirm != 'yes':
            raise SystemExit('Aborted')

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        passphrase = stack.check_passphrase(args.passphrase)
        self.confirm_action('Are you sure you want to migrate stack'
                            ' from %s to %s'
                            ' [yes|NO]: ' % (
                                stack.stack_version, args.stack_version))
        migrations.migrate_stack(
            stack,
            stack.stack_version,
            args.stack_version,
            verbose=args.verbose,
            passphrase=passphrase,
        )


class MonitorCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Monitor a stack's event log. """

    name = 'monitor'
    min_version = '0.8.0'
    allowed_states = {'init', 'ready', 'active', 'corrupt', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        if not stack.deploy:
            raise SystemExit('No deployments for: %s' % args.stack)
        self.check_requirements(stack)
        ecs_api = boto3.client('ecs', region_name=stack.region)
        seen = set()
        cfn_stack = cloudformation.describe_stack(stack)
        service_keys = [x['OutputValue'] for x in cfn_stack['Outputs']
                        if 'Service' in x['OutputKey']]
        while True:
            new_events = []
            services = ecs_api.describe_services(
                cluster=stack.cluster,
                services=service_keys,
            )['services']
            for service in services:
                for ev in service['events']:
                    if ev['id'] in seen:
                        continue
                    else:
                        new_events.append((ev['createdAt'], ev['message']))
                        seen.add(ev['id'])
            new_events.sort()
            for x in new_events:
                dt = x[0].strftime('%I:%M %p %Z')
                shellish.vtmlprint("[<cyan>%s</cyan>] %s" % (dt, x[1]))
            time.sleep(1)


class SetEnvCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Set or clear an env variable in a stack configuration.

    Note that the env var won't take effect until the next deploy. """

    name = 'setenv'
    allowed_states = {'ready', 'active', 'corrupt', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('key', help='Env key. Eg. `MY_ENV_VAR`')
        self.add_argument('value', nargs='?', help='Env value to set. Omit to '
                          'clear an existing value.')

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        self.check_requirements(stack)
        if args.value is None:
            if args.key not in stack.env:
                raise SystemExit("Not Found: %s" % args.key)
            print("WARNING: Clearing env var:", args.key)
            del stack.env[args.key]
        else:
            if not args.value:
                print("WARNING: Setting empty env:", args.key)
            stack.env[args.key] = args.value
        stack.save('Set ENV variable: %s' % args.key)


class GetEnvCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Get the value of an env variable from a stack configuration. """

    name = 'getenv'
    allowed_states = {'ready', 'active', 'corrupt', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('key', nargs='?', help='Env key. Eg. `MY_ENV_VAR`')

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        self.check_requirements(stack)
        if args.key and args.key not in stack.env:
            raise SystemExit("Not Found: %s" % args.key)
        keys = [args.key] if args.key else stack.env.keys()
        for key in keys:
            print('%s=%s' % (key, shlex.quote(stack.env[key])))


class SetStackStateCommand(s3.S3BucketMixin, base.HostingCommand):
    '''Override stack status'''

    name = 'set-state'
    allowed_states = {'init', 'ready', 'deploying', 'partial_deploy', 'corrupt'}

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('state', help='State to set the stack to')
        self.add_argument('--passphrase', env='ODYSSEY_STACK_PASSPHRASE',
                          help='To avoid being prompted for a passphrase on '
                          'protected stacks, you can set it on the command '
                          'line or via the env.')

    def run(self, args):
        stack = self.get_stack(args.stack)
        new_state = args.state
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        if new_state not in ['active', 'test']:
            raise SystemExit('Not a valid new state')
        stack.check_passphrase(args.passphrase)
        self.check_requirements(stack)

        old_state = stack.state

        if new_state == old_state:
            raise SystemExit("Stack already at desired status")

        stack.update_state(new_state)
        stack._save('Updated state from %s to %s by %s' % (
            old_state,
            new_state,
            os.environ['OPERATOR_IDENT']))


class CreateCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Create a new Odyssey stack in AWS <b>[use caution]</b>. """

    name = 'create'
    dns_compat = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('--verbose', '-v',
                          action='store_true', default=False)
        self.add_argument('--region', default='us-east-2',
                          help='Region to host the stack.')
        self.add_argument('--elb-security-group',
                          default='homecu-only-homer-elb-sg',
                          help='ELB security group name.')
        self.add_argument('--eni-security-group', default='homer-ecs-sg',
                          help='ENI security group name.')
        self.add_argument('--elb-int-security-group',
                          default='homer-int-elb-sg',
                          help='Internal ELB security group name.  '
                          'This  security group is used for internal '
                          'services of the stack, e.g., scheduler.')
        self.add_argument('--pub-subnet-prefix', default='homer-public',
                          help='Naming prefix scheme for public subnets.')
        self.add_argument('--priv-subnet-prefix', default='homer-private',
                          help='Naming prefix scheme for private subnets.')
        self.add_argument('--domain', default='homecu.io', help='Default '
                          'domain used for cert selection and DNS.')
        self.add_argument('--hostnames', nargs='+', help='Override the '
                          'hostname(s) registered for this stack.  Use "" for '
                          'a domain-only DNS entry. By default the stack name '
                          'is used.')
        self.add_argument('--https-cert-hostname', default='*',
                          help='ACM certificate hostname for HTTPS.  Note, '
                          'value is combined with `--domain`.')
        self.add_argument('--ecs-cluster', default='homer',
                          help='ECS cluster to provision services on.')
        self.add_argument('--rds-config', default='default',
                          help='Name of config file in S3 used for RDS '
                          'configuration. The S3 URL is formed from the '
                          'triplet (region, ecs-cluster, rds-config).  E.g. '
                          '`s3://homecu.odyssey.rds/us-west-2/default/'
                          'default.json`.')
        self.add_argument('--env', nargs='+', metavar="KEY=VALUE",
                          help='Supplemental ECS task-definition environment '
                          'key=value pair(s) for this deploy.  Note this is '
                          'applied to all task definitions for a stack.')
        self.add_argument('--odyssey-version', default='latest',
                          help='Odyssey Tag or version to deploy')
        self.add_argument('--environment', default='dev',
                          help=('Environment tag for stack, e.g., dev or prod'
                                '*Notice* This is not to be confused with the'
                                '--env flag'))
        self.add_argument('--environment_name', default='homer',
                          help='Environment Name for the stack, e.g., homer')

    def check_name(self, name):
        """ Raise exception if stack name is not valid. """
        if len(name) > 12:
            raise SystemExit('Name too long')
        if not self.dns_compat.match(name):
            raise SystemExit('Name is not DNS compatible')

    def create_passphrase(self):
        while True:
            first = getpass.getpass('Enter optional lock passphrase: ')
            second = getpass.getpass('Re-enter optional lock passphrase: ')
            if first != second:
                shellish.vtmlprint("<b><red>Passphrases do not match")
                continue
            if not first:
                shellish.vtmlprint("<b>WARNING: No passphrase will be "
                                   "required for this stack.")
            return first or None

    def run(self, args):
        self.check_name(args.stack)
        if self.get_stack(args.stack) is not None:
            raise SystemExit('Stack already exists: %s' % args.stack)
        if args.hostnames is None:
            args.hostnames = [args.stack]
        stack = s3.StackTracker(self.stacks.Object(args.stack))
        passphrase = self.create_passphrase()
        phash = passphrase and stack.gen_passphrase_hash(passphrase)

        stack.set('_passphrase_hash', phash or '', meta=True)
        stack.set("stack_version", base.VERSION)
        stack.set("uuid", str(uuid.uuid4()))
        stack.set("state", 'init')
        stack.set("creator", os.environ['OPERATOR_IDENT'])
        stack.set("region", args.region)
        stack.set("cluster", args.ecs_cluster)
        stack.set("created", base.localnow().isoformat())
        stack.set("name", args.stack)
        stack.set("dns", ['%s.%s' % (x, args.domain) for x in args.hostnames])
        env = dict(x.split('=') for x in args.env) if args.env else {}
        stack.set("env", env)
        stack.set("app_service_count", 1)
        stack.set("deploy", {})
        stack.set("changelog", [])
        stack.set("aws", {})
        stack.set("version", args.odyssey_version)
        stack.set('environment', args.environment)
        stack.set('environment_name', args.environment_name)
        stack._save('Created Stack')

        create_stack_aws_resources(
            stack,
            region=args.region,
            ecs_cluster=args.ecs_cluster,
            elb_security_group=args.elb_security_group,
            elb_internal_security_group=args.elb_int_security_group,
            eni_security_group=args.eni_security_group,
            rds_config=args.rds_config,
            pub_subnet_prefix=args.pub_subnet_prefix,
            priv_subnet_prefix=args.priv_subnet_prefix,
            https_cert_hostname=args.https_cert_hostname,
            domain=args.domain,
            hostnames=args.hostnames,
            verbose=args.verbose,
        )


class UpgradeCommand(s3.S3BucketMixin, base.HostingCommand):
    """Set the stack version and deploy CFN template"""

    name = 'upgrade'
    min_version = '0.8.3'
    allowed_states = {'ready', 'active', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('--verbose', '-v', action='store_true')
        self.add_argument('--passphrase', env='ODYSSEY_STACK_PASSPHRASE',
                          help='To avoid being prompted for a passphrase on '
                          'protected stack, you can set it on the command '
                          'line or via the env.')
        self.add_argument('new_version', action='store',
                          help='Version or tag to deploy')
        self.add_argument('--run-db-migrations', action='store_true',
                          help='Run database migrations after upgrade. '
                          'Notice, this implies `--check-exit-status`.')
        self.add_argument('--check-exit-status', action='store_true',
                          help='Wait and report exit status of update')

    def run(self, args):
        stack = self.get_stack(args.stack)

        if stack is None:
            raise SystemExit("Stack not found: %s" % args.stack)
        passphrase = stack.check_passphrase(args.passphrase)
        self.check_requirements(stack)
        stack.update_state('deploying')
        stack.set('version', args.new_version)
        git_sha = os.environ['GIT_REV']
        upgrade_cfn_stack(
            stack,
            git_sha,
            passphrase,
            verbose=args.verbose,
            wait_until_complete=args.check_exit_status,
            run_db_migrations=args.run_db_migrations,
        )


class RemoveCommand(s3.S3BucketMixin, base.HostingCommand):
    """ Remove an Odyssey stack from AWS <b>[use caution]</b>. """

    name = 'rm'
    min_version = "0.8.0"
    allowed_states = {'init', 'ready', 'active', 'partial_deploy'}

    def setup_args(self, parser):
        self.add_stack_argument()
        self.add_argument('--force', '-f', action='store_true',
                          help='Force removal without confirmation.')
        self.add_argument('--best-effort', action='store_true',
                          help='Attempt to remove corrupt stacks.')
        self.add_argument('--drop-database', action='store_true',
                          default=False,
                          help='Drops the database and user. Use CAUTION, '
                          'This operation cannot be undone!')
        self.add_argument('--check-exit-status', action='store_true',
                          help='Wait for completion of remove action')

    def confirm_action(self, message):
        confirm = input(message)
        if confirm != 'yes':
            raise SystemExit('Aborted')

    def run(self, args):
        stack = self.get_stack(args.stack)
        if stack is None:
            raise SystemExit('Stack not found: %s' % args.stack)
        passphrase = stack.check_passphrase()
        if args.best_effort:
            print("WARNING: Skipping requirements check!")
        else:
            self.check_requirements(stack)
        if not args.force:
            self.confirm_action("Confirm removal of `%s` stack: [yes|NO] " %
                                args.stack)
        if args.drop_database:
            self.confirm_action("Dropping database as part of removal. "
                                "Are you sure? [yes|NO] ")

        wait_until_complete = args.check_exit_status

        stack.update_state('removing')
        try:
            cloudformation.delete_stack(
                stack,
                wait_until_complete=wait_until_complete)
            if args.drop_database and stack.db:
                print('Deleting database...')
                self.remove_db(stack.db.get('name', None),
                               stack.region,
                               stack.cluster,
                               stack.db.get('rds_config', None))
        except Exception as e:
            if args.best_effort:
                shellish.vtmlprint("<red><b>IGNORING ERROR: %s" % e)
            else:
                shellish.vtmlprint("<red><b>CRITICAL ERROR: %s" % e)
                stack.update_state('corrupt')
                raise e from None
        else:
            stack.delete(passphrase)
            print('Done.')

    def remove_db(self, db_ident, region, cluster, rds_config=None):
        if not db_ident:
            raise ValueError('No database identity was provided, aborting!')
        if not rds_config:
            rds_config = 'default'
        db_config = get_rds_config(region, cluster, rds_config)
        sql = (
            'DROP DATABASE IF EXISTS %s;' % db_ident,
            'DROP USER IF EXISTS %s;' % db_ident,
        )
        submit_rsql_job(sql, db_config, region, cluster)


def create_stack_aws_resources(stack,
                               region,
                               ecs_cluster,
                               elb_security_group,
                               elb_internal_security_group,
                               eni_security_group,
                               rds_config,
                               pub_subnet_prefix,
                               priv_subnet_prefix,
                               https_cert_hostname,
                               domain,
                               hostnames,
                               **kwargs):
    '''Create required AWS resources for given stack'''
    verbose = kwargs.get('verbose', False)
    db_config = get_rds_config(region,
                               ecs_cluster,
                               rds_config)

    def get_vpc_subnets(vpc, prefix):
        def name_tag(tags):
            return [x['Value'] for x in tags
                    if x['Key'] == 'Name'][0]
        return [x for x in vpc.subnets.all()
                if name_tag(x.tags).startswith(prefix)]

    def get_sg(name, region):
        ec2_api = boto3.client('ec2', region_name=region)
        return ec2_api.describe_security_groups(Filters=[{
            'Name': 'group-name',
            'Values': [name]
        }])['SecurityGroups'][0]

    def get_https_cert(name, region):
        acm_api = boto3.client('acm', region_name=region)
        certs = acm_api.list_certificates()['CertificateSummaryList']
        for x in certs:
            if x['DomainName'] == name:
                return x

    def get_hosted_zone_id(domain):
        r53_api = boto3.client('route53')
        response = r53_api.list_hosted_zones_by_name(
            DNSName=domain,
            MaxItems='1'
        )
        if len(response.get('HostedZones', [])) == 0:
            raise SystemExit("Unable to find hosted zone by name")
        return response['HostedZones'][0]['Id']

    elb_sg = get_sg(elb_security_group, region)
    elb_int_sg = get_sg(elb_internal_security_group, region)
    eni_sg = get_sg(eni_security_group, region)
    ec2_res = boto3.resource('ec2', region_name=region)
    vpc = ec2_res.Vpc(elb_sg['VpcId'])
    elb_subnets = get_vpc_subnets(vpc, pub_subnet_prefix)
    if not elb_subnets:
        raise SystemExit("No usable subnets found")
    eni_subnets = get_vpc_subnets(vpc, priv_subnet_prefix)
    if not eni_subnets:
        raise SystemExit("No usable private subnets found")

    cert_name = '%s.%s' % (https_cert_hostname, domain)
    cert = get_https_cert(cert_name, region)

    hosted_zone_id = get_hosted_zone_id(domain)

    if cert is None:
        raise SystemExit("No HTTPS cert found for: %s" % cert_name)

    stack.set("aws", {
        'vpc_id': vpc.vpc_id,
        'certificate_arn': cert['CertificateArn'],
        'elb_security_groups': [elb_sg['GroupId']],
        'elb_internal_security_groups': [elb_int_sg['GroupId']],
        'eni_security_groups': [eni_sg['GroupId']],
        'private_subnet_ids': [x.id for x in eni_subnets],
        'subnet_ids': [x.id for x in elb_subnets],
        'hosted_zone_id': hosted_zone_id,
    })

    if kwargs.get('create_database', True):
        db_creds = create_db(stack, db_config)
        stack.set("db", {
            "host": db_config['host'],
            "port": db_config['port'],
            "name": db_creds['db'],
            "user": db_creds['user'],
            "password": db_creds['password'],
            "rds_config": rds_config,
        })

    stack_cf_params = create_cloudformation_stack_params(stack, **kwargs)

    stack_cf_id = cloudformation.create_cloudformation_stack(
        stack,
        stack_cf_params,
        verbose=verbose
    )
    if stack_cf_id is None:
        raise SystemExit("Unable to create Cloudformation Stack for Stack")

    print("Created Cloudformation Stack: %s" % stack_cf_id)

    stack.deploy = _deploy_dictionary_(stack)

    stack.update_state('active')


def upgrade_cfn_stack(stack, git_sha, passphrase, **kwargs):
    def deploy(stack, git_sha, **kwargs):
        '''perform stack deployment'''
        stack_cf_params = create_cloudformation_stack_params(stack, **kwargs)

        return cloudformation.deploy_changeset(
            stack,
            git_sha,
            stack_cf_params,
            **kwargs)
    try:
        run_db_migrations = kwargs.get('run_db_migrations', False)
        wait_until_complete = (kwargs.get('wait_until_complete', False) or
                               run_db_migrations)
        verbose = kwargs.get('verbose', False)
        kws = {k: v for k, v in kwargs.items()
               if k not in ['verbose',
                            'wait_until_complete']}
        deploy_status = deploy(
            stack,
            git_sha,
            verbose=verbose,
            wait_until_complete=wait_until_complete,
            **kws,
        )
        if deploy_status:
            stack.deploy = _deploy_dictionary_(stack)
            stack.update_state('active')
            stack._save('Deployed %s from %s' % (
                git_sha, os.environ['PROJECT_NAME']))
            if run_db_migrations:
                if verbose:
                    print('Submitting database migrations...')
                cumanage.run_cumanage_task(stack,
                                           ['db', 'migrate'],
                                           wait_until_complete=True,
                                           passphrase=passphrase)
        else:
            stack.update_state('partial_deploy')
            stack._save('Deploy failure of %s from %s' % (
                git_sha, os.environ['PROJECT_NAME']))
    except Exception as ex:
        if verbose:
            print(ex, file=sys.stderr)
        stack.update_state('partial_deploy')
        stack._save('Deploy failure of %s from %s' % (
            git_sha, os.environ['PROJECT_NAME']))
        raise SystemExit("Failed to deploy stack")


def get_rds_config(region, cluster, rds_config):
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(RDS_CONFIG_BUCKET)
    config_obj = bucket.Object(
        RDS_CONFIG_PATH_FMT % {'region': region,
                               'ecs_cluster': cluster,
                               'rds_config': rds_config})
    data = config_obj.get()
    return json.loads(data['Body'].read().decode())


def create_db(stack, db_config):
    stack_name = re.sub('[^a-z0-9]', '', stack.name.lower())
    user_uuid = str(stack.uuid).replace('-', '_')
    ident = 'odyssey_%s_%s' % (stack_name, user_uuid)
    password = pwgen(50)
    sql = (
        "CREATE USER %s WITH PASSWORD '%s';" % (ident, password),
        "CREATE DATABASE %s;" % ident,
        "REVOKE CONNECT ON DATABASE %s FROM PUBLIC;" % ident,
        "GRANT ALL ON DATABASE %s TO %s;" % (ident, ident)
    )
    print('Creating RDS USER:', ident)
    print('Creating RDS DB:', ident)
    resp = submit_rsql_job(sql, db_config, stack.region, stack.cluster)

    assert not resp['failures'], resp['failures']
    return {
        "db": ident,
        "user": ident,
        "password": password
    }


def submit_rsql_job(sql, db_config, region, cluster, operator=None):
    if not operator:
        operator = os.environ['OPERATOR_IDENT']
    client = boto3.client('ecs', region_name=region)
    resp = client.run_task(**{
        "cluster": cluster,
        "taskDefinition": 'rds-psql',
        "startedBy": operator,
        "overrides": {
            "containerOverrides": [{
                "environment": [
                    {
                        "name": "PGHOST",
                        "value": db_config['host']
                    },
                    {
                        "name": "PGPORT",
                        "value": str(db_config['port'])
                    },
                    {
                        "name": "PGPASSWORD",
                        "value": db_config['password']
                    },
                    {
                        "name": "PGUSER",
                        "value": db_config['user']
                    },
                ],
                "name": 'psql',
                "command": [' && '.join('psql -c "%s"' % x for x in sql)]
            }]
        }
    })
    assert not resp['failures'], resp['failures']
    return resp


def create_cloudformation_stack_params(stack, **kwargs):
    '''Create necessary Params dictionary for cloudformation'''
    registry = REGISTRIES[stack.region]
    oauth_client = get_client_oauth_data(stack.dns[0])
    return {
        'VpcId': stack.aws['vpc_id'],
        'ECSCluster': stack.cluster,
        'StackName': stack.name,
        'StackVersion': stack.version,
        'StackDomain': stack.dns[0],
        'ContainerRegistry': registry,
        'StackDatabaseHost': stack.db['host'],
        'StackDatabasePort': str(stack.db['port']),
        'StackDatabaseName': stack.db['name'],
        'StackDatabaseUsername': stack.db['user'],
        'StackDatabasePassword': stack.db['password'],
        'Oauth2ClientId': oauth_client[0],
        'Oauth2ClientSecret': oauth_client[1],
        'AppServiceDesiredCount': str(stack.app_service_count),
        'CertificateArn': stack.aws['certificate_arn'],
        'HostedZoneId': stack.aws['hosted_zone_id'],
        'ELBSecurityGroupsIds': ','.join(stack.aws['elb_security_groups']),
        'InternalELBSecurityGroupIds': ','.join(
            stack.aws['elb_internal_security_groups']),
        'ENISecurityGroupIds': ','.join(stack.aws['eni_security_groups']),
        'PrivateSubnetIds': ','.join(stack.aws['private_subnet_ids']),
        'SubnetIds': ','.join(stack.aws['subnet_ids']),
        'IsMigration': kwargs.get('IsMigration', 'False'),
        'Environment': stack.environment,
        'EnvironmentName': stack.environment_name,
    }


def _deploy_dictionary_(stack):
    count = stack.deploy.get('count', 0)
    return {
        'count': count + 1,
        'updated': base.localnow().isoformat(),
        'project': os.environ['PROJECT_NAME'],
        'build_ident': os.environ['OPERATOR_IDENT'],
        'git_rev': os.environ['GIT_REV'],
        'git_branch': os.environ['GIT_BRANCH'],
        'git_repo': os.environ['GIT_REPO'],
        'docker_version': os.environ['DOCKER_VERSION'],
    }