#!/usr/bin/env python
#
# Copyright (c) Greenplum Inc 2008. All Rights Reserved.
#
from gppylib.mainUtils import addMasterDirectoryOptionForSingleClusterProgram, addStandardLoggingAndHelpOptions, gp, simple_main, \
                              ExceptionNoStackTraceNeeded, UserAbortedException
import os
import re
import shutil
import socket
import stat
import sys
import time
from datetime import datetime
from optparse import OptionGroup
from sets import Set

try:
    import gpmfr
    import threading
    import yaml
    from collections import defaultdict
    from getpass import getpass
    from gppylib import gplog
    from gppylib import pgconf
    from gppylib import userinput
    from gppylib.commands.base import Command
    from gppylib.db import dbconn
    from gppylib.gparray import GpArray
    from gppylib.gpparseopts import OptParser, OptChecker
    from gppylib.operations import Operation
    from gppylib.operations.backup_utils import *
    from gppylib.operations.dump import *
    from gppylib.operations.utils import DEFAULT_NUM_WORKERS
except ImportError, e:
    sys.exit('Cannot import modules.  Please check that you have sourced greenplum_path.sh.  Detail: ' + str(e))

INJECT_ROLLBACK = False


EXECNAME = 'gpcrondump'
GPCRONDUMP_PID_FILE = 'gpcrondump.pid'
FREE_SPACE_PERCENT = 10

logger = gplog.get_default_logger()
conn = None
lock_file_name = None

class GpCronDump(Operation):

    def __init__(self, options, args):
        if args:
            logger.warn("please note that some of the arguments (%s) aren't valid and will be ignored.", args)

        sys_args = sys.argv[1:]
        self.context = Context(options)
        self.options_list = " ".join(sys_args)

        if "-h" in sys_args and "-H" in sys_args:
            raise Exception("-H option cannot be selected with -h option. ")

        self.interactive = options.interactive
        self.cur_host = socket.gethostname()
        self.pre_vacuum = options.pre_vacuum
        self.post_vacuum = options.post_vacuum
        self.rollback = options.rollback

        self.list_backup_files = options.list_backup_files
        self.list_filter_tables = options.list_filter_tables

        self.email_details = None
        # This variable indicates whether we need to exit after verifying the DDBoost credentials
        self.context.ddboost_verify_and_exit = False

        # TODO: if action is 'append', wouldn't you expect a lack of input to result in [], as opposed to None?
        if not self.context.ddboost_hosts: self.context.ddboost_hosts = []

        if len(self.context.dump_databases) > 0:
            self.context.target_db = self.context.dump_databases[0]

        self.include_email_file = options.include_email_file

        if self.context.incremental and len(self.context.dump_databases) == 0:
            raise ExceptionNoStackTraceNeeded("Must supply -x <database name> with incremental option")

        if len(self.context.dump_databases) > 1:
            if self.context.incremental:
                raise ExceptionNoStackTraceNeeded('multi-database backup is not supported with incremental backup: %s databases selected' % len(self.context.dump_databases))
            if self.context.timestamp_key:
                raise ExceptionNoStackTraceNeeded('multi-database backup is not supported with -K option')

        if self.context.clear_dumps_only and self.context.incremental:
            raise Exception('-o option cannot be selected with incremental backup')

        if not (self.context.clear_dumps_only or self.context.clear_dumps) and self.context.cleanup_date:
            raise ExceptionNoStackTraceNeeded("Must supply -c or -o with --cleanup-date option")
        elif not (self.context.clear_dumps_only or self.context.clear_dumps) and self.context.cleanup_total:
            raise ExceptionNoStackTraceNeeded("Must supply -c or -o with --cleanup-total option")
        elif (self.context.clear_dumps_only or self.context.clear_dumps) and not (self.context.cleanup_date or self.context.cleanup_total):
            self.context.cleanup_total = '1'

        if self.list_backup_files and not self.context.timestamp_key:
            raise Exception('Must supply -K option when listing backup files')

        if not (self.context.clear_dumps_only or bool(self.context.ddboost_hosts) or bool(self.context.ddboost_user) or
                self.context.ddboost_config_remove or self.context.ddboost_show_config):
            if len(self.context.dump_databases) == 0:
                if 'PGDATABASE' in os.environ:
                    self.context.dump_databases = [os.environ['PGDATABASE']]
                    logger.info("Setting dump database to value of $PGDATABASE which is %s" % self.context.dump_databases[0])
                else:
                    if self.context.ddboost_verify is True:
                        # We would expect to verify the credentials here and return some exit code,
                        # but __init__() should return None, not 'int' - hence we are forced to use
                        # some kind of flag
                        self.context.ddboost_verify_and_exit = True
                    else:
                        raise ExceptionNoStackTraceNeeded("Must supply -x <database name> because $PGDATABASE is not set")

        if self.context.ddboost_storage_unit and not self.context.ddboost and not self.only_ddboost_options():
            raise Exception('Must specify --ddboost option together with the --ddboost-storage-unit')

        if self.list_backup_files and (self.context.ddboost or bool(self.context.ddboost_verify) or bool(self.context.ddboost_hosts) or bool(self.context.ddboost_user) or bool(self.context.ddboost_config_remove)):
            raise Exception('list backup files not supported with ddboost option')

        if self.list_filter_tables and (not self.context.dump_prefix or not self.context.incremental):
            raise Exception('list filter tables option requires --prefix and --incremental')

        if self.include_email_file:
            self._validate_parse_email_File()

        # NetBackup params check
        if self.context.netbackup_service_host or self.context.netbackup_policy or self.context.netbackup_schedule:
            if not self.context.netbackup_service_host:
                raise Exception("NetBackup service hostname (--netbackup-service-host) param should be provided in order to use NetBackup")
            if not self.context.netbackup_policy:
                raise Exception("NetBackup policy name (--netbackup-policy) param should be provided in order to use NetBackup")
            if not self.context.netbackup_schedule:
                raise Exception("NetBackup schedule name (--netbackup-schedule) param should be provided in order to use NetBackup")

        param_dict = {'Service Hostname':self.context.netbackup_service_host, 'Policy Name':self.context.netbackup_policy, 'Schedule Name':self.context.netbackup_schedule, 'Keyword':self.context.netbackup_keyword}
        validate_netbackup_params(param_dict)

        if self.context.ddboost and self.context.netbackup_service_host:
            raise Exception('--ddboost is not supported with NetBackup')

        self.replicate = options.replicate
        self.max_streams = options.max_streams
        self.quiet = options.quiet
        self.verbose = options.verbose

        # disk space checks
        if self.context.free_space_percent and self.context.ddboost:
            raise Exception('-f option cannot be selected with ddboost')

        if self.context.free_space_percent and self.context.incremental:
            raise Exception('-f option cannot be selected with incremental backup')

        if self.context.free_space_percent:
            self.context.free_space_percent = int(self.context.free_space_percent)
        else:
            self.context.free_space_percent = FREE_SPACE_PERCENT

        if self.context.ddboost:
            self.context.free_space_percent = None
            logger.info('Bypassing disk space checks due to DDBoost parameters')
        elif self.context.incremental:
            self.context.free_space_percent = None
            logger.info('Bypassing disk space checks for incremental backup')
        elif options.bypass_disk_check:
            logger.info("Bypassing disk space check as '-b' option is specified")
            self.context.free_space_percent = None

        #Check if backup_dir contains the absolute path
        if self.context.backup_dir:
            if not os.path.isabs(self.context.backup_dir):
                raise Exception("\"%s\" is not an absolute path. Please specify the absolute path where the backup files will be placed on each host." % self.context.backup_dir)

        if self.context.ddboost:
            if (not self.replicate and self.max_streams) or (self.replicate and self.max_streams is None):
                raise ExceptionNoStackTraceNeeded("--max-streams must be specified along with --replicate")
            if self.replicate:
                try:
                    if int(self.max_streams) <= 0:
                        raise ValueError()
                except ValueError:
                    raise ExceptionNoStackTraceNeeded("--max-streams must be a number greater than zero")
                # List of tuples (dbname, timestamp), to be replicated to remote
                # Data Domain after they are backed up to local Data Domain.
                self.dump_timestamps = []
            if self.context.backup_dir:
                raise ExceptionNoStackTraceNeeded('-u cannot be used with DDBoost parameters.')
            dd = gpmfr.DDSystem("local")
            self.context.dump_dir = dd.DDBackupDir
        elif self.replicate or self.max_streams:
            raise ExceptionNoStackTraceNeeded("--replicate and --max-streams cannot be used without --ddboost")
        else:
            self.context.dump_dir = 'db_dumps'

        # Cannot combine include and exclude schema filters
        if self.context.exclude_schema_file and self.context.dump_schema:
            raise Exception('-s can not be selected with --exclude-schema-file option')

        if self.context.include_schema_file and self.context.dump_schema:
            raise Exception('-s can not be selected with --schema-file option')

        if self.context.dump_schema and self.context.exclude_dump_schema:
            raise Exception('-s can not be selected with -S option')

        if self.context.exclude_schema_file and self.context.exclude_dump_schema:
            raise Exception('-S can not be selected with --exclude-schema-file option')

        if self.context.include_schema_file and self.context.exclude_dump_schema:
            raise Exception('-S can not be selected with --schema-file option')

        if self.context.include_schema_file and self.context.exclude_schema_file:
            raise Exception('--exclude-schema-file can not be selected with --schema-file option')


        # Incremental selections not allowed with schema filters
        if self.context.dump_schema and self.context.incremental:
            raise Exception('-s option can not be selected with incremental backup')

        if self.context.exclude_dump_schema and self.context.incremental:
            raise Exception('-S option can not be selected with incremental backup')

        if self.context.include_schema_file and self.context.incremental:
            raise Exception('--schema-file option can not be selected with incremental backup')

        if self.context.exclude_schema_file and self.context.incremental:
            raise Exception('--exclude-schema-file option can not be selected with incremental backup')


        # Cannot combine table and schema filters
        if self.context.exclude_schema_file and (self.context.include_dump_tables_file or self.context.exclude_dump_tables_file):
            raise Exception('--table-file and --exclude-table-file can not be selected with --exclude-schema-file option')

        if self.context.include_schema_file and (self.context.include_dump_tables_file or self.context.exclude_dump_tables_file):
            raise Exception('--table-file and --exclude-table-file can not be selected with --schema-file option')

        if self.context.dump_schema and (self.context.include_dump_tables_file or self.context.exclude_dump_tables_file):
            raise Exception('--table-file and --exclude-table-file can not be selected with -s option')

        if self.context.exclude_dump_schema and (self.context.include_dump_tables_file or self.context.exclude_dump_tables_file):
            raise Exception('--table-file and --exclude-table-file can not be selected with -S option')

        if self.context.exclude_schema_file and (self.context.include_dump_tables or self.context.exclude_dump_tables):
            raise Exception('-t and -T can not be selected with --exclude-schema-file option')

        if self.context.include_schema_file and (self.context.include_dump_tables or self.context.exclude_dump_tables):
            raise Exception('-t and -T can not be selected with --schema-file option')

        if self.context.dump_schema and (self.context.include_dump_tables or self.context.exclude_dump_tables):
            raise Exception('-t and -T can not be selected with -s option')

        if self.context.exclude_dump_schema and (self.context.include_dump_tables or self.context.exclude_dump_tables):
            raise Exception('-t and -T can not be selected with -S option')


        if self.context.include_dump_tables and self.context.incremental:
            raise Exception('include table list can not be selected with incremental backup')

        if self.context.exclude_dump_tables and self.context.incremental:
            raise Exception('exclude table list can not be selected with incremental backup')

        if self.context.include_dump_tables_file and self.context.incremental:
            raise Exception('include table file can not be selected with incremental backup')

        if self.context.exclude_dump_tables_file and self.context.incremental:
            raise Exception('exclude table file can not be selected with incremental backup')

        if self.context.clear_dumps and self.context.incremental:
            raise Exception('-c option can not be selected with incremental backup')

        if self.context.clear_catalog_dumps and self.context.incremental:
            raise Exception('-C option can not be selected with incremental backup')

        if self.context.include_schema_file and self.context.include_dump_tables:
            raise Exception('-t can not be selected with --schema-file option')

        if self.context.include_dump_tables and self.context.include_dump_tables_file:
            raise Exception('-t can not be selected with --table-file option')

        if self.context.include_dump_tables and self.context.exclude_dump_tables_file:
            raise Exception('-t can not be selected with --exclude-table-file option')

        if self.context.exclude_dump_tables and self.context.exclude_dump_tables_file:
            raise Exception('-T can not be selected with --exclude-table-file option')

        if self.context.exclude_dump_tables and self.context.include_dump_tables_file:
            raise Exception('-T can not be selected with --table-file option')

        if self.context.include_dump_tables and self.context.exclude_dump_tables:
            raise Exception('-t can not be selected with -T option')

        if self.context.include_dump_tables_file and self.context.exclude_dump_tables_file:
            raise Exception('--table-file can not be selected with --exclude-table-file option')

        if ('--inserts' in self.context.output_options or '--oids' in self.context.output_options or '--column-inserts' in self.context.output_options) and self.context.incremental:
            raise Exception('--inserts, --column-inserts, --oids cannot be selected with incremental backup')

        if self.context.include_schema_file or self.context.dump_schema or self.context.exclude_schema_file or self.context.exclude_dump_schema:
            self.validate_dump_schema()


        if self.context.incremental:
            self.context.generate_dump_timestamp()
            if not self.context.netbackup_service_host:
                self.context.full_dump_timestamp = get_latest_full_dump_timestamp(self.context)
            else:
                self.context.full_dump_timestamp = get_latest_full_ts_with_nbu(self.context)

    def validate_dump_schema(self):
        if self.context.dump_schema:
            for schema in self.context.dump_schema:
                if schema in CATALOG_SCHEMA:
                    raise Exception("can not specify catalog schema '%s' using -s option" % schema)
        elif self.context.include_schema_file:
            schema_list = get_lines_from_file(self.context.include_schema_file)
            for schema in schema_list:
                if schema in CATALOG_SCHEMA:
                    raise Exception("can not include catalog schema '%s' in schema file '%s'" % (schema, self.context.include_schema_file))
        elif self.context.exclude_dump_schema:
            for schema in self.context.exclude_dump_schema:
                if schema in CATALOG_SCHEMA:
                    raise Exception("can not specify catalog schema '%s' using -S option" % schema)
        elif self.context.exclude_schema_file:
            schema_list = get_lines_from_file(self.context.exclude_schema_file)
            for schema in schema_list:
                if schema in CATALOG_SCHEMA:
                    raise Exception("can not exclude catalog schema '%s' in schema file '%s'" % (schema, self.context.exclude_schema_file))

    def getHostSet(self):
        hostlist = self.context.gparray.get_hostlist(includeMaster=True)
        hostset = Set(hostlist)
        return hostset

    # check if one of gpcrondump option was specified, except of ddboost options
    def only_ddboost_options(self):
        return (len(self.context.dump_databases) == 0 and (not self.context.dump_schema)
                and (not self.context.backup_dir) and self.context.history
                and not (self.context.include_dump_tables or self.context.output_options or self.context.clear_dumps_only or self.pre_vacuum
                         or self.context.clear_dumps_only or self.context.dump_global or self.rollback or self.context.exclude_dump_tables
                         or self.context.dump_config or self.context.clear_dumps_only or self.context.clear_dumps or self.post_vacuum))

    def validateUserName(self, userName):
        if not re.search(r'^[a-zA-Z0-9-_]{1,30}$', userName):
            legal_username_str = """
            The username length must be between 1 to 30 characters.

            The following characters are allowed:
                1) Lowercase letters (a-z)
                2) Uppercase letters (A-Z)
                3) Numbers (0-9)
                4) Special characters (- and _)

            Note: whitespace characters are not allowed.
            """
            raise ExceptionNoStackTraceNeeded(legal_username_str)

    def validatePassword(self, password):
        if not re.search(r'^[a-zA-Z0-9!@#$%^&+=\*\\/\(\)-_~;:\'\"<>\{\}\[\]|\?\.\,`"]{1,40}$', password):
            legal_password_str = """
            The password length must be between 1 to 40 characters.

            The following characters are allowed:
                1) Lowercase letters (a-z)
                2) Uppercase letters (A-Z)
                3) Numbers (0-9)
                4) Special characters (! @ # $ % ^ & + = * \ / - _ ~ ; : ' " { [ ( < > ) ] } | ? . and ,).
            """
            raise ExceptionNoStackTraceNeeded(legal_password_str)

    def writeToFile(self, content, filePathName):
        f = open(filePathName, 'w')
        f.write(content)
        f.close()
        os.chmod(filePathName, stat.S_IREAD | stat.S_IWRITE)

    def removeDDBoostConfig(self, gpHostsSet):
        """
        Remove DD Boost configuration (local and remote) from master and segment
        servers.
        """
        for index, host in enumerate(gpHostsSet):
            ddcmd = "rm -f $HOME/DDBOOST*CONFIG*"
            logger.debug("Removing DD Boost config on host %s: %s" %
                         (host, ddcmd))
            cmdline = 'gpssh -h %s \'%s\'' % (host, ddcmd)
            if not os.system(cmdline):
                logger.info("config removed, host: %s " % host)
            else:
                logger.error("problem removing config, host: %s " % host)
                return
        logger.debug("DD Boost config removed from all GPDB servers.")

    def createDDBoostConfig(self, gpHostsSet, password):
        """
        Creates config file for local/remote Data Domain on master and all the
        segment servers.  A brief outline of this function is as follows.

           1. [Optional] Ping Data Domain host.  Abort if DD host not reachable
           from master.

           2. Enforce that local and remote DD systems are different hosts.
           Block the user from configuring them as the same host.

           3. If local DD is being configured and a config file for local DD
           already exists on the master, show its content.  Prompt the user for
           replacing the contents.  Only if 'y', go ahead.  Else abort.
           Similarly for remote DD.

           4. After configuration is written, run "gpddboost --verify".  This
           may fail if username/password is incorrect.  It will also create a
           storage unit specified by ddboost_storage_unit if it doesn't exist.

           5. The previous steps verify user-specified DD Boost config
           parameters.  Upon successful verification, this step crates the DD
           Boost config file on master and all segment hosts.

           Gotcha: One case when local and remote Data Domains may be
           misconfigured as the same host may go undetected in the current
           algorithm.  We currently compare new hostname being set for remote
           Data Domain with hostname of the local Data Domain that is already
           configured on the master node.  If they match, we flag the error and
           abort.  When a Data Domain has multiple NICs with different IP
           addresses for load balancing, gpcrondump distributes these addresses
           evenly within GPDB segment servers.  Consider the case when local
           Data Domain with IPs IP1 and IP2 is already configured on GPDB and
           the master node got IP1.  Now if the remote Data Domain is being
           configured by mistake with IP2, we will not detect the fact that IP2
           is the local Data Domain.  As a result, this misconfiguration will
           succeed without error.  As of Q1, 2013, we leave this issue to future
           releases.
        """
        logger.debug("DDboost host(s): %s" % self.context.ddboost_hosts)
        numberOfDdboostNics = len(self.context.ddboost_hosts)
        if self.context.ddboost_ping:
            for dd in self.context.ddboost_hosts:
                cmd = Command("Ping DD Host", "ping -c 5 %s" % dd)
                cmd.run()
                if not cmd.was_successful():
                    msg = "Data Domain host %s not reachable by ping. " % dd
                    msg += "Use --ddboost-skip-ping option to skip this step."
                    logger.error(msg)
                    return
            logger.debug("Connectivity to Data Domain host(s) verified.")
        localDD = None
        remoteDD = None
        try:
            localDD = gpmfr.DDSystem("local")
            logger.debug(
                "Found existing configuration for local Data Domain.")
        except:
            pass
        try:
            remoteDD = gpmfr.DDSystem("remote")
            logger.debug(
                "Found existing configuration for remote Data Domain.")
        except:
            pass
        if self.context.ddboost_remote:
            pairDD = localDD
            selfDD = remoteDD
        else:
            pairDD = remoteDD
            selfDD = localDD
        if pairDD and pairDD.hostname in self.context.ddboost_hosts:
            msg = "%s Data Domain is already configured with hostname %s." +\
                " Local and remote Data Domains must be different."
            logger.error(msg % (pairDD.id, pairDD.hostname))
            return
        if selfDD:
            logger.info("Found existing %s Data Domain configuration:" % \
                            selfDD.id)
            logger.info("\tHostname: %s" % selfDD.hostname)
            logger.info("\tDefault Backup Directory: %s" % \
                            selfDD.DDBackupDir)
            if not userinput.ask_yesno(None,
                                       "\nDo you want to replace " +
                                       "this configuration?",
                                       'N'):
                return
        ddcmdTemplate = "source " + os.environ["GPHOME"] + \
            "/greenplum_path.sh; " + os.environ["GPHOME"] + \
            "/bin/gpddboost  --setCredential "+ \
            "--hostname=%s --user=" + re.escape(self.context.ddboost_user)

        # Backup directory and DDBoost storage unit will not be part of config file for remote DD server
        if self.context.ddboost_remote:
            ddcmdTemplate += " --remote"
        else:
            ddcmdTemplate += " --defaultBackupDirectory=" + self.context.ddboost_backupdir

            if self.context.ddboost_storage_unit:
                ddcmdTemplate += " --ddboost-storage-unit=" + self.context.ddboost_storage_unit

        # Create a DD Boost config on master for each DD Boost host.  Try
        # logging into each DD Boost using the config.
        for dd in self.context.ddboost_hosts:
            ddcmd = ddcmdTemplate % dd
            logger.debug("Verifying configuration: "+ddcmd)
            ddcmd += " --password=" + re.escape(password)
            cmd = Command("Configuring DD Boost on localhost", ddcmd)
            cmd.run()
            if not cmd.was_successful():
                r = cmd.get_results()
                raise Exception("gpddboost failed to run on localhost. %s" % \
                                    r.printResult())

            if self.context.ddboost_remote:
                selfDD = gpmfr.DDSystem("remote", self.context.ddboost_backupdir, self.context.ddboost_storage_unit)
            else:
                selfDD = gpmfr.DDSystem("local", self.context.ddboost_backupdir, self.context.ddboost_storage_unit)

            # This step creates storage unit specified by ddboost_storage_unit or read
            # from cluster's config file on the Data Domain if one doesn't exist.
            # It raises exception on failure.
            logger.debug("Verifying connection to DD Host (%s)." % dd)
            selfDD.verifyLogin()
            logger.debug("Connection succeeded.")

        # At this point, verification (steps 1-4) are complete and step 5
        # starts.
        logger.debug("Writing config files on GPDB servers.")
        for index, host in enumerate(gpHostsSet):
            # If more then one NIC is specified, we configure the GP hosts with
            # these ddboost NICs in a round robin manner.  This is to address
            # the case when a single Data Domain host has multiple IP addresses
            # for load balancing.  We distribute segment servers evenly across
            # the NICs.
            currentDdboostHost = self.context.ddboost_hosts[index % numberOfDdboostNics]
            ddcmd = ddcmdTemplate % currentDdboostHost
            logger.debug("Writing configuration on host %s: %s --password=*" %
                         (host, ddcmd))
            # Note: the password can't be printed to log/stdout and therefore
            # was concatenated to ddcmd only after priniting it to log.
            ddcmd += " --password=" + re.escape(password)
            cmdline = 'gpssh -h %s \'%s\'' % (host, ddcmd)
            if 0 == os.system(cmdline):
                logger.info("config delivered successfully, host: %s " % host)
            else:
                logger.error("problem delivering config, host: %s " % host)
                return
        logger.debug("DD Boost configured on all GPDB servers.")

    # Verify the DDBoost credentials using the credentials that stored on the Master
    # TODO: verify also all the hosts in self.context.ddboost_hosts (ping to the hosts, I think there is some Python function for that)
    def _ddboostVerify(self):
        verifyCmdLine = os.environ['GPHOME'] + '/bin/gpddboost --verify'
        if self.context.ddboost_storage_unit:
            verifyCmdLine += ' --ddboost-storage-unit %s' % self.context.ddboost_storage_unit
        logger.debug("Executing gpddboost to verify DDBoost credentials: %s" % verifyCmdLine)
        if not os.system(verifyCmdLine):
            logger.info("The specified DDBoost credentials are OK")
        else:
            raise ExceptionNoStackTraceNeeded('Failed to connect to DD_host with DD_user and the DD_password')

    # Creates a temp file from the schema list provided by user if include_schema_file is not provided,
    # else sets schema_file to include_schema_file
    def generate_schema_list_file(self):
        schema_file = None
        if self.context.include_schema_file:
            schema_list = get_lines_from_file(self.context.include_schema_file)
            schema_file = create_temp_file_from_list(schema_list, 'schema_list')
        elif self.context.dump_schema:
            schema_file = create_temp_file_from_list(self.context.dump_schema, 'schema_list')
            self.context.dump_schema = []
        elif self.context.exclude_dump_schema:
            schema_list = get_include_schema_list_from_exclude_schema(self.context, self.context.exclude_dump_schema)
            schema_file = create_temp_file_from_list(schema_list, 'schema_list')
            self.context.exclude_dump_schema = []
        elif self.context.exclude_schema_file:
            schema_list = get_include_schema_list_from_exclude_schema(self.context, get_lines_from_file(self.context.exclude_schema_file))
            schema_file = create_temp_file_from_list(schema_list, 'schema_list')
        self.context.schema_file = schema_file

    # This method is called only when prefix option is specified with schema filter,
    # for full backup.
    # Method updates the include_dump_tables_file based on the schema list provided,
    # i.e All the tables in the specified schema's are added to the include_dump_tables_file.

    def generate_include_table_list_from_schema_file(self):
        dump_schemas = get_lines_from_file(self.context.schema_file)
        table_list = []
        for schema in dump_schemas:
            tables = get_user_table_list_for_schema(self.context, schema)
            for table in tables:
                table_name = table[0] + "." + table[1]
                table_list.append(table_name)
        return create_temp_file_from_list(table_list, 'include_dump_tables_file')

    def get_include_exclude_for_dump_database(self, dirty_file):
        """
        This is to empty the self.include_dump_tables and self.exclude_dump_tables, write all
        include tables and exclude tables into include tables file and exclude tables file respectively.
        """
        include_file = None
        exclude_file = None

        if self.context.incremental:
            return (dirty_file, exclude_file)

        # Make sure that this happens right before the include_dump_tables
        # check. We are relying on the filtering that happens on the
        # include_dump_tables check
        if self.context.include_dump_tables_file:
            self.context.include_dump_tables = get_lines_from_file(self.context.include_dump_tables_file)

        if self.context.include_dump_tables:
            dump_tables_without_pg_temp = []
            for table in self.context.include_dump_tables:
                if table.startswith('pg_temp_'):
                    continue
                dump_tables_without_pg_temp.append(table)
            include_file = expand_partitions_and_populate_filter_file(self.context,
                                                                      dump_tables_without_pg_temp,
                                                                      'include_dump_tables_file')
            self.context.include_dump_tables = []

        if self.context.exclude_dump_tables:
            exclude_file = expand_partitions_and_populate_filter_file(self.context, self.context.exclude_dump_tables, 'exclude_dump_tables_file')
            self.context.exclude_dump_tables = []

        if self.context.exclude_dump_tables_file:
            exclude_tables = get_lines_from_file(self.context.exclude_dump_tables_file)
            exclude_file = expand_partitions_and_populate_filter_file(self.context,
                                                                      exclude_tables,
                                                                      'exclude_dump_tables_file')

        return (include_file, exclude_file)

    # Thread to poll for existence of a signal file telling us that we can drop the pg_class lock
    def lock_handler(self, timestamp, thread_stop):
        dump_dir = self.context.get_backup_dir(timestamp)
        global lock_file_name
        lock_file_name = "%s/gp_lockfile_%s" % (dump_dir, timestamp[8:])
        while not thread_stop.is_set():
            if os.path.exists(lock_file_name):
                thread_stop.set()
                try:
                    global conn
                    if conn:
                        logger.info("Releasing pg_class lock")
                        conn.commit()
                        conn.close()
                        conn = None
                    else:
                        logger.debug("Did not release pg_class lock because connection was already closed.")
                except Exception, e:
                    logger.info("Failed to release pg_class lock, backup will proceed as normal.")
            thread_stop.wait(5)
        try:
            # Check that the file still exists and if so remove it.
            # This avoids throwing the exception if the file has already been removed by the main thread.
            if os.path.exists(lock_file_name):
                os.remove(lock_file_name)
        # This should only throw the Exception if "os.remove" fails which means there was a genuine failure.
        except Exception, e:
            logger.warn("Could not remove signal file %s, backup will proceed without releasing pg_class lock." % lock_file_name)

    def execute(self):
        if bool(self.context.ddboost_hosts) or bool(self.context.ddboost_user):
            # check that both options are specified
            if not (bool(self.context.ddboost_hosts) and bool(self.context.ddboost_user)):
                raise ExceptionNoStackTraceNeeded('For DDBoost config, both options are required --ddboost-host <ddboost_hostname> --ddboost-user <ddboost_user>.')

            # check that no other gpcrondump option was specified
            if self.only_ddboost_options() and not self.context.ddboost_config_remove:
                if not self.context.ddboost_remote and not self.context.ddboost_backupdir:
                    raise ExceptionNoStackTraceNeeded("--ddboost-backupdir must be specified when configuring local Data Domain.")
                self.validateUserName(userName=self.context.ddboost_user)
                hostset = self.getHostSet()
                p = getpass()
                self.validatePassword(password=p)
                self.createDDBoostConfig(hostset, p)
                return 0
            else:
                raise ExceptionNoStackTraceNeeded('The options --ddboost-host and --ddboost-user are standalone. They are NOT used in conjunction with any other gocrondump options (only with --ddboost-verify).')

        if self.context.ddboost_config_remove:
            if self.only_ddboost_options():
                hostset = self.getHostSet()
                # The username and the password must be at least 2 characters long, so we just use 2 spaces for each
                self.removeDDBoostConfig(hostset)
                return 0
            else:
                raise ExceptionNoStackTraceNeeded('The option --ddboost-config-remove is standalone. It is NOT used in conjunction with any other gocrondump options.')

        if self.context.clear_dumps_only:
            logger.info('Clearing dumps only...')
            self.context.generate_dump_timestamp()
            DeleteOldestDumps(self.context).run()
            return 0

        # If --ddboost-verify or --ddboost is specified, we check the credentials that stored on the Master
        # In case of --ddboost we force the verification (MPP-17510)
        if bool(self.context.ddboost_verify) or bool(self.context.ddboost):
            # DDBoost verify will create storage unit on DDBoost Server if not exist, few scenairos which do not need:
            # --list-backup-file (already pre checked and not supported with --ddboost)
            # --ddboost-config-remove (a return issued ahead), clear dumps only( -o option handled ahead),
            # --list-filter-tables(currently not supported with ddboost in the document, but no reason/comment found,
            # maybe add support later?), and --incremental, incremental dump of database should not be on different
            # storage unit from other backup set
            if not self.list_filter_tables and not self.context.incremental:
                self._ddboostVerify()

            # Check if we need to exit now
            if self.context.ddboost_verify_and_exit is True:
                return 0

        if self.context.post_script:
            self._validate_run_program()

        self._validate_dump_target()

        dump_db_info_list = []
        # final_exit_status := numeric outcome of this python program
        # current_exit_status := numeric outcome of one particular dump,
        #                        in this loop over the -x arguments
        final_exit_status = current_exit_status = 0

        if self.context.ddboost_show_config:
            self.show_ddboost_config()
            return 0

        for dumpdb in self.context.dump_databases:
            self.context.target_db = dumpdb

            if not self.context.incremental:
                self.context.generate_dump_timestamp()

            if self.list_backup_files:
                self._list_backup_files()
                logger.info('Successfully listed the names of backup files and pipes')
                return 0
            if self.list_filter_tables:
                db = self.context.target_db
                self._list_filter_tables(db, get_filter_file(self.context))
                return 0

            validate_current_timestamp(self.context)
            if self.interactive:
                self._prompt_continue()

            if self.pre_vacuum:
                logger.info('Commencing pre-dump vacuum')
                VacuumDatabase(self.context)

            if self.context.free_space_percent:
                ValidateGpToolkit(self.context)

            global conn
            conn = None
            try:
                thread_stop = threading.Event()
                lock_thread = threading.Thread(target=self.lock_handler, args=(self.context.timestamp, thread_stop))
                lock_thread.start()

                LOCK_TABLE_SQL = "LOCK TABLE pg_catalog.pg_class IN EXCLUSIVE MODE;"
                conn = dbconn.connect(dbconn.DbURL(port=self.context.master_port, dbname=self.context.target_db), utility=True)
                dbconn.execSQL(conn, LOCK_TABLE_SQL)

                ao_partition_list = get_ao_partition_state(self.context)
                co_partition_list = get_co_partition_state(self.context)
                heap_partitions = get_dirty_heap_tables(self.context)
                last_operation_list = get_last_operation_data(self.context)

                self._verify_tablenames(ao_partition_list, co_partition_list, heap_partitions)

                dirty_partitions = None
                dirty_file = None
                if self.context.incremental:
                    dirty_partitions = get_dirty_tables(self.context, ao_partition_list, co_partition_list, last_operation_list)
                    if self.context.dump_prefix and get_filter_file(self.context):
                        update_filter_file(self.context)
                    dirty_partitions = filter_dirty_tables(self.context, dirty_partitions)
                    dirty_file = write_dirty_file_to_temp(dirty_partitions)

                self.generate_schema_list_file()
                (include_file, exclude_file) = self.get_include_exclude_for_dump_database(dirty_file)
                self.context.include_dump_tables_file = include_file
                self.context.exclude_dump_tables_file = exclude_file

                if self.context.schema_file and not self.context.incremental:
                    self.context.include_dump_tables_file = self.generate_include_table_list_from_schema_file()

                dump_outcome = DumpDatabase(self.context).run()

                post_dump_outcome = PostDumpDatabase(self.context, timestamp_start = dump_outcome['timestamp_start']).run()

                if self.context.dump_stats:
                    if current_exit_status == 0:
                        self.context.timestamp = post_dump_outcome['timestamp']
                        DumpStats(self.context).run()
                        if self.context.netbackup_service_host and self.context.netbackup_policy and self.context.netbackup_schedule:
                            backup_file_with_nbu(self.context, "stats")
                    else:
                        logger.info("Skipping statistics dump due to issues during post-dump checks.")

                if self.context.netbackup_service_host and self.context.netbackup_policy and self.context.netbackup_schedule:
                    backup_file_with_nbu(self.context, "cdatabase")
                    backup_file_with_nbu(self.context, "report")

                if self.context.incremental:
                    if dirty_file and os.path.isfile(dirty_file):
                        if self.context.ddboost:
                            time.sleep(5)
                        os.remove(dirty_file)
                        remove_file_on_segments(self.context, dirty_file)
                    write_dirty_file(self.context, dirty_partitions)
                    if self.context.netbackup_service_host:
                        backup_file_with_nbu(self.context, "dirty_table")

                if self.context.schema_file:
                    schema_filename = self.context.generate_filename("schema")
                    shutil.copyfile(self.context.schema_file, schema_filename)
                    remove_file_on_segments(self.context, self.context.schema_file)
                if self.context.schema_file and os.path.exists(self.context.schema_file):
                    os.remove(self.context.schema_file)
                self.cleanup_files_on_segments()

                write_state_file(self.context, "ao", ao_partition_list)
                write_state_file(self.context, "co", co_partition_list)
                write_last_operation_file(self.context, last_operation_list)

                if self.context.netbackup_service_host and self.context.netbackup_policy and self.context.netbackup_schedule:
                    backup_state_files_with_nbu(self.context)
                    schema_filename = self.context.generate_filename("schema")
                    if os.path.exists(schema_filename) and not self.context.incremental:
                        backup_file_with_nbu(self.context, "schema")

                if self.context.ddboost:
                    backup_file_with_ddboost(self.context, "report")
                    schema_filename = self.context.generate_filename("schema")
                    if os.path.exists(schema_filename) and not self.context.incremental:
                        backup_file_with_ddboost(self.context, "schema")

                current_exit_status = max(dump_outcome['exit_status'], post_dump_outcome['exit_status'])
                if self.replicate and current_exit_status == 0:
                    self.dump_timestamps.append((self.context.target_db, post_dump_outcome["timestamp"]))
                final_exit_status = max(final_exit_status, current_exit_status)

                if self.context.incremental:
                    if final_exit_status:
                        logger.info('non-zero exit status from gp_dump, skipping generation of increments file')
                    else:
                        # if this fails, it raises an exception and bombs out RC and to the screen
                        CreateIncrementsFile(self.context).run()
                        if self.context.ddboost:
                            backup_file_with_ddboost(self.context, "increments", timestamp=self.context.full_dump_timestamp)

                        write_partition_list_file(self.context, post_dump_outcome['timestamp'])
                        if self.context.netbackup_service_host:
                            backup_file_with_nbu(self.context, "increments", timestamp=self.context.full_dump_timestamp)
                            backup_file_with_nbu(self.context, "partition_list")

            finally:
                thread_stop.set()
                try:
                    if lock_file_name and os.path.exists(lock_file_name):
                        os.remove(lock_file_name)
                    if conn:
                        conn.commit()
                        conn.close()
                        conn = None
                except Exception, e:
                    logger.debug("Failed to remove signal file or close connection, lock_handler thread probably already got to it.")

            if self.context.history:
                # This certainly does not belong under DumpDatabase, due to CLI assumptions. Note the use of options_list.
                UpdateHistoryTable(self.context,
                                   time_start = dump_outcome['time_start'],
                                   time_end = dump_outcome['time_end'],
                                   options_list = self.options_list,
                                   dump_exit_status = dump_outcome['exit_status'],
                                   timestamp = post_dump_outcome['timestamp'],
                                   pseudo_exit_status = current_exit_status).run()

            if self.context.dump_global:
                if current_exit_status == 0:
                    DumpGlobal(self.context, timestamp = post_dump_outcome['timestamp']).run()
                    if self.context.netbackup_service_host and self.context.netbackup_policy and self.context.netbackup_schedule:
                        backup_file_with_nbu(self.context, "global")
                else:
                    logger.info("Skipping global dump due to issues during post-dump checks.")

            deleted_dump_set = None
            if current_exit_status > 0 or INJECT_ROLLBACK:
                if current_exit_status >= 2 and self.rollback:
                    # A current_exit_status of 2 or higher likely indicates either a serious issue in gp_dump
                    # or that the PostDumpDatabase could not determine a timestamp. Thus, do not attempt rollback.
                    logger.warn("Dump request was incomplete, however cannot rollback since no timestamp was found.")
                    MailDumpEvent("Report from gpcrondump on host %s [FAILED]" % self.cur_host,
                                  "Failed for database %s with return code %d, dump files not rolled back, because no new timestamp was found. [Start=%s End=%s] Options passed [%s]"
                                  % (self.context.target_db, current_exit_status, dump_outcome['time_start'], dump_outcome['time_end'], self.options_list)).run()
                    raise ExceptionNoStackTraceNeeded("Dump incomplete, rollback not processed")
                elif self.rollback:
                    logger.warn("Dump request was incomplete, rolling back dump")
                    DeleteCurrentDump(self.context).run()
                    MailDumpEvent("Report from gpcrondump on host %s [FAILED]" % self.cur_host,
                                  "Failed for database %s with return code %d dump files rolled back. [Start=%s End=%s] Options passed [%s]"
                                  % (self.context.target_db, current_exit_status, dump_outcome['time_start'], dump_outcome['time_end'], self.options_list)).run()
                    raise ExceptionNoStackTraceNeeded("Dump incomplete, completed rollback")
                else:
                    logger.warn("Dump request was incomplete, not rolling back because -r option was not supplied")
                    MailDumpEvent("Report from gpcrondump on host %s [FAILED]" % self.cur_host,
                                  "Failed for database %s with return code %s dump files not rolled back. [Start=%s End=%s] Options passed [%s]"
                                  % (self.context.target_db, current_exit_status, dump_outcome['time_start'], dump_outcome['time_end'], self.options_list)).run()
                    raise ExceptionNoStackTraceNeeded("Dump incomplete, rollback not processed")
            else:
                if self.context.clear_dumps:
                    deleted_dump_set = DeleteOldestDumps(self.context).run()

            if self.post_vacuum:
                logger.info('Commencing post-dump vacuum...')
                VacuumDatabase(self.context).run()

            dump_type = 'Incremental' if self.context.incremental else 'Full database'
            self._status_report(post_dump_outcome['timestamp'],
                                dump_outcome,
                                current_exit_status,
                                deleted_dump_set,
                                dump_type)

            if self.context.dump_config:
                dump_info_dict = defaultdict(dict)
                dump_info_dict['host'] = self.cur_host
                dump_info_dict['dbname'] = self.context.target_db
                dump_info_dict['exit_status'] = current_exit_status
                dump_info_dict['start_time'] = dump_outcome['time_start']
                dump_info_dict['end_time'] = dump_outcome['time_end']
                dump_info_dict['options_passed'] = self.options_list

                dump_db_info_list.append(dump_info_dict)
            else:
                self._send_email(self.context.target_db, current_exit_status, dump_outcome['time_start'], dump_outcome['time_end'])

        if self.context.dump_config:
            config_outcome = DumpConfig(self.context).run()
            if self.context.netbackup_service_host and self.context.netbackup_policy and self.context.netbackup_schedule:
                backup_config_files_with_nbu(self.context)

            for dump_db_info in dump_db_info_list:
                self._send_email(dump_db_info['dbname'], dump_db_info['exit_status'], dump_db_info['start_time'], dump_outcome['time_end'])

        if self.context.ddboost and self.replicate:
            logger.info("Backup to local Data Domain successful.")
            for name, ts in self.dump_timestamps:
                p = gpmfr.mfr_parser()
                arglist = ["--replicate", ts, "--max-streams", self.max_streams, "--master-port", str(self.context.master_port)]
                if self.quiet:
                    arglist.append("--quiet")
                if self.verbose:
                    arglist.append("--verbose")
                if not self.interactive:
                    arglist.append("-a")
                if not self.context.ddboost_ping:
                    arglist.append("--skip-ping")
                if self.context.ddboost_storage_unit:
                    arglist.append('--ddboost-storage-unit')
                    arglist.append(self.context.ddboost_storage_unit)
                logger.info("Replicating %s to remote Data Domain. (gpmfr.py %s)" % (name, " ".join(arglist)))
                mfropt, mfrargs = p.parse_args(arglist, None)
                mfr = gpmfr.GpMfr(mfropt, mfrargs)
                try:
                    mfr.execute()
                except Exception, e:
                    logger.error(e)
                    logger.error("Backup was successful but replication to remote Data Domain failed.")
                finally:
                    mfr.restoreLogLevel()

        if self.context.post_script:
            self._run_program()

        if final_exit_status == 0:
            os._exit(0)

        return final_exit_status

    def cleanup_files_on_segments(self):
        for tmp_file in [self.context.include_dump_tables_file, self.context.exclude_dump_tables_file, self.context.schema_file]:
            if tmp_file and os.path.isfile(tmp_file):
                os.remove(tmp_file)
                remove_file_on_segments(self.context, tmp_file)

    def _get_files_file_list(self, master):
        file_list = []
        master_file_list = ['cdatabase', 'ao', 'co', 'last_operation', 'report', 'status']

        if (self.context.include_dump_tables_file or self.context.exclude_dump_tables_file or \
                                         self.context.include_dump_tables or self.context.exclude_dump_tables):
            if self.context.dump_prefix:
                master_file_list.append('filter')
            master_file_list.append('table')

        for filetype in master_file_list:
            file_list.append('%s:%s' % (master.getSegmentHostName(), self.context.generate_filename(filetype)))

        if self.context.incremental:
            file_list.append('%s:%s' % (master.getSegmentHostName(), self.context.generate_filename("increments", timestamp=self.context.full_dump_timestamp)))

        return file_list

    def _get_pipes_file_list(self, master, segdbs):
        pipe_list = []
        master_pipe_list = ['metadata', 'postdata']

        for segdb in segdbs:
            pipe_list.append('%s:%s' % (segdb.getSegmentHostName(), self.context.generate_filename("dump", dbid=segdb.getSegmentDbId())))
            if self.context.dump_config:
                pipe_list.append('%s:%s' % (segdb.getSegmentHostName(), self.context.generate_filename("segment_config", dbid=segdb.getSegmentDbId())))

        if self.context.dump_global:
            master_pipe_list.append("global")

        if self.context.dump_config:
            master_pipe_list.append("master_config")

        if self.context.dump_stats:
            master_pipe_list.append("stats")

        for filetype in master_pipe_list:
            pipe_list.append('%s:%s' % (master.getSegmentHostName(), self.context.generate_filename(filetype)))

        return pipe_list


    def show_ddboost_config(self):
        cmdStr = 'gpddboost --show-config'
        if self.context.ddboost_remote:
            cmdStr += ' --remote'
        cmd = Command('Get the ddboost configuration information', cmdStr)
        cmd.run(validateAfter=True)
        config_info = cmd.get_results().stdout
        config_items = config_info.strip().split('\n')[2:]
        logger.info('DDBoost Server Configuration Details.')
        logger.info('-----------------------------------------------------')
        for item in config_items:
            logger.info(item)
        logger.info('-----------------------------------------------------')


    def _list_backup_files(self):
        segdbs = self.context.get_current_primaries()

        # This function only cares about future backups, so always use the new filename format, don't check for an existing backup set
        self.context.use_old_filename_format = False

        pipe_list = self._get_pipes_file_list(self.context.gparray.master, segdbs)
        file_list = self._get_files_file_list(self.context.gparray.master)

        pipes_list_fname = self.context.generate_filename("pipes")
        files_list_fname = self.context.generate_filename("files")

        dirname = os.path.dirname(pipes_list_fname)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)

        write_lines_to_file(pipes_list_fname, pipe_list)
        logger.info('Added the list of pipe names to the file: %s' % pipes_list_fname)

        write_lines_to_file(files_list_fname, file_list)
        logger.info('Added the list of file names to the file: %s' % files_list_fname)

    def _list_filter_tables(self, db, filter_filename):
        logger.info("---------------------------------------------------")
        if not filter_filename:
            logger.info('No filter file found for database %s and prefix %s.' % (db, self.context.dump_prefix[:-1]))
            logger.info('All tables in %s will be included in the dump.' % db)
            return
        tables = get_lines_from_file(filter_filename)
        logger.info('Filtering %s for the following tables:' % db)
        for table in tables:
            logger.info('%s' % table)

    def _get_table_names_from_partition_list(self, partition_list):
        tablenames = []
        for part in partition_list:
            fields = part.split(',')
            if len(fields) != 3:
                raise Exception('Invalid partition entry "%s"' % part)
            tname = '%s.%s' % (fields[0].strip(), fields[1].strip())
            tablenames.append(tname)
        return tablenames

    def _verify_tablenames(self, ao_partition_list, co_partition_list, heap_partitions):
        tablenames = []

        tablenames.extend(self._get_table_names_from_partition_list(ao_partition_list))
        tablenames.extend(self._get_table_names_from_partition_list(co_partition_list))
        tablenames.extend(heap_partitions)

        tablenames = set(tablenames)

        check_funny_chars_in_names(tablenames)

    def _prompt_continue(self):
        logger.info("---------------------------------------------------")
        logger.info("Master Greenplum Instance dump parameters")
        logger.info("---------------------------------------------------")
        if len(self.context.include_dump_tables) > 0 or self.context.include_dump_tables_file:
            logger.info("Dump type                            = Single database, specific table")
            logger.info("---------------------------------------------------")
            if len(self.context.include_dump_tables) > 0:
                logger.info("Table inclusion list ")
                logger.info("---------------------------------------------------")
                for table in self.context.include_dump_tables:
                    logger.info("Table name                             = %s" % table)
                logger.info("---------------------------------------------------")
            if self.context.include_dump_tables_file:
                logger.info("Table file name                      = %s" % self.context.include_dump_tables_file)
                logger.info("---------------------------------------------------")
        elif len(self.context.exclude_dump_tables) > 0 or self.context.exclude_dump_tables_file:
            logger.info("Dump type                            = Single database, exclude table")
            logger.info("---------------------------------------------------")
            if len(self.context.exclude_dump_tables) > 0:
                logger.info("Table exclusion list ")
                logger.info("---------------------------------------------------")
                for table in self.context.exclude_dump_tables:
                    logger.info("Table name                               = %s" % table)
                logger.info("---------------------------------------------------")
            if self.context.exclude_dump_tables_file:
                logger.info("Table file name                      = %s" % self.context.exclude_dump_tables_file)
                logger.info("---------------------------------------------------")
        elif self.context.incremental:
            logger.info("Dump type                            = Incremental")
            filter_name = get_filter_file(self.context)
            if filter_name:
                logger.info("Filtering tables using:")
                logger.info("\t Prefix                        = %s" % (self.context.dump_prefix[:-1]))
                logger.info("\t Full dump timestamp           = %s" % (self.context.full_dump_timestamp))
                logger.info("---------------------------------------------------")
        else:
            logger.info("Dump type                            = Full database")
        logger.info("Database to be dumped                = %s" % self.context.target_db)
        if self.context.dump_schema:
            logger.info("Schema to be dumped                  = %s" % self.context.dump_schema)
        if self.context.backup_dir:
            logger.info("Dump directory                       = %s" % self.context.backup_dir)
        logger.info("Master port                          = %s" % self.context.master_port)
        logger.info("Master data directory                = %s" % self.context.master_datadir)
        if self.context.post_script:
            logger.info("Run post dump program                = %s" % self.context.post_script)
        else:
            logger.info("Run post dump program                = Off")
                    # TODO: failed_primary_count. do we care though? the end user shouldn't be continuing if
                    # we've already detected a primary failure. also, that particular validation is currently
                    # occurring after the _prompt_continue step in GpCronDump, as it should be...
                    #if [ $FAILED_PRIMARY_COUNT -ne 0 ];then
                    #    LOG_MSG "[WARN]:-Failed primary count             = $FAILED_PRIMARY_COUNT $WARN_MARK" 1
                    #else
                    #    LOG_MSG "[INFO]:-Failed primary count             = $FAILED_PRIMARY_COUNT" 1
                    #fi

                    # TODO: TRY_COMPRESSION is a result of insufficient disk space, compelling us to
                    # attempt compression in order to fit on disk
                    #if [ $TRY_COMPRESSION -eq 1 ];then
                    #    LOG_MSG "[INFO]:-Compression override             = On" 1
                    #else
                    #    LOG_MSG "[INFO]:-Compression override             = Off" 1
                    #fi
        on_or_off = {False: "Off", True: "On"}
        logger.info("Rollback dumps                       = %s" % on_or_off[self.rollback])
        logger.info("Dump file compression                = %s" % on_or_off[self.context.compress])
        logger.info("Clear old dump files                 = %s" % on_or_off[self.context.clear_dumps])
        logger.info("Update history table                 = %s" % on_or_off[self.context.history])
        logger.info("Secure config files                  = %s" % on_or_off[self.context.dump_config])
        logger.info("Dump global objects                  = %s" % on_or_off[self.context.dump_global])
        if self.pre_vacuum and self.post_vacuum:
            logger.info("Vacuum mode type                     = pre-dump, post-dump")
        elif self.pre_vacuum:
            logger.info("Vacuum mode type                     = pre-dump")
        elif self.post_vacuum:
            logger.info("Vacuum mode type                     = post-dump")
        else:
            logger.info("Vacuum mode type                     = Off")
        if self.context.clear_catalog_dumps:
            logger.info("Additional options                   = -c")
        if self.context.free_space_percent:
            logger.info("Ensuring remaining free disk         > %d" % self.context.free_space_percent)

        if not userinput.ask_yesno(None, "\nContinue with Greenplum dump", 'N'):
            raise UserAbortedException()

    def _status_report(self, timestamp, dump_outcome, exit_status, deleted_dump_set, dump_type):
        logger.info("Dump status report")
        logger.info("---------------------------------------------------")
        logger.info("Target database                          = %s" % self.context.target_db)
        logger.info("Dump subdirectory                        = %s" % timestamp[0:8])
        logger.info("Dump type                                = %s" % dump_type)
        if self.context.clear_dumps:
            logger.info("Clear old dump directories               = On")
            logger.info("Backup set deleted                       = %s" % deleted_dump_set)
        else:
            logger.info("Clear old dump directories               = Off")
        logger.info("Dump start time                          = %s" % dump_outcome['time_start'])
        logger.info("Dump end time                            = %s" % dump_outcome['time_end'])
        # TODO: logger.info("Number of segments dumped...
        if exit_status != 0:
            if self.rollback:
                logger.warn("Status                                   = FAILED, Rollback Called")
            else:
                logger.warn("Status                                   = FAILED, Rollback Not Called")
            logger.info("See dump log file for errors")
            logger.warn("Dump key                                 = Not Applicable")
        else:
            logger.info("Status                                   = COMPLETED")
            logger.info("Dump key                                 = %s" % timestamp)
        if self.context.compress:
            logger.info("Dump file compression                    = On")
        else:
            logger.info("Dump file compression                    = Off")
        if self.pre_vacuum and self.post_vacuum:
            logger.info("Vacuum mode type                         = pre-dump, post-dump")
        elif self.pre_vacuum:
            logger.info("Vacuum mode type                         = pre-dump")
        elif self.post_vacuum:
            logger.info("Vacuum mode type                         = post-dump")
        else:
            logger.info("Vacuum mode type                         = Off")
        if exit_status != 0:
            logger.warn("Exit code not zero, check log file")
        else:
            logger.info("Exit code zero, no warnings generated")
        logger.info("---------------------------------------------------")

    def _validate_dump_target(self):
        if len(self.context.dump_databases) > 1:
            if self.context.dump_schema or self.context.include_schema_file:
                raise ExceptionNoStackTraceNeeded("Cannot include specific schema if multiple database dumps are requested")
            if self.context.exclude_dump_schema or self.context.exclude_schema_file:
                raise ExceptionNoStackTraceNeeded("Cannot exclude specific schema if multiple database dumps are requested")
            if len(self.context.include_dump_tables) > 0 or self.context.include_dump_tables_file:
                raise ExceptionNoStackTraceNeeded("Cannot supply a table dump list if multiple database dumps are requested")
            if len(self.context.exclude_dump_tables) > 0 or self.context.exclude_dump_tables_file:
                raise ExceptionNoStackTraceNeeded("Cannot exclude specific tables if multiple database dumps are requested")
            logger.info("Configuring for multiple database dump")

        for dumpdb in self.context.dump_databases:
            self.context.target_db = dumpdb
            ValidateDatabaseExists(self.context).run()

        if self.context.dump_schema:
            for schema in self.context.dump_schema:
                ValidateSchemaExists(self.context, schema).run()

    def _validate_run_program(self):
        #Check to see if the file exists
        cmd = Command('Seeking post script', "which %s" % self.context.post_script)
        cmd.run()
        if cmd.get_results().rc != 0:
            cmd = Command('Seeking post script file', '[ -f %s ]' % self.context.post_script)
            cmd.run()
            if cmd.get_results().rc != 0:
                logger.warn("Could not locate %s" % self.context.post_script)
                self.context.post_script = None
                return
        logger.info("Located %s, will call after dump completed" % self.context.post_script)

    def _run_program(self):
        Command('Invoking post script', self.context.post_script).run()

    def _get_pgport(self):
        env_pgport = os.getenv('PGPORT')
        if not env_pgport:
            return self.context.master_port
        return env_pgport

    def _validate_parse_email_File(self):
        if os.path.isfile(self.include_email_file) is False:
            raise Exception("\'%s\' file does not exist." % self.include_email_file)
        if not self.include_email_file.endswith('.yaml'):
            raise Exception("\'%s\' is not \'.yaml\' file. File containing email details should be \'.yaml\' file." % self.include_email_file)
        if (os.path.getsize(self.include_email_file) > 0) is False:
            raise Exception("\'%s\' file is empty." % self.include_email_file)
        email_key_list = ["DBNAME", "FROM", "SUBJECT"]
        try:
            with open(self.include_email_file, 'r') as f:
                doc = yaml.load(f)
            self.email_details = doc['EMAIL_DETAILS']
            for email in self.email_details:
                if not all(keys in email for keys in email_key_list):
                    raise Exception("File not formatted")
                if not email['DBNAME']:
                    raise Exception("Database name is not provided")
        except Exception as e:
            raise Exception("\'%s\' file is not formatted properly:" % self.include_email_file)

    def _send_email(self, dump_database, exit_status, start_time, end_time):
        default_email = True
        default_subject = "Report from gpcrondump on host %s [COMPLETED]" % self.cur_host
        default_msg = "Completed for database %s with return code %d [Start=%s End=%s] Options passed [%s]" % (dump_database, exit_status, start_time, end_time, self.options_list)
        if self.include_email_file:
            for email in self.email_details:
                if dump_database in email['DBNAME']:
                    default_email = False
                    if not email['FROM']:
                        MailDumpEvent(email['SUBJECT'], default_msg).run()
                    elif not email['SUBJECT']:
                        MailDumpEvent(default_subject, default_msg, email['FROM']).run()
                    else:
                        MailDumpEvent(email['SUBJECT'], default_msg, email['FROM']).run()
        if default_email is True:
            MailDumpEvent(default_subject, default_msg).run()

def create_parser():
    parser = OptParser(option_class=OptChecker,
                       version='%prog version 5.21.2 build 5.21.2_arenadata6 409.el7',
                       description='Dumps a Greenplum database')

    addStandardLoggingAndHelpOptions(parser, includeNonInteractiveOption=True)

    addTo = OptionGroup(parser, 'Connection opts')
    parser.add_option_group(addTo)
    addMasterDirectoryOptionForSingleClusterProgram(addTo)

    addTo = OptionGroup(parser, 'Dump options')
    addTo.add_option('-r', action='store_true', dest='rollback', default=False,
                     help="Rollback dump files if dump failure detected [default: no rollback]")
    addTo.add_option('-b', action='store_true', dest='bypass_disk_check', default=False,
                     help="Bypass disk space checking [default: check disk space]")
    addTo.add_option('-j', action='store_true', dest='pre_vacuum', default=False,
                     help="Run vacuum before dump starts.")
    addTo.add_option('-k', action='store_true', dest='post_vacuum', default=False,
                     help="Run vacuum after dump has completed successfully.")
    addTo.add_option('-z', action='store_false', dest='compress', default=True,
                     help="Do not use compression [default: use compression]")
    addTo.add_option('-f', dest='free_space_percent', metavar="<0-99>",
                     help="Percentage of disk space to ensure is reserved after dump.")
    addTo.add_option('-c', action='store_true', dest='clear_dumps', default=False,
                     help="Clear old dump directories [default: do not clear]. Will remove the oldest dump directory other than the current dump directory.")
    addTo.add_option('-o', action='store_true', dest='clear_dumps_only', default=False,
                     help="Clear dump files only. Do not run a dump. Like -c, this will clear the oldest dump directory, other than the current dump directory.")
    addTo.add_option('-s', action='append', dest='dump_schema', metavar="<schema name>",
                     help="Dump the schema contained within the database name supplied via -x. Option can be used more than once")
    addTo.add_option('--schema-file', dest='include_schema_file', metavar="<filename>",
                     help="Dump tables from only the schema named in this file for the specified database. Option can be used only once.")
    addTo.add_option('-S', action='append', dest='exclude_dump_schema', metavar="<schema name>",
                     help="Exclude the specified schema's, in database specified through -x, from the dump.")
    addTo.add_option('--exclude-schema-file', dest='exclude_schema_file', metavar="<filename>",
                     help="Exclude the schemas named in this file from the dump. Option can be used only once.")
    addTo.add_option('-x', default=[], action='append', dest='dump_databases', metavar="<database name>",
                     help="Database name(s) to dump. Multiple database names will preclude the schema and table options.")
    addTo.add_option('-g', action='store_true', dest='dump_config', default=False,
                     help="Dump configuration files: postgresql.conf, pg_ident.conf, and pg_hba.conf.")
    addTo.add_option('-G', action='store_true', dest='dump_global', default=False,
                     help="Dump global objects, i.e. user accounts")
    addTo.add_option('-C', action='store_true', dest='clear_catalog_dumps', default=False,
                     help="Clean (drop) schema prior to dump [default: do not clean]")
    addTo.add_option('-R', dest='post_script', metavar="<program name>",
                     help="Run named program after successful dump. Note: program will only be invoked once, even if multi-database dump requested.")
    addTo.add_option('-B', dest='batch_default', type='int', default=DEFAULT_NUM_WORKERS, metavar="<number>",
                     help="Dispatches work to segment hosts in batches of specified size [default: %s]" % DEFAULT_NUM_WORKERS)
    addTo.add_option('-t', action='append', dest='include_dump_tables', metavar="<schema.tableN>",
                     help="Dump the named table(s) for the specified database. -t can be provided multiple times to include multiple tables.")
    addTo.add_option('-T', action='append', dest='exclude_dump_tables', metavar="<schema.tableN>",
                     help="Exclude the named table(s) from the dump. -T can be provided multiple times to exclude multiple tables.")
    addTo.add_option('-K', dest='timestamp_key', metavar="<YYYYMMDDHHMMSS>",
                     help="Timestamp key for the dump.")

    addTo.add_option('--cleanup-date', dest='cleanup_date', default=None, metavar='<yyyymmdd date>',
                     help="Remove backup sets for a yyyymmdd date.")
    addTo.add_option('--cleanup-total', dest='cleanup_total', default=None, metavar='<n int>',
                     help="Remove the n oldest backup sets based on the backup timestamp.")
    addTo.add_option('--list-backup-files', dest='list_backup_files', default=False, action='store_true',
                     help="Files created during the dump operation for a particular input timestamp")
    addTo.add_option('--prefix', dest='local_dump_prefix', default='', metavar='<filename prefix>',
                     help="Prefix to be added to all files created in the dump")
    addTo.add_option('--list-filter-tables', dest='list_filter_tables', default=False, action='store_true',
                     help="List tables to be included in a dump based on the current filter.")

    # TODO: HACK to remove existing -h
    help = parser.get_option('-h')
    parser.remove_option('-h')
    help._short_opts.remove('-h')
    parser.add_option(help)
    addTo.add_option('-h', action='store_true', dest='history', default=True,
                         help="Record details of database dump in database table %s in database supplied via -x option."
                              "Utility will create table if it does not currently exist."
                              "This option will be deprecated in a future release." % UpdateHistoryTable.HISTORY_TABLE)

    addTo.add_option('-H', action='store_false', dest='history', default=True,
                     help="Disable recording details of database dump in database table %s in database supplied via -x option."
                          "If not specified, the utility  will create/update the history table." % UpdateHistoryTable.HISTORY_TABLE)

    addTo.add_option('-u', dest='backup_dir', metavar="<BACKUPFILEDIR>",
                     help="Directory where backup files are placed [default: data directories]")
    addTo.add_option('-E', dest='encoding', metavar="<encoding>",
                     help="Dump the data under the given encoding")
    addTo.add_option('--clean', const='--clean', action='append_const', dest='output_options',
                     help="Clean (drop) schema prior to dump")
    addTo.add_option('--inserts', const='--inserts', action='append_const', dest='output_options',
                     help="Dump data as INSERT, rather than COPY, commands.")
    addTo.add_option('--column-inserts', const='--column-inserts', action='append_const', dest='output_options',
                     help="Dump data as INSERT commands with colun names.")
    addTo.add_option('--oids', const='--oids', action='append_const', dest='output_options',
                     help="Include OIDs in dump.")
    addTo.add_option('--no-owner', const='--no-owner', action='append_const', dest='output_options',
                     help="Do not output commands to set object ownership.")
    addTo.add_option('--no-privileges', const='--no-privileges', action='append_const', dest='output_options',
                     help="Do not dump privileges (grant/revoke).")
    addTo.add_option('--use-set-session-authorization', const='--use-set-session-authorization', action='append_const', dest='output_options',
                     help="Use SESSION AUTHORIZATION commands instead of ALTER OWNER commands.")
    addTo.add_option('--rsyncable', const='--rsyncable', action='append_const', dest='output_options',
                     help="Pass the --rsyncable option to gzip, if compression is being used.")
    addTo.add_option('--table-file', dest='include_dump_tables_file', metavar="<filename>",
                     help="Dump the tables named in this file for the specified database. Option can be used only once.")
    addTo.add_option('--exclude-table-file', dest='exclude_dump_tables_file', metavar="<filename>",
                     help="Exclude the tables named in this file from the dump. Option can be used only once.")
    addTo.add_option('--email-file', dest='include_email_file', metavar="<filename>",
                     help="Customize the 'Sender' and 'Subject' of the email to be sent after backup")
    addTo.add_option('--dump-stats', action="store_true", dest='dump_stats', default=False,
                     help="Dump database statistics")

    parser.add_option_group(addTo)

    # Incremental option(s)
    incrOpt = OptionGroup(parser, "Incremental")
    incrOpt.add_option('--incremental', action='store_true', dest='incremental', default=False, help='Dump incremental backup.')
    parser.add_option_group(incrOpt)

    # NetBackup option(s)
    nbuOpt = OptionGroup(parser, "NetBackup")
    nbuOpt.add_option('--netbackup-service-host', dest='netbackup_service_host', metavar="<server name>",
                      help="NetBackup service hostname")
    nbuOpt.add_option('--netbackup-policy', dest='netbackup_policy', metavar="<policy name>",
                      help="NetBackup policy name")
    nbuOpt.add_option('--netbackup-schedule', dest='netbackup_schedule', metavar="<schedule name>",
                      help="NetBackup schedule name")
    nbuOpt.add_option('--netbackup-block-size', dest='netbackup_block_size', metavar="<block size>",
                      help="NetBackup data transfer block size")
    nbuOpt.add_option('--netbackup-keyword', dest='netbackup_keyword', metavar="<keyword>",
                      help="NetBackup Keyword")
    parser.add_option_group(nbuOpt)

    # DDBoost option(s)
    ddOpt = OptionGroup(parser, "DDBoost")
    ddOpt.add_option('--ddboost', dest='ddboost', help="Dump to DDBoost using ~/.ddconfig", action="store_true", default=False)
    ddOpt.add_option('--replicate', dest='replicate', help="Post dump, replicate the backup to remote Data Domain system.", action="store_true", default=False)
    ddOpt.add_option('--max-streams', dest='max_streams', action="store", default=None,
                     help="Maximum number of Data Domain I/O streams to be used for replication.")
    parser.add_option_group(ddOpt)

    # DDBoostConfig option(s)
    ddConfigOpt = OptionGroup(parser, "DDBoostConfig")
    # ddboost-host may have more then one host
    ddConfigOpt.add_option('--ddboost-host', dest='ddboost_hosts', action='append', default=None,
                           help="Configuration of ddboost hostname.")
    ddConfigOpt.add_option('--ddboost-user', dest='ddboost_user', action='store', default=None,
                           help="Configuration of ddboost user.")
    ddConfigOpt.add_option('--ddboost-backupdir', dest='ddboost_backupdir', action='store', default=None,
                           help="Default backup directory on local Data Domain system.")
    ddConfigOpt.add_option('--ddboost-remote', dest='ddboost_remote', action='store_true', default=False,
                           help="Configuration parameters for remote Data Domain system.")
    ddConfigOpt.add_option('--ddboost-storage-unit', dest='ddboost_storage_unit', action='store', default=None,
                           help="Storage unit where backup files are placed on the ddboost server")
    ddConfigOpt.add_option('--ddboost-config-remove', dest='ddboost_config_remove', action='store_true', default=False,
                           help="Remove ~/.ddconfig file.")
    ddConfigOpt.add_option('--ddboost-verify', dest='ddboost_verify', action='store_true', default=False,
                           help="Verify DDBoost credentials on DDBoost host.")
    ddConfigOpt.add_option('--ddboost-skip-ping', dest='ddboost_ping', action='store_false', default=True,
                           help="Ping DDBoost host as a sanity check before writing configuration.")
    ddConfigOpt.add_option('--ddboost-show-config', dest='ddboost_show_config', action='store_true', default=False,
                           help="Show DDBoost and MFR related configuration information: DDBoost hostname, \n"
                                "username, default backup directory, default storage unit.")
    parser.add_option_group(ddConfigOpt)

    parser.setHelp([
    """
Crontab entry (example):
SHELL=/bin/bash
5 0 * * * . $HOME/.bashrc; $GPHOME/bin/gpcrondump -x template1 -aq >> <name of cronout file>
Set the shell to /bin/bash (default for cron is /bin/sh
Dump the template1 database, start process 5 minutes past midnight on a daily basis
    """,
    """
Mail configuration
This utility will send an email to a list of email addresses contained in a file
named mail_contacts. This file can be located in the GPDB super user home directory
or the utility bin directory ${GPHOME}/bin. The format of the file is one email
address per line. If no mail_contacts file is found in either location, a warning message
will be displayed.
    """
    ])

    return parser

if __name__ == '__main__':
    sys.argv[0] = EXECNAME                                                              # for cli_help
    simple_main(create_parser, GpCronDump, {"pidfilename" : GPCRONDUMP_PID_FILE,
                                            "programNameOverride" : EXECNAME})        # for logger
