#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Line too long - pylint: disable=C0301
# Invalid name  - pylint: disable=C0103
#
# Copyright (c) EMC 2011
# Copyright (c) Greenplum Inc 2010
# All Rights Reserved. 
#
"""
Activates a standby master instance when the primary master instance has 
failed.  Will promote the standby, update the system catalog
tables to make the standby master instance as the new master.
"""

from builtins import range
import os
import sys
import signal
import socket
import glob
import time
import shutil
import tempfile
from datetime import datetime, timedelta

# import GPDB modules
try:
    from pygresql import pg as pygresql
    from gppylib.commands import unix, gp, pg
    from gppylib.commands.base import ExecutionError
    from gppylib.db import dbconn
    from gppylib.gpparseopts import OptParser, OptChecker, OptionGroup, SUPPRESS_HELP
    from gppylib.gplog import get_default_logger, setup_tool_logging, enable_verbose_logging, get_logger_if_verbose
    from gppylib import gparray
    from gppylib import pgconf
    from gppylib.userinput import ask_yesno
except ImportError as e_:
    sys.exit('ERROR: Cannot import modules.  Please check that you '
             'have sourced greengage_path.sh.  Detail: ' + str(e_))

EXECNAME = os.path.split(__file__)[-1]

# Threshold values
LOG_TIME_THRESHOLD_MINS = 120

GP_RECOVERY_CONF_FILE = "recovery.conf"

GPACTIVATESTANDBY_SUCCESS = 0
GPACTIVATESTANDBY_INTERNAL_FAILURE = 1
GPACTIVATESTANDBY_FAILED = 2
GPACTIVATESTANDBY_SUCCESS_WITH_EXCEPTIONS = 3

_description = sys.modules[__name__].__doc__
_usage = "\n"


class GpActivateStandbyException(Exception):
    "Generic exception for all things activatestandby"
    pass


#-------------------------------------------------------------------------
def parseargs():
    """Parses and validates command line args."""
    
    parser = OptParser(option_class=OptChecker,
                       description=' '.join(_description.split()),
                       version='%prog version 6.32.0 build 5040+git544dae00b64')
    parser.setHelp([])
    parser.remove_option('-h')
    
    # General options section
    optgrp = OptionGroup(parser, 'General options')
    optgrp.add_option('-h', '-?', '--help', dest='help', action='store_true',
                      help='display this help message and exit')
    optgrp.add_option('-v', dest='version', action='store_true',
                      help='display version information and exit')
    parser.add_option_group(optgrp)

    # Logging options section
    optgrp = OptionGroup(parser, 'Logging options')
    optgrp.add_option('-q', '--quiet', action='store_true',
                      help='quiet mode, do not log progress to screen')
    optgrp.add_option('-l', '--logfile', type='string', default=None,
                      help='alternative logfile directory')
    optgrp.add_option('-a', help='don\'t ask to confirm standby master activation',
                      dest='confirm', default=True, action='store_false')
    parser.add_option_group(optgrp)

    # Standby activation options section
    optgrp = OptionGroup(parser, 'Standby activation options')
    optgrp.add_option('-d', '--master-data-directory', dest='master_data_dir',
                      type='string', help='standby master data directory')
    optgrp.add_option('-f', '--force', action='store_true',
                      help='force activation if standby process not running')
    optgrp.add_option('--ignore', dest='ignore', type='string', help=SUPPRESS_HELP)
    parser.add_option_group(optgrp)
    
    parser.set_defaults(quiet=False, force=False)

    # Parse the command line arguments
    (options, args) = parser.parse_args()

    if options.help:
        parser.print_help()
        parser.exit(0, None)

    if options.version:
        parser.print_version()
        parser.exit(0, None)


    if not options.master_data_dir:
        logger.info('Option -d or --master-data-directory not set. Checking environment variable MASTER_DATA_DIRECTORY')
        env_master_data_dir = os.getenv('MASTER_DATA_DIRECTORY', None)
        # check for environment variable MASTER_DATA_DIRECTORY
        if not env_master_data_dir:
            logger.fatal('Both -d parameter and MASTER_DATA_DIRECTORY environment variable are not set.')
            logger.fatal('Required option master data directory is missing.')
            parser.exit(2, None)
        options.master_data_dir = env_master_data_dir
        
    # We have to normalize this path for a later comparison
    options.master_data_dir = os.path.abspath(options.master_data_dir)

    if options.logfile and not os.path.exists(options.logfile):
        logger.fatal('Log directory %s does not exist.' % options.logfile)
        parser.exit(2, None)

    # The default logging for gpactivatestandby is verbose
    if not options.quiet:
        enable_verbose_logging()

    # There shouldn't be any args
    if len(args) > 0:
        logger.error('Unknown arguments:')
        for arg in args:
            logger.error('  %s' % arg)
        parser.exit(2, None)
        
    return options, args


#-------------------------------------------------------------------------
def print_results(array, hostname, options, successfully=False):
    """Prints out the summary of the operation."""
    
    logger.info('-----------------------------------------------------')
    if successfully:
        logger.info('The activation of the standby master has completed successfully.')
    else:
        logger.info('The activation of the standby master has completed.')
        logger.warning('Some non-critical exceptions were encountered during activation process.')
        logger.warning('Cluster state needs to be analyzed before proceeding with further instructions.')
    logger.info('%s is now the new primary master.' % hostname)
    logger.info('You will need to update your user access mechanism to reflect')
    logger.info('the change of master hostname.')
    logger.info('Do not re-start the failed master while the fail-over master is')
    logger.info('operational, this could result in database corruption!')
    logger.info('MASTER_DATA_DIRECTORY is now %s if' % options.master_data_dir)
    logger.info('this has changed as a result of the standby master activation, remember')
    logger.info('to change this in any startup scripts etc, that may be configured')
    logger.info('to set this value.')
    logger.info('MASTER_PORT is now %d, if this has changed, you' % array.master.getSegmentPort())
    logger.info('may need to make additional configuration changes to allow access')
    logger.info('to the Greengage instance.')
    logger.info('Refer to the Administrator Guide for instructions on how to re-activate')
    logger.info('the master to its previous state once it becomes available.')
    logger.info('Query planner statistics must be updated on all databases')
    logger.info('following standby master activation.')
    logger.info('When convenient, run ANALYZE against all user databases.')
    logger.info('-----------------------------------------------------')


#-------------------------------------------------------------------------
def print_summary(options):
    # Too many statements - pylint: disable=R0915

    """Print summary of the action and asks user if they want to
    continue with the activation."""

    require_force = False
    
    # check the standby master
    standby_running = check_standby_running(options)
    if not standby_running:
        require_force = True
        
    logger.info('-----------------------------------------------------')
    logger.info('Standby data directory    = %s' % options.master_data_dir)
    logger.info('Standby port              = %s' % options.standby_port)
    if options.logfile:
        logger.info('Log directory             = %s' % options.logfile)
    logger.info('Standby running           = %s' % ('yes' if standby_running else 'no'))
    logger.info('Force standby activation  = %s' % ('yes' if options.force else 'no'))
    logger.info('-----------------------------------------------------')

    check_original_master_status(options)

    # Check if we require a force
    if require_force and not options.force:
        logger.warning('If you wish to continue you must use the -f option to force')
        logger.warning('the activation process.')
        raise GpActivateStandbyException('Force activation required')
    if options.confirm:
        yn = ask_yesno(None, 'Do you want to continue with standby master activation?', 'N')
        if not yn:
            raise GpActivateStandbyException('User canceled') 

#-------------------------------------------------------------------------
def check_original_master_status(options):
    original_master_hostname = ""
    original_master_port = 0

    recovery_conf_file_location = os.path.join(options.master_data_dir, GP_RECOVERY_CONF_FILE) 
    if not os.path.exists(recovery_conf_file_location):
        logger.warning('Please make sure the command gpactivatestandby is executed on current Standby host')
        raise GpActivateStandbyException('Critical required file on standby \"%s\" is not present.' % recovery_conf_file_location)

    with open(recovery_conf_file_location) as read_file:
        for line in read_file:
            tokens = line.split()
            if tokens[0] == "primary_conninfo":
                for smaller_tokens in tokens:
                    smallest_token = smaller_tokens.split('=')
                    if smallest_token[0] == "host":
                        original_master_hostname = smallest_token[1]
                    elif smallest_token[0] == "port":
                        original_master_port = int(smallest_token[1])

    cmd = pg.ReadPostmasterTempFile.remote('Read postmaster file', original_master_port, original_master_hostname)
    (file_exists, _, _) = cmd.getResults()
    if file_exists:
        logger.warning('Appears that there is an active postgres process on %s port=%d' % (original_master_hostname, original_master_port))
        logger.warning('Need to stop current master before activating the standby.')
        logger.warning('Rare possibility is just /tmp/.s.PGSQL.%d and /tmp/.s.PGSQL.%d.* files exist on %s.' % (original_master_port, original_master_port, original_master_hostname))
        logger.warning('Need to delete these files')
        logger.warning('Then call this utility again.')
        raise GpActivateStandbyException('Active postgres process on master')
 
#-------------------------------------------------------------------------
def read_standby_port_from_postgresqlconf(options):
        logger.info("Obtaining standby's port from standby data directory")
        try:
            pgconf_dict = pgconf.readfile(options.master_data_dir + "/postgresql.conf")
        except Exception as e:
            raise GpActivateStandbyException("Couldn't read postgresql.conf in specified directory: %s." % str(e))
        options.standby_port = pgconf_dict.int('port')

#-------------------------------------------------------------------------
def get_config(port):
    """Retrieves configuration information from the catalog."""
    
    logger.info('Reading current configuration...')
    dburl = dbconn.DbURL(port=port, hostname='localhost')
    array = gparray.GpArray.initFromCatalog(dburl, utility=True)
    
    return array

#-------------------------------------------------------------------------
def check_standby_running(options):
    """Checks if the standby postmaster process is running."""
    
    return gp.getPostmasterPID('localhost', options.master_data_dir) > 0


#-------------------------------------------------------------------------
def check_or_start_standby(options):
    """
    Check if standby postmaster is running.  We need the process
    to activate it, but there could be some cases where user wants
    to activate it anyway, with potentional data loss.  In such case,
    we'll end up restarting the master after promoting it.
    """

    pid = gp.get_postmaster_pid_locally(options.master_data_dir) 


    if pid > 0:
        # we are good.  proceed with normal operation
        logger.info('found standby postmaster process')
        return True

    if not options.force:
        logger.error('Standby postmaster is not running.')
        logger.error('Use -f option to bring the system anyway')
        raise GpActivateStandbyException('postmaster is not running')
    else:
        # XXX: We don't have enough knowledge to bring up the standby
        # but there could be a situation where user needs to activate
        # it anyway.  We'll restart with the full options after
        # promoting the standby.  This is nothing but a bailout and
        # this could lose some of the latest changes from the primary.
        recovery_conf_file_location = os.path.join(options.master_data_dir, GP_RECOVERY_CONF_FILE)
        fd, trigger_file = tempfile.mkstemp(dir=options.master_data_dir)
        trigger_entry = "trigger_file = '" + trigger_file + "'"

        with open(recovery_conf_file_location, 'w') as f:
            f.write("standby_mode = 'on'\n")
            f.write("primary_conninfo = 'dummy'\n")
            f.write(trigger_entry)
        f.close()

        with open(trigger_file, 'w') as f:
            f.write('');
        f.close();

        start_master(options)

        return False

#-------------------------------------------------------------------------
def start_master(options):
    """Starts the master."""

    logger.info('Starting standby master database in utility mode...')
    gp.NewGpStart.local('Start GPDB', masterOnly=True, masterDirectory=options.master_data_dir)

#-------------------------------------------------------------------------
def stop_master():
    """Stops the master."""
    
    logger.info('Stopping standby master...')
    gp.GpStop.local('Stop GPDB', masterOnly=True, fast=True)
 
#-------------------------------------------------------------------------
def promote_standby(master_data_dir, port):
    """Promote standby"""

    logger.info('Promoting standby...')
    cmd = gp.Command('pg_ctl promote',
                     'pg_ctl promote -D %s' % master_data_dir)
    cmd.run(validateAfter=True)

    logger.debug('Waiting for connection...')
    for i in range(600):
        try:
            dburl = dbconn.DbURL(port=port, hostname='localhost')
            with dbconn.connect(dburl, utility=True, logConn=False) as conn:
                # When the new master is available for connections, we should
                # run a CHECKPOINT to force the new TimeLineID to be written to
                # the pg_control file.
                dbconn.execSQL(conn, 'CHECKPOINT')
            logger.info('Standby master is promoted')
            return True
        except pygresql.InternalError as e:
            pass
        time.sleep(1)

    return False


#-------------------------------------------------------------------------
def check_standby_activation(options, err):
    """
    Check if standby was promoted and running.
    """

    # Here we check that gpstop or something else accidentally hasn't stopped
    # our standby we're activating.
    pid = gp.get_postmaster_pid_locally(options.master_data_dir)

    if pid <= 0:
        logger.fatal('Internal error activating standby master: %s' % str(err))
        return GPACTIVATESTANDBY_INTERNAL_FAILURE
    
    # We should wrap this in try/catch because we could get here after
    # unsuccessful promotion and standby may not be able to accept queries
    try:
        array = get_config(options.standby_port)
    except Exception as e:
        logger.warning("Couldn't get cluster configuration: %s." % str(e))
        logger.fatal('Internal error activating standby master: %s' % str(err))
        return GPACTIVATESTANDBY_INTERNAL_FAILURE
    
    master_hostname = array.master.getSegmentHostName()
    master_data_dir = os.path.normpath(array.master.getSegmentDataDirectory())
    master_port = array.master.getSegmentPort()

    standby_hostname = socket.gethostname()
    standby_fqdn = socket.getfqdn()
    standby_data_dir = os.path.normpath(options.master_data_dir)
    standby_port = options.standby_port

    if not ((master_hostname == standby_hostname or
            master_hostname == standby_fqdn) and
            master_data_dir == standby_data_dir and
            master_port == standby_port):
        logger.fatal('Error activating standby master: %s' % str(err))
        return GPACTIVATESTANDBY_FAILED

    logger.warning('Encountered exception: %s.' %str(err))
    print_results(array, unix.getLocalHostname(), options, False)

    return GPACTIVATESTANDBY_SUCCESS_WITH_EXCEPTIONS
    

#-------------------------------------------------------------------------
# Main
#-------------------------------------------------------------------------

"""
Return codes:
  0 - Standby activated successfully
  1 - Command encountered an unexpected internal failure that couldn't be handled by the normal error-handling path
  2 - Standby activation failed
  3 - Standby activated successfully but exceptions were encountered
"""

# setup logging
logger = get_default_logger()
setup_tool_logging(EXECNAME, unix.getLocalHostname(), unix.getUserName())

# parse args and options
(options_, args_) = parseargs()

# if we got a new log dir, we can now set it up.
if options_.logfile:
    setup_tool_logging(EXECNAME, unix.getLocalHostname(), unix.getUserName(), logdir=options_.logfile)

try:
    read_standby_port_from_postgresqlconf(options_)

    print_summary(options_)


    # disable keyboard interrupt to prevent users from canceling
    # out of the process at a very bad time.  If there is a partial
    # update to the gp_configuration catalog and the user cancels
    # you get stuck where you can't go forward and you can't go
    # backwards.
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    requires_restart = not check_or_start_standby(options_)

    # promote standby, only if the standby is running in recovery
    if not requires_restart:
        res = promote_standby(options_.master_data_dir, options_.standby_port)
        if not res:
            raise GpActivateStandbyException('Either the set port is incorrect or '
                                             'the postmaster could not come up.')

    # now we can access the catalog.  promote action has already updated
    # catalog, so array.master is the old (promoted) standby at this point.
    array_ = get_config(options_.standby_port)

    # If we forced to start utility master, this is the time to restart
    # cluster so that the new master becomes dispatch mode.
    if requires_restart:
        cmd = gp.GpStop.local('GPDB restart', restart=True, datadir=options_.master_data_dir)

    # At this point, cancel isn't all that bad so re-enable 
    # keyboard interrupt.
    signal.signal(signal.SIGINT, signal.default_int_handler)
    
    print_results(array_, unix.getLocalHostname(), options_, True)

# Even with raised exceptions we still could have promoted standby.
# User needs to be notified about this with separate return code.
except ExecutionError as e:
    return_code = check_standby_activation(options_, e)
    sys.exit(return_code)

except GpActivateStandbyException as e:
    # We still can possibly get GPACTIVATESTANDBY_INTERNAL_FAILURE
    if "the postmaster could not come up" in str(e):
        return_code = check_standby_activation(options_, e)
        sys.exit(return_code)

    logger.fatal('Error activating standby master: %s' % str(e))
    sys.exit(GPACTIVATESTANDBY_FAILED)


except Exception as e:
    logger.fatal('Error activating standby master: %s' % str(e))
    sys.exit(GPACTIVATESTANDBY_INTERNAL_FAILURE)

sys.exit(GPACTIVATESTANDBY_SUCCESS)
