fabric/modules/deploy.py

from fabric.api import env, task
from fabric.operations import run
from fabric.operations import local

# from fabric.contrib.files import upload_template
# from utils import print_run

from .utils import virtualenv_source, booleanize, loggify
from .utils import print_console
from .utils import link_create

from .nginx import upload as upload_nginx
from .nginx import remove as nginx_remove
from .supervisor import upload as upload_supervisor
from .supervisor import remove as supervisor_remove


from .pip import setup_virtualenv
from .pip import bootstrap_pip
from .pip import install as pip_install

from .django import create_project
from .django import generate as django_generate
from .django import collectstatic as django_collectstatic

from . import initialize
from . import docker
from . import database as db


import os
import logging


@task
def setup_rootpath():
    configuration = env.config

    if env.debug:
        logging.basicConfig(
            format='\n%(levelname)s: deploy.setup_rootpath %(message)s',
            level=logging.DEBUG)

    cmd_mkdir_list = list()

    cmd_mkdir_list.append(
        "mkdir -p {rootpath}/{{logs,private,public,scripts}}".format(
            rootpath=configuration.paths.server.root))

    for key in configuration.paths.server.backups.keys():
        path = configuration.paths.server.backups[key]
        cmd_mkdir_list.append(
            "mkdir -p {rootpath_backups}".format(rootpath_backups=path))

    cmd_mkdir_list.append(
        "mkdir -p {rootpath}/logs/{{django,gunicorn,nginx,supervisor}}".format(
            rootpath=configuration.paths.server.root))

    cmd_mkdir_list.append(
        "mkdir -p {rootpath}/public/media/{{dynamic,static}}".format(
            rootpath=configuration.paths.server.root))

    if env.debug:
        for cmd_mkdir in cmd_mkdir_list:
            logging.debug("run(\"\n\t%s\n\t\"" % cmd_mkdir)

    else:

        #
        # create the directory structure in the rootpath (usually
        # /var/www/projectname.branch)

        for cmd_mkdir in cmd_mkdir_list:
            run(cmd_mkdir)


@task
def bootstrap():
    configuration = env.config

    if env.debug:
        logger = loggify('deploy', 'bootstrap')

    #
    # not doing a full sync, because we have to set up the rootpath,
    # virtualenv, files, dir structure, etc.  This means we aren't
    # going to upload gunicorn and supervisor until after we've done
    # everything else at the end of the bootstrapping process

    sync(full=False)

    # continue setting up the rootpath and virtualenv
    setup_rootpath()

    setup_virtualenv()
    bootstrap_pip()

    # create the django project
    create_project()

    #
    # link virtualenv to rootpath/private/virtualenv

    src_virtual = configuration.virtualenv.paths.root
    dst_virtual = configuration.paths.server.virtual

    #
    # link templates to rootpath/private/templates

    src_templates = configuration.paths.django.templates
    dst_templates = configuration.paths.server.django.templates

    #
    # link the django code in the project directory to the appropriate location
    # in the rootpath directory

    src_code = configuration.paths.django.root
    dst_code = configuration.paths.server.django.code

    #
    # I corrected the linking code so that it deletes already existing
    # links before creating them, otherwise you get really weird errors
    # where the a link is recreated within the destination link

    if env.debug:
        logger.debug("virtualenv.root      : %s"
                     % configuration.virtualenv.paths.root)
        logger.debug("virtualenv.bin       : %s\n" %
                     configuration.virtualenv.paths.bin)

        logger.debug("paths.server\n")

        logger.debug(" - root\t: %s" % configuration.paths.server.root)

        logger.debug(" - media\t: %s" %
                     configuration.paths.server.media.static)

        logger.debug(" - virtual\t: %s" % configuration.paths.server.virtual)

        logger.debug(" - django.code\t: %s\n" %
                     configuration.paths.server.django.code)

        logger.debug("django templates  : %s" %
                     configuration.paths.django.templates)

        logger.debug("django root       : %s" %
                     configuration.paths.django.root)

        logger.debug("django settings   : %s" %
                     configuration.paths.django.settings.root)

        logger.debug("django local      : %s" %
                     configuration.paths.django.settings.local)

        logger.debug(link_create(src_virtual, dst_virtual, debug=True))
        logger.debug(link_create(src_templates, dst_templates, debug=True))
        logger.debug(link_create(src_code, dst_code, debug=True))

    else:
        link_create(src_virtual, dst_virtual)
        link_create(src_templates, dst_templates)
        link_create(src_code, dst_code)

    #
    # create and link the scripts that manage the server
    # e.g. nginx, supervisor, gunicorn

    print_console("creating gunicorn script")
    django_generate('gunicorn', True)
    django_generate('local', True)

    print_console("creating supervisor script")
    upload_supervisor()

    print_console("creating nginx script")
    upload_nginx()

    #
    # instantiate docker containers if any

    print_console("check to see if docker containers are used")

    if hasattr(configuration, "docker"):
        print_console("generating docker configuration file")
        docker.generate()

        print_console("creating and starting docker container")
        docker.create()
    else:
        print_console("no docker containers are being used. pass")

    #
    # create and initialize the database

    print_console("in db.generate")
    db.generate()

    print_console("in db.init")
    db.init()


@task
def bootstrap_part1():
    # import database as db

    # configuration = env.config

    # if env.debug:
    #    logger = loggify('deploy', 'bootstrap_part1')

    #
    # not doing a full sync, because we have to set up the rootpath,
    # virtualenv, files, dir structure, etc.  This means we aren't
    # going to upload gunicorn and supervisor until after we've done
    # everything else at the end of the bootstrapping process

    sync(full=False)

    # continue setting up the rootpath and virtualenv
    setup_rootpath()

    setup_virtualenv()


@task
def bootstrap_part2():
    configuration = env.config

    if env.debug:
        logger = loggify('deploy', 'bootstrap')

    # create the django project
    create_project()

    #
    # link virtualenv to rootpath/private/virtualenv

    src_virtual = configuration.virtualenv.paths.root
    dst_virtual = configuration.paths.server.virtual

    #
    # link templates to rootpath/private/templates

    src_templates = configuration.paths.django.templates
    dst_templates = configuration.paths.server.django.templates

    #
    # link the django code in the project directory to the appropriate location
    # in the rootpath directory

    src_code = configuration.paths.django.root
    dst_code = configuration.paths.server.django.code

    #
    # I corrected the linking code so that it deletes already existing
    # links before creating them, otherwise you get really weird errors
    # where the a link is recreated within the destination link

    if env.debug:
        logger.debug("virtualenv.root      : %s"
                     % configuration.virtualenv.paths.root)
        logger.debug("virtualenv.bin       : %s\n" %
                     configuration.virtualenv.paths.bin)

        logger.debug("paths.server\n")

        logger.debug(" - root\t: %s" % configuration.paths.server.root)

        logger.debug(" - media\t: %s" %
                     configuration.paths.server.media.static)

        logger.debug(" - virtual\t: %s" % configuration.paths.server.virtual)

        logger.debug(" - django.code\t: %s\n" %
                     configuration.paths.server.django.code)

        logger.debug("django templates  : %s" %
                     configuration.paths.django.templates)

        logger.debug("django root       : %s" %
                     configuration.paths.django.root)

        logger.debug("django settings   : %s" %
                     configuration.paths.django.settings.root)

        logger.debug("django local      : %s" %
                     configuration.paths.django.settings.local)

        logger.debug(link_create(src_virtual, dst_virtual, debug=True))
        logger.debug(link_create(src_templates, dst_templates, debug=True))
        logger.debug(link_create(src_code, dst_code, debug=True))

    else:
        link_create(src_virtual, dst_virtual)
        link_create(src_templates, dst_templates)
        link_create(src_code, dst_code)

    #
    # create and link the scripts that manage the server
    # e.g. nginx, supervisor, gunicorn

    print_console("creating gunicorn script")
    django_generate('gunicorn', True)
    django_generate('local', True)

    print_console("creating supervisor script")
    upload_supervisor()

    print_console("creating nginx script")
    upload_nginx()

    #
    # instantiate docker containers if any

    print_console("check to see if docker containers are used")

    if hasattr(configuration, "docker"):
        print_console("generating docker configuration file")
        docker.generate()

        print_console("creating and starting docker container")
        docker.create()
    else:
        print_console("no docker containers are being used. pass")

    #
    # create and initialize the database

    print_console("in db.generate")
    db.generate()

    print_console("in db.init")
    db.init()


@task
def sync(full=True, extras=False):

    full = booleanize(full)
    extras = booleanize(extras)

    # parameter full is for use after bootstrapping the entire directory
    # structure, files etc.  Otherwise, when bootstrapping you can end up
    # trying to link to files in directories which don't exist
    #
    # so full sync is by default ON, but when using bootstrap, turn it off

    # parameter extras determines whether the extras directory is included

    from fabric.contrib.project import rsync_project

    configuration = env.config

    debug_prefix = "DEBUG: deploy.sync"

    #
    # ensure remote directory exists

    remote_dir = "{prj_name}/{prj_branch}".format(
        prj_name=configuration.project.name,
        prj_branch=configuration.project.branch)

    #
    # add a slash to the end of the directory so that rsync will upload
    # everything inside the directory as opposed to moving the directory over

    local_dir = configuration.paths.project.local + "/"

    command_mkdir_remote = "mkdir -p {remote}".format(remote=remote_dir)

    excludeitems = (".git", "*.swp", "*.swo", ".DS_Store", "*.pyc", "*.bak",
                    "build/*", "/extras", "/opt")

    if env.debug:
        print("\n%s debug: %s" % (debug_prefix, env.debug))

        print("\n%s project.name     %s" %
              (debug_prefix, configuration.project.name))

        print("%s project.branch   %s" %
              (debug_prefix, configuration.project.branch))

        print("%s path.project     %s" %
              (debug_prefix, configuration.paths.project.root))

        print("\n%s run(%s)" % (debug_prefix, command_mkdir_remote))

        print("\n{debug_prefix} rsync_project(\n\tremote_dir={remote_dir},"
              "\n\tlocal_dir={local_dir},\n\texclude={excludeitems})".format(
                  debug_prefix=debug_prefix,
                  remote_dir=remote_dir,
                  local_dir=local_dir,
                  excludeitems=excludeitems))

        # print("\n%s override:  %s " %
        #   (debug_prefix, configuration.overrides.keys()))

        # print("%s has overrides? %s" %
        #   (debug_prefix, (len(configuration.overrides.keys()) > 0)))

    else:

        # if we are working on the development branch
        # either SYNCING TO OR FROM, then just exit at this point

        if configuration.project.branch == "development":
            print("""
            ------------------------------------
            NOTE: this is a hack for the function deploy.sync()

            When I originally set up sync and bootstrap, I did not consider
            what would happen if I wanted to bootstrap the branch I was running
            on.

            That is, if I am running commands on the development branch and
            then I ask development to bootstrap or sync, I get a probem
            whereby I could potentially end up overwriting my own directory
            which would be pointless.

            So I decided that as a temporary measure if I do any kind of
            bootstrapping or anything that calls sync and I am doing it on the
            development branch server, then the sync will fail and print out
            this message.

            Instead of just keeping it in a comment, I thought it better to
            print it out so that no matter what I should always be aware of
            the issue and maybe later I can fix it.

            Right after this message gets printed, the sync function is told to
            "return" without any arguments
            ------------------------------------\n
            """)

            #
            # exit the function without any arguments

            return

        # print("remote_dir: %s" % remote_dir)
        # print("local_dir: %s" % local_dir)
        # import sys
        # sys.exit()

        run(command_mkdir_remote)

        rsync_project(remote_dir=remote_dir, local_dir=local_dir,
                      exclude=excludeitems)

        if full:
            pip_install('--all')

    # overrides()

    #
    #
    # NOTE: if using full synch

    if full:
        upload_supervisor()
        django_generate('gunicorn', True)
        upload_nginx()

        django_collectstatic()


@task
def media(source_branch):
    """
    sync media files from source branch to current branch

    source_branch - the branch to which we are copying media files
    """
    configuration = env.config

    print("project.branch: %s" % configuration.project.branch)

    #
    # if we use initialize.environemnt, we will overwrite all project
    # information, using get_config instead

    configuration_src = initialize.get_config(source_branch)
    configuration_dst = configuration

    print("branch_src: %s" % configuration_src.project.branch)
    print("branch_dst: %s" % configuration_dst.project.branch)

    print("src - server_media_dynamic: %s" %
          configuration_src.paths.server.media.dynamic)

    print("dst - server_media_dynamic: %s" %
          configuration_dst.paths.server.media.dynamic)

    # add a trailing slash to the directories
    dynamic_src = "%s/" % configuration_src.paths.server.media.dynamic
    dynamic_dst = "%s/" % configuration_dst.paths.server.media.dynamic

    # rsync can only sync to one remote, so check and see if the dst or
    # src project.host is pointing to localhost
    if configuration_dst.project.host == 'localhost':
        cmd_rsync = "rsync -pthrvz  --rsh='ssh  -p 22' " \
            " {user_src}@{host_src}:{path_src} " \
            " {path_dst}".format(
                user_src=configuration_src.project.user,
                host_src=configuration_src.project.host,
                path_src=dynamic_src,
                user_dst=configuration_dst.project.user,
                host_dst=configuration_dst.project.host,
                path_dst=dynamic_dst,
            )
    else:
        cmd_rsync = "rsync -pthrvz  --rsh='ssh  -p 22' " \
            " {path_src} " \
            " {user_dst}@{host_dst}:{path_dst}".format(
                user_src=configuration_src.project.user,
                host_src=configuration_src.project.host,
                path_src=dynamic_src,
                user_dst=configuration_dst.project.user,
                host_dst=configuration_dst.project.host,
                path_dst=dynamic_dst,
            )

    print(cmd_rsync)

    upstream = True
    remote_dir = dynamic_dst
    local_dir = dynamic_src

    if configuration_src.project.host == "localhost":
        remote_dir = dynamic_dst
        local_dir = dynamic_src
        upstream = True

        # TODO
        # get rid of all this
        #
        # rsync relies on env.host_string to determine what the username and
        # host is
        # print("before: %s" % env.host_string)
        # configuration_dst = initialize.environment(source_branch)
        # print("after: %s" % env.host_string)
        # import sys
        # sys.exit()

        print("upstream: %s" % upstream)
        print("remote_dir: %s" % remote_dir)
        print("local_dir: %s" % local_dir)
        print("\ncopy from {src} to {dst}\n".format(
            src=configuration_src.project.host,
            dst=configuration_dst.project.host))

    elif configuration_dst.project.host == "localhost":
        remote_dir = dynamic_src
        local_dir = dynamic_dst
        upstream = False

        print("upstream: %s" % upstream)
        print("remote_dir: %s" % remote_dir)
        print("local_dir: %s" % local_dir)
        print("\ncopy from {src} to {dst}\n".format(
            src=configuration_src.project.host,
            dst=configuration_dst.project.host))
    else:
        print_console("no moving media files from staging to production")
        return

    cmd_msg = "rsync_project(remote_dir={remote_dir}, " \
        "local_dir={local_dir}, upload={upstream})".format(
            remote_dir=remote_dir,
            local_dir=local_dir,
            upstream=upstream)

    print(cmd_msg)

    from fabric.contrib.project import rsync_project
    rsync_project(remote_dir=remote_dir,
                  local_dir=local_dir,
                  upload=upstream)


@task
def test():
    configuration = env.config

    projectpath = configuration.paths.project.root
    local_dir = configuration.paths.project.local + "/"

    print(hasattr(configuration, 'docker'))
    print("project path     : %s" % projectpath)
    print("local dir        : %s" % local_dir)


@task
def remove(full=True):
    """
    remove the project, delete the database
    """
    full = booleanize(full)

    configuration = env.config

    if env.debug:
        logger = loggify('deploy', 'remove')

    import sys
    if env.branch == "development" and not env.debug and full:
        print("""
        ------------------------------
        WARNING:

        You were just about to delete the development branch
        from your computer!

        THIS IS A BAD IDEA BECAUSE YOU MIGHT BE DOING IT BY ACCIDENT

        Exiting NOW.
        --------------------------------
        """)
        sys.exit()

    #
    # prepare to remove remote directory

    remote_dir = "{prj_name}/{prj_branch}".format(
        prj_name=configuration.project.name,
        prj_branch=configuration.project.branch)

    remote_dir_parent = os.path.join(
        configuration.paths.project.root,
        '..')

    command_rm_project = "rm -Rf {projectpath}".format(
        projectpath=configuration.paths.project.root)

    command_rm_project_parent = "rm -Rf {projectparent}".format(
        projectparent=remote_dir_parent)

    command_rm_rootpath = "rm -Rf {rootpath}".format(
        rootpath=configuration.paths.server.root)

    command_rm_virtualenv = "rmvirtualenv {virtualenv_name}".format(
        virtualenv_name=configuration.virtualenv.name)

    if env.debug:

        logger.debug("project.name          : %s" % configuration.project.name)

        logger.debug("project.branch        : %s"
                     % configuration.project.branch)

        logger.debug("paths.project         : %s"
                     % configuration.paths.project.root)

        logger.debug("remote_dir            : %s" % remote_dir)
        logger.debug("remote_dir parent     : %s" % remote_dir_parent)

        logger.debug("rootpath              : %s"
                     % configuration.paths.server.root)

        logger.debug("--- removal commands ---")
        logger.debug("remove project dir    : %s" % command_rm_project)

        logger.debug("remove parent dir     : %s" % command_rm_project_parent)

        logger.debug("rootpath              : %s" % command_rm_rootpath)

        logger.debug("virtualenv            : %s" % command_rm_virtualenv)

        value = run("ls %s -l | wc -l" % remote_dir_parent)
        logger.debug("value                 : %s" % value)
        logger.debug("len value             : %s" % len(value))
        logger.debug("type value            : %s" % type(value))

        logger.debug("value == 2  : %s" % (value == "2"))

    else:

        #
        # NOTE: I have to put the imports for these functions here,
        # because otherwise they interfere with this modules version
        # of "remove"

        nginx_remove()
        supervisor_remove()

        db.drop_all()

        #
        # check to see if the parent directory contains anything else

        # remote_ls_value = run("ls %s -l | wc -l" % remote_dir_parent)

        # if remote_ls_value == "2":
        #     run(command_rm_project_parent)
        # else:
        #     run(command_rm_project)

        #
        # only remove the project diretory if this is a full
        # removal.

        if full:
            run(command_rm_project)

        run(command_rm_rootpath)

        with virtualenv_source():
            run(command_rm_virtualenv)


@task
def ssh():
    configuration = env.config

    if env.debug:
        logger = loggify('deploy', 'ssh')

    if configuration.project.ssh:
        cmd_ssh = "ssh {user}@{host} -i {sshkey}".format(
            user=configuration.project.user,
            host=configuration.project.host,
            sshkey=configuration.project.ssh)
    else:
        cmd_ssh = "ssh {user}@{host}".format(
            user=configuration.project.user,
            host=configuration.project.host)

    if env.debug:
        logger.debug("ssh command: %s" % cmd_ssh)
    else:
        local(cmd_ssh)