fabric/modules/initialize.py
Ronny Abraham 0728560972 major tweaks to getting this to work
deleted:    fabfile.pyc
	modified:   modules/deploy.py
	modified:   modules/initialize.py
	modified:   modules/nginx.py
	new file:   resources/ronnyabraham.pem
2016-09-07 21:35:45 +03:00

998 lines
31 KiB
Python

import os
# import yaml
import fabric.utils
import maintenance
from fabric.api import env, task
from utils import check_debug
# from utils import loggify
# import configuration values from PRJ_ROOT/sites/meta/configuration
class DataObject(object):
def keys(self):
return self.__dict__.keys()
def value(self, key):
return self.__dict__[key]
def __getitem__(self, key):
return self.__dict__[key]
def addbranch(self, name):
setattr(self, name, DataObject(dict()))
def __init__(self, d):
for a, b in d.items():
if isinstance(b, (list, tuple)):
setattr(self, a, [DataObject(x) if isinstance(x, dict)
else x for x in b])
else:
setattr(self, a, DataObject(b) if isinstance(b, dict) else b)
def extend_object(envobject, dictname):
setattr(envobject, dictname, fabric.utils._AttributeDict())
def environment(branchname):
env.config = get_config(branchname)
env.branch = branchname
env.hosts = [env.config.project.host]
env.user = env.config.project.user
env.host_string = env.hosts[0]
# set database host to the docker_ip
host = env.config.server.database.host
from docker import docker_ip
if host == 'docker':
env.config.server.database.host = docker_ip()
return env.config
def add_maintenance(dataobject, layout, config, sectionname):
"""
helper function to add maintenance information to the dataobject
dataobject - will be used as the configuration object in the fabric modules
layout - dictionary containing the layout values for this branch
config - dictionary containing the config values for this branch
sectionname - name of the section of the maintenance information in config
"""
if not hasattr(dataobject.maintenance, sectionname):
dataobject.maintenance.addbranch(sectionname)
_maintenance = getattr(dataobject.maintenance, sectionname)
#
# add the execute attribute
_maintenance.execute = config['maintenance'][sectionname]['execute']
#
# if there is an editor attribute, add it
if 'editor' in config['maintenance'][sectionname]:
_maintenance.editor = config['maintenance'][sectionname]['editor']
#
# add the commands
if not hasattr(_maintenance, "commands"):
_maintenance.addbranch("commands")
keys = config['maintenance'][sectionname]['commands'].keys()
for k in keys:
setattr(_maintenance.commands, k,
config['maintenance'][sectionname]['commands'][k])
def add_template(dataobject, layout, config, section, template_name="conf"):
"""
this is a helper function to allow me to 'more easily' add the info I need
from the config file on templates to the dataobject
dataobject - will be used as the configuration object in the fabric modules
layout - dictionary containing the layout values for this branch
config - dictionary containing the config values for this branch
section - name of the section of the template information in config
template_name - name of which template we are looking for in that
particular section of the templates configuration info, default to "conf"
returns nothing - modifies the passed dataobject.
"""
# dataobject.templates.section
# dataobject.templates.section.path.local
# dataobject.templates.section.path.dest
# dataobject.templates.section.tname.src
# dataobject.templates.section.tname.dst
# if env.debug:
# logger = loggify('intialize', 'add_template')
if not hasattr(dataobject.templates, section):
dataobject.templates.addbranch(section)
# shortcut to working on the branch we just create above
_template = getattr(dataobject.templates, section)
# create a "path" branch to the _templates
if not hasattr(_template, "path"):
_template.addbranch("path")
# first get the default template values from layout
var_section_path = os.path.join(
layout['paths']['templates']['conf'],
layout['templates'][section]['path']
)
var_section_source = layout['templates'][section][template_name]['source']
var_section_output = layout['templates'][section][template_name]['output']
if 'templates' in config and section in config['templates']:
_config = config['templates'][section]
var_section_path = _config['path']
var_section_source = _config[template_name]['source']
var_section_output = _config[template_name]['output']
# define the local, and dest paths
_template.path.local = os.path.join(
dataobject.paths.project.local,
var_section_path)
_template.path.remote = os.path.join(
dataobject.paths.project.root,
var_section_path)
if not hasattr(_template, template_name):
_template.addbranch(template_name)
conf_template = getattr(_template, template_name)
conf_template.src = var_section_source
conf_template.dst = var_section_output
def get_config(branchname):
# get a logger object
# if env.debug:
# logger = loggify('intialize', 'get_config')
# create two yaml dictionaries based on the branch configuration file
# and the standard file layout file
config = maintenance.load_configuration("config", branchname)
layout = maintenance.load_configuration("layout", branchname)
# fabric_config = maintenance.load_configuration("fabric", branchname)
# maintenance.check_version(branchname)
dataobject = DataObject(dict())
dataobject.addbranch('project')
dataobject.project.name = config['project']['name']
dataobject.project.branch = config['project']['branch']
dataobject.project.extension = config['project']['extension']
dataobject.project.extendedname = "{name}.{ext}".format(
name=dataobject.project.name,
ext=dataobject.project.extension)
dataobject.project.host = config['project']['host']
dataobject.project.user = config['project']['user']
dataobject.project.group = config['project']['group']
dataobject.project.sudo = config['project']['sudo']
#
# django settings directory
# NOTE:
# this is a tricky problem, because sometimes the project.name will NOT
# be what I originally set the settings folder name to. For example, I
# created a project named 'raquelsanchez' but then the production url was
# set to 'raquelsanchezart', well, those two names don't match, and I don't
# feel like going through the complication of changing the wsgi files and
# folder names for the settings directory.
#
# so over here, we check to see if django.settings_folder is the same as
# project.name, if it is, great, if not we set it to whatever is in
# config.django.settings_folder
dataobject.project.addbranch('django')
# this is the default name for the django settings_dir
dataobject.project.django.settings_folder = \
config['project']['name']
if 'settings_folder' in config['django']:
dataobject.project.django.settings_folder = \
config['django']['settings_folder']
dataobject.addbranch('paths')
dataobject.paths.addbranch('project')
dataobject.paths.project.local = maintenance.get_project_root()
if 'home' in config['project']['paths']:
project_home_dir = config['project']['paths']['home']
else:
project_home_dir = os.path.join(
dataobject.project.name,
dataobject.project.branch)
dataobject.paths.project.root = os.path.join(
config['project']['paths']['root'],
project_home_dir)
# DEBUG REMOVE
if check_debug(env):
import utils
utils.printvar('project.branch', dataobject.project.branch)
utils.printvar('project.root', dataobject.paths.project.root)
utils.prompt_continue()
#
# these are the locations of the scripts/conf file both remote and local
dataobject.paths.addbranch('conf')
dataobject.paths.conf.remote = os.path.join(
dataobject.paths.project.root,
layout['paths']['templates']['conf'])
dataobject.paths.conf.local = os.path.join(
dataobject.paths.project.local,
layout['paths']['templates']['conf'])
#
# tools used in development
dataobject.paths.addbranch('tools')
dataobject.paths.tools.fabric = os.path.join(
dataobject.paths.project.root,
layout['paths']['tools']['fabric']['root'])
dataobject.addbranch('tools')
dataobject.tools.addbranch('fabric')
dataobject.tools.fabric.addbranch('templates')
dataobject.tools.fabric.templates.conf = os.path.join(
dataobject.paths.project.root,
dataobject.paths.tools.fabric,
layout['paths']['tools']['fabric']['templates']['conf'])
dataobject.tools.fabric.templates.meta = os.path.join(
dataobject.paths.project.root,
dataobject.paths.tools.fabric,
layout['paths']['tools']['fabric']['templates']['meta'])
dataobject.tools.fabric.templates.readmes = os.path.join(
dataobject.paths.project.root,
dataobject.paths.tools.fabric,
layout['paths']['tools']['fabric']['templates']['readmes'])
#
# paths for django
dataobject.paths.addbranch('django')
dataobject.paths.django.root = os.path.join(
dataobject.paths.project.root, layout['paths']['django']['root'])
dataobject.paths.django.apps = os.path.join(
dataobject.paths.django.root,
layout['paths']['django']['apps'])
dataobject.paths.django.templates = os.path.join(
dataobject.paths.project.root,
'templates')
dataobject.paths.django.fixtures = os.path.join(
dataobject.paths.project.root,
layout['paths']['extras']['fixtures'])
dataobject.paths.django.addbranch('settings')
dataobject.paths.django.settings.root = os.path.join(
dataobject.paths.django.root,
dataobject.project.django.settings_folder)
dataobject.paths.django.settings.local = os.path.join(
dataobject.paths.django.settings.root,
layout['paths']['django']['settings.local'])
#
# path to supervisor configuration directory on target machine
dataobject.paths.addbranch("supervisor")
dataobject.paths.supervisor.conf = config['supervisor']['paths']['conf']
#
# django local settings name
dataobject.addbranch('imports')
dataobject.imports.settings = \
"{projectname}.{settings_local}.{projectbranch}".format(
projectname=dataobject.project.django.settings_folder,
settings_local=layout['paths']['django']['settings.local'],
projectbranch=dataobject.project.branch)
#
# server information
dataobject.addbranch('server')
#
# nginx server
dataobject.server.addbranch('nginx')
dataobject.server.nginx.port = config['nginx']['port']
if config['project']['host'] == "localhost":
dataobject.server.nginx.host = "{projectname}.{ext}".format(
ext=dataobject.project.extension,
projectname=dataobject.project.name)
else:
dataobject.server.nginx.host = config['project']['host']
dataobject.server.addbranch('django')
dataobject.server.django.port = config['django']['port']
dataobject.server.django.host = config['django']['host']
#
# initialize the database server information
_init_database(dataobject, config, layout)
#
# initialize the virtualenv information
_init_virtualenv(dataobject, config, layout)
#
# dataobject Templates
dataobject.addbranch('templates')
#
# django template information
dataobject.templates.addbranch('django')
add_template(dataobject, layout, config, "django", "settings")
add_template(dataobject, layout, config, "django", "local")
add_template(dataobject, layout, config, "django", "wsgi")
#
# add templates
# database template information
add_template(dataobject, layout, config, "database", "init")
add_template(dataobject, layout, config, "database", "re_init")
add_template(dataobject, layout, config, "database", "drop_db")
add_template(dataobject, layout, config, "database", "drop_all")
#
# gunicorn template information
add_template(dataobject, layout, config, "gunicorn")
# supervisor template information
add_template(dataobject, layout, config, "supervisor")
# nginx template information
add_template(dataobject, layout, config, "nginx")
# docker template information
# make sure we have docker information available, otherwise spit out
# that we aren't doing it
if 'docker' in config:
if 'database' in config['docker']:
add_template(dataobject, layout, config, "docker", "database")
else:
print "NOTE: docker.database does not exist for this branch"
else:
print "NOTE: docker information does not exist for this branch"
#
# nginx information
_init_nginx(dataobject, config, layout)
#
# initialize the root server directory information
# ie paths, etc.
_init_root_server(dataobject, config, layout)
_init_backups(dataobject, config, layout)
_init_logging(dataobject, layout, config)
_init_docker(dataobject, layout, config)
_init_overrides(dataobject, layout, config)
#
# maintenance commands
dataobject.addbranch("maintenance")
add_maintenance(dataobject, layout, config, 'nginx')
add_maintenance(dataobject, layout, config, 'supervisor')
return dataobject
def _init_database(configuration, config, layout):
"""
initialize the database server information
"""
configuration.server.addbranch('database')
configuration.server.database.name = config['database']['name']
configuration.server.database.port = config['database']['port']
configuration.server.database.host = config['database']['host']
configuration.server.database.backend = config['database']['backend']
configuration.server.database.user = \
config['database']['users']['default']['name']
configuration.server.database.password = \
config['database']['users']['default']['pass']
configuration.server.database.addbranch('admin')
configuration.server.database.admin.user = \
config['database']['users']['admin']['name']
configuration.server.database.admin.password = \
config['database']['users']['admin']['pass']
def _init_virtualenv(configuration, config, layout):
"""
initialize all the virtualenv information
"""
#
# virtualenv
configuration.addbranch('virtualenv')
#
# workon_home variable
configuration.virtualenv.workon = config['virtualenv']['workon']
virtualenv_requirements = "{branch}.txt".format(
branch=configuration.project.branch)
configuration.virtualenv.requirements = os.path.join(
configuration.paths.project.root,
layout['virtualenv']['requirements'],
virtualenv_requirements)
#
# determine the virtualenv name, if it is set as "Null"
# the craete it based on the project name and the extension
# associated with this project
if 'name' in config['virtualenv']:
virtualenv_name = config['virtualenv']['name']
else:
virtualenv_name = configuration.project.extendedname
configuration.virtualenv.name = virtualenv_name
#
# paths used by the virtualenv configuration
configuration.virtualenv.addbranch('paths')
#
# the location of the virtualenv inside of WORKON_HOME
configuration.virtualenv.paths.root = os.path.join(
configuration.virtualenv.workon,
configuration.virtualenv.name)
#
# virtualenv bin directory
configuration.virtualenv.paths.bin = os.path.join(
configuration.virtualenv.paths.root,
'bin')
#
# virtualenv site-packages directory (I hate looking it up)
configuration.virtualenv.paths.sitepackages = os.path.join(
configuration.virtualenv.paths.root,
"lib", "python2.7", "site-packages")
#
# path to the activate file for this virtualenv
configuration.virtualenv.activate = os.path.join(
configuration.virtualenv.paths.bin,
'activate')
def _init_nginx(configuration, config, layout):
"""
all nginx configuration info is done here
"""
configuration.addbranch("nginx")
#
# nginx enabled, and available directory paths
#
# NOTE: on some installations of Nginx, there aren't both a
# sites-available and sites-enabled directories, for example
# on mac brew, there is only "servers". So where this is true,
# I set sites-available to None, and I dump everything in
# whatever I called the sites-enabled directory
configuration.nginx.sites_enabled = os.path.join(
config['nginx']['paths']['root'],
config['nginx']['paths']['enabled'])
if config['nginx']['paths']['available'] is None:
configuration.nginx.sites_available = configuration.nginx.sites_enabled
else:
configuration.nginx.sites_available = os.path.join(
config['nginx']['paths']['root'],
config['nginx']['paths']['available'])
# nginx conf file name
configuration.nginx.addbranch("conf")
configuration.nginx.conf.name = "{name}.conf".format(
name=configuration.project.extendedname)
# nginx path to conf file location
configuration.nginx.conf.destination = os.path.join(
configuration.nginx.sites_available,
configuration.nginx.conf.name)
def _init_root_server(configuration, config, layout):
"""
initialize all the information necessary for the root
server. ie, paths, etc.
"""
#
# the main server directory, which is made public to nginx,
# supervisor, etc.
configuration.paths.addbranch("server")
configuration.paths.server.root = os.path.join(
config['rootpath'],
configuration.project.extendedname)
#
# server virtual environment directory
configuration.paths.server.virtual = os.path.join(
configuration.paths.server.root,
'private', 'virtualenv')
#
# server scripts directory
configuration.paths.server.scripts = os.path.join(
configuration.paths.server.root,
'scripts')
configuration.paths.server.code = os.path.join(
configuration.paths.server.root,
'private',
'code')
configuration.paths.server.logs = os.path.join(
configuration.paths.server.root,
'logs')
configuration.paths.server.addbranch("django")
configuration.paths.server.django.templates = os.path.join(
configuration.paths.server.root,
'private', 'templates')
configuration.paths.server.django.code = os.path.join(
configuration.paths.server.root,
'private', 'code')
configuration.paths.server.addbranch("media")
configuration.paths.server.media.static = os.path.join(
configuration.paths.server.root,
config['media']['paths']['root'],
config['media']['paths']['static'])
configuration.paths.server.media.dynamic = os.path.join(
configuration.paths.server.root,
config['media']['paths']['root'],
config['media']['paths']['dynamic'])
def _init_backups(configuration, config, layout):
"""
initialize paths for the server backup director
"""
#
# paths for the server backup directory
configuration.paths.server.addbranch("backups")
# these are the default backups paths based on the parent
# server directory path
# root backups directory
configuration.paths.server.backups.root = os.path.join(
configuration.paths.server.root,
layout['paths']['backups']['root'])
# database subdirectory of the root backups directory
configuration.paths.server.backups.database = os.path.join(
configuration.paths.server.backups.root,
layout['paths']['backups']['database'])
configuration.paths.server.backups.fixtures = os.path.join(
configuration.paths.server.backups.root,
layout['paths']['backups']['fixtures'])
configuration.paths.server.backups.media = os.path.join(
configuration.paths.server.backups.root,
layout['paths']['backups']['media'])
# check to see if there is an overriden folder name for the backup path
# if there is, then apply THE FULL PATH, don't tack it onto the root server
# path. Here I am assuming that the full corrected path is being given
if 'backups' in config:
# root backups directory
if 'root' in config['backups']['paths']:
configuration.paths.server.backups.root = \
config['backups']['paths']['root']
# database subdirectory of the root backups directory
if 'database' in config['backups']['paths']:
configuration.paths.server.backups.database = \
config['backups']['paths']['root']
def _init_logging(configuration, layout, config):
"""
initialize all logging information
"""
configuration.addbranch("logging")
#
# logging for nginx
configuration.logging.addbranch("nginx")
#
# if 'nginx' log directory paths are defined
# in this configuration file then apply them
if 'nginx' in config['logging']['paths']:
configuration.logging.nginx.access = \
config['logging']['paths']['nginx']['access']
configuration.logging.nginx.error = \
config['logging']['paths']['nginx']['error']
else:
# we don't have anything special defined, use the
# the standard logs directory and give it standard paths
configuration.logging.nginx.access = os.path.join(
configuration.paths.server.logs,
'nginx', 'access.log')
configuration.logging.nginx.error = os.path.join(
configuration.paths.server.logs,
'nginx', 'error.log')
#
# logging for supervisor
configuration.logging.addbranch("supervisor")
#
# if 'nginx' log directory paths are defined
# in this configuration file then apply them
if 'supervisor' in config['logging']['paths']:
configuration.logging.nginx.access = \
config['logging']['paths']['supervisor']['access']
configuration.logging.nginx.error = \
config['logging']['paths']['supervisor']['error']
else:
# we don't have anything special defined, use the
# the standard logs directory and give it standard paths
configuration.logging.supervisor.out = os.path.join(
configuration.paths.server.logs,
'supervisor', 'out.log')
configuration.logging.supervisor.err = os.path.join(
configuration.paths.server.logs,
'supervisor', 'err.log')
#
# django logging
configuration.logging.addbranch('django')
configuration.logging.django.addbranch('handlers')
log_keys = layout['logging']['django']['handlers']
logging_path_project = layout['paths']['logging']['django']['project']
logging_path_server = layout['paths']['logging']['django']['server']
#
# the logs for django are handled differently from other log paths
# for the django log handlers, I want to place them in the a subdirectory
# of the project directory and link that directory to the server log
# directory
# find out and set the log paths to the project directory
for log_handler_key in log_keys:
configuration.logging.django
log_handler_file = \
layout['logging']['django']['handlers'][log_handler_key]['file']
handler_path_project = os.path.join(
configuration.paths.project.root,
logging_path_project, log_handler_file)
handler_path_server = os.path.join(
configuration.paths.server.logs,
logging_path_server, log_handler_file)
configuration.logging.django.handlers.addbranch(log_handler_key)
configuration_handler = getattr(configuration.logging.django.handlers,
log_handler_key)
configuration_handler.addbranch('name')
handler_name = \
layout['logging']['django']['handlers'][log_handler_key]['name']
configuration_handler.name.project = handler_name
configuration_handler.name.server = "server.%s" % handler_name
configuration_handler.addbranch('path')
configuration_handler.path.project = handler_path_project
configuration_handler.path.server = handler_path_server
#
# gunicorn logging
configuration.logging.addbranch('gunicorn')
configuration.logging.gunicorn.access = os.path.join(
configuration.paths.server.logs,
'gunicorn', 'access.log')
configuration.logging.gunicorn.error = os.path.join(
configuration.paths.server.logs,
'gunicorn', 'error.log')
def _init_docker(configuration, layout, config):
"""
docker configuration
"""
if 'docker' in config:
configuration.addbranch("docker")
#
# compose project name configuration option
configuration.docker.name = "{project_name}_{project_branch}".format(
project_name=configuration.project.name,
project_branch=configuration.project.branch)
if 'name' in config['docker'] and config['docker']['name']:
configuration.docker.name = config['docker']['name']
#
# host information
if 'host' in config['docker']:
configuration.docker.host = config['docker']['host']
#
# configuration info for docker database
if 'database' in config:
configuration.docker.addbranch("database")
configuration.docker.database.host = \
config['docker']['database']['host']
container_extension = config['docker']['database'].get(
'extension', "")
if container_extension:
container_extension = "_" + container_extension
#
# set default container name value
#
# if the container name config option is set
# then use the name given in the configuration file
configuration.docker.database.container_name = \
"{project_name}_{project_extension}{container_ext}".format(
project_name=configuration.project.name,
project_extension=configuration.project.extension,
container_ext=container_extension
)
if 'name' in config['docker']['database'] and \
config['docker']['database']['name']:
configuration.docker.database.container_name = \
config['docker']['database']['name']
#
# not sure what "service name" is for
configuration.docker.database.service_name = \
"{docker_name}_database".format(
docker_name=configuration.docker.name)
configuration.docker.database.port = \
config['docker']['database']['port']
configuration.docker.database.image = \
config['docker']['database']['image']
configuration.docker.database.addbranch("env")
configuration.docker.database.env.user = \
config['docker']['database']['env']['user']
configuration.docker.database.env.password = \
config['docker']['database']['env']['pass']
configuration.docker.database.env.dbname = \
config['docker']['database']['env']['name']
def _init_overrides(configuration, layout, config):
#
# overrides - this is where we specify what unusual overrides we need to do
# NOTE:
# the purpose of the override code is to allow us to contain project
# specific information outside of the "scripts" directory. One of the
# biggest challenges with setting up multiple projects using the current
# fabric scripts is making the fabric scripts project-independent
#
# I am hoping that by abstracting the override code from the rest of the
# fabric scripts I can achieve that goal
configuration.addbranch("overrides")
configuration.paths.addbranch("overrides")
configuration.paths.overrides.addbranch("templates")
configuration.paths.overrides.templates.root = os.path.join(
configuration.paths.project.root,
layout["paths"]["extras"]["templates"])
configuration.paths.overrides.modules = os.path.join(
configuration.paths.project.root,
layout["paths"]["extras"]["modules"])
# print configuration.paths.overrides.modules
configuration.overrides.addbranch("modules")
configuration.paths.overrides.templates.files = os.path.join(
configuration.paths.overrides.templates.root,
"files")
configuration.paths.overrides.templates.build = os.path.join(
configuration.paths.overrides.templates.root,
"build")
# NOTE:
# this is the code where we append the override path to the current sys
# path. After we do that, we load up the initialization code that is
# specific to this project (if any).
# check to see if overrides is in the configuration
# if not, just return
import sys
sys.path.append(configuration.paths.overrides.modules)
import override
override.initialize(configuration, config, layout, env)
# import utils
# location = "initialize._init_override"
# utils.print_console("beg debug %s" % location)
# print "paths.project.root : %s" \
# % configuration.paths.project.root
# print "paths.overrides.templates.root : %s" \
# % configuration.paths.overrides.templates.root
# print "paths.overrides.modules : %s" \
# % configuration.paths.overrides.modules
# utils.print_console("end debug %s" % location)
@task
def create_local(branch=None):
if branch is None:
configuration = env.config
branch = configuration.project.branch
else:
configuration = get_config(branch)
template_destination = os.path.join(
configuration.paths.django.settings.local,
'{branchname}.py'.format(branchname=branch))
context = dict()
context['project_name'] = configuration.project.name
context['branch'] = branch
context['server_media_static'] = configuration.paths.server.media.static
context['server_media_dynamic'] = configuration.paths.server.media.dynamic
from fabric.contrib.files import upload_template
upload_template(
filename=configuration.templates.django.settings_local,
destination=template_destination,
context=context,
use_jinja=True,
backup=True,
template_dir=configuration.templates.django.path)