from fabric.api import env, local, task from modules.utils import virtualenv from fabric.context_managers import lcd import fabric.operations as fabric_ops from fabric.contrib.files import exists from .utils import loggify, print_run, booleanize from .utils import generate_template_build_path from .utils import generate_template_files_path from .utils import handle_help import os def generate_secret_key(): """ helper to generate django secret key """ import random import string SECRET_KEY = ''.join([random.SystemRandom().choice( "{}{}{}".format( string.ascii_letters, string.digits, '!#$%()*+,-./:;<=>?@[\\]_{}~')) for i in range(50)]) return SECRET_KEY @task def test(args=None): print("debug - testing checkapp(sorl.thumbnail): %s" % check_app("sorl.thumbnail")) @task def commandline(args="", workingdir=None): """ prints out what you need to run on the command line to invoke manage.py """ configuration = env.config if workingdir: commandline = "{djangoroot}/manage.py {args} " \ "--pythonpath='{djangoroot}' " \ "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, djangosettings=configuration.imports.settings,) else: commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}'" \ " --settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, djangosettings=configuration.imports.settings,) print(commandline) @task def manage(args="", workingdir=None): configuration = env.config # changes the working directory to the djangoroot from fabric.context_managers import cd with virtualenv(): if workingdir: with cd(workingdir): fabcommand = "{djangoroot}/manage.py {args} " \ "--pythonpath='{djangoroot}' " \ "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, djangosettings=configuration.imports.settings, ) if env.debug: print("fabcommand: %s" % fabcommand) print("workingdir: %s" % workingdir) else: output = fabric_ops.run( fabcommand, # MAKE SURE THIS IS ALWAYS HERE! shell='/bin/bash') with cd(configuration.paths.django.root): output = fabric_ops.run( "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, djangosettings=configuration.imports.settings, ), # MAKE SURE THIS IS ALWAYS HERE! shell='/bin/bash' ) # fabric.run has the ability to give me back the output return output # NOTE: # there was a major problem using fabric commands of "local" or "prefix" # to work on remote machines, the problem was that for whatever cracked # up reason, fabric would assume I'm using a /bin/sh shell, and /bin/sh # CANNOT run all the commends that /bin/bash can. SO YOU MUST SPECIFY # shell='/bin/bash' in all uses of local! @task def admin(args="help"): configuration = env.config from fabric.context_managers import cd with virtualenv(): with cd(configuration.paths.django.root): fabric_ops.run( "django-admin {args} --pythonpath='{djangoroot}' " "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, djangosettings=configuration.imports.settings, ), # MAKE SURE THIS IS ALWAYS HERE! shell='/bin/bash' ) @task def collectstatic(): """ makes sure the static media directories exist """ configuration = env.config exists(configuration.paths.server.media.static) exists(configuration.paths.server.media.dynamic) manage("collectstatic --noinput") @task def run(args=None): configuration = env.config command = "runserver {host}:{port}".format( host=configuration.server.django.host, port=configuration.server.django.port) output = manage(command) return output @task def startapp(appname='help'): """ wrapper for the django.startapp takes name of app and creates in in code/apps appname - name of app """ configuration = env.config msg_help = """ django.startapp takes one of two values: \thelp - this help message \tappname - the name of the app you want to start """ import sys if handle_help(appname, msg_help, 'help'): sys.exit() command = "startapp {appname}".format( appname=appname) manage(command, workingdir=configuration.paths.django.apps) # with lcd(configuration.paths.django.apps): # manage(command) @task def installed_apps(): """ List the currently installed apps in the settings.py file for this project """ configuration = env.config printecho = "print('\\n')" printcommand = "print('\\n').join([ item for item" \ " in {settings}.INSTALLED_APPS])".format( settings=configuration.imports.settings) command = "python -c \"import {settings}; {printecho};" \ " {printcommand}; {printecho}\"".format( settings=configuration.imports.settings, printecho=printecho, printcommand=printcommand) with lcd(configuration.paths.django.root): local(command) @task def src(): """ locate the django source files in the site-packages directory """ command = """ python -c " import sys sys.path = sys.path[1:] import django print(django.__path__)" """ command = """ python -c "import sys; sys.path=sys.path[1:];""" \ """ import django; print(django.__path__)[0]" """ with virtualenv(): local(command) @task def create_project(): configuration = env.config logger = loggify("django", "create_project") project_path = configuration.paths.django.root project_name = configuration.project.name import os full_project_path = os.path.join(project_path, project_name) django_cmd = \ "django-admin startproject {project_name} {project_path}".format( project_name=configuration.project.name, project_path=project_path) manage_path = "%s/manage.py" % project_path logger.debug("django_root : %s" % configuration.paths.django.root) logger.debug("project_path : %s" % project_path) # I accidentally deleted the code directory, this checks to see if the # project path exists, if not, create it. if not exists(project_path): fabric_ops.run("mkdir -p %s" % project_path) if exists(manage_path): fabric_ops.run("rm %s" % manage_path) if exists(full_project_path): # backup whatever is there fabric_ops.run("mv {project_path}/{project_name}" " {project_path}/{project_name}.old".format( project_name=project_name, project_path=project_path)) with virtualenv(): fabric_ops.run(django_cmd) django_path = "{project_path}/{project_name}".format( project_name=configuration.project.name, project_path=project_path) fabric_ops.run("mkdir %s/_settings" % django_path) fabric_ops.run("touch %s/_settings/__init__.py" % django_path) generate('settings', True) generate('local', True) generate('wsgi', True) def generate_scripts(template_name, make_copy=False): """ this is a function meant to generate django settings files There are a number of different types of django settings files so instead of generating all of them at the same time (sometimes I want the local, sometimes I want the main, etc), I decided to create a function that can look up the type of scripts I want and generate those. The function is meant to be wrapped up in another funciton that will call the type of script I want """ configuration = env.config # make sure to booleanize ALL boolean values! make_copy = booleanize(make_copy) if env.debug: logger = loggify("django", "generate_scripts") project_name = configuration.project.name project_branch = configuration.project.branch project_path = configuration.paths.django.root secret_key = generate_secret_key() files_name = getattr(configuration.templates.django, template_name).src build_name = getattr(configuration.templates.django, template_name).dst build_path = generate_template_build_path('django', template_name) files_path = generate_template_files_path('django') context = dict() context['project_name'] = project_name context['project_branch'] = project_branch context['secret_key'] = secret_key copy_path = "{project_path}/{project_name}".format( project_path=project_path, project_name=project_name) if template_name == 'local': copy_path = "{project_path}/{project_name}/_settings".format( project_path=project_path, project_name=project_name) build_name = "%s.py" % project_branch # generate the local generate scripts generate('local.generated', make_copy) elif template_name == 'local.generated': copy_path = "{project_path}/{project_name}/_settings".format( project_path=project_path, project_name=project_name) build_name = "%s_generated.py" % project_branch # add extra local specfic context variables # NOTE: # if you change project settings, you MUST run generate context['paths_django_templates'] = \ configuration.paths.django.templates context['server_database_backend'] = \ configuration.server.database.backend context['server_database_name'] = configuration.server.database.name context['server_database_user'] = configuration.server.database.user context['server_database_password'] = \ configuration.server.database.password context['server_database_host'] = configuration.server.database.host context['server_database_port'] = configuration.server.database.port context['paths_server_media_static'] = \ configuration.paths.server.media.static context['paths_server_media_dynamic'] = \ configuration.paths.server.media.dynamic context['paths_project_root'] = configuration.paths.project.root context['project_django_allowedhosts'] = \ configuration.project.django.allowedhosts # convenience variable naming, otherwise it's too long to deal with file_debug_handler = configuration.logging.django.handlers.file_debug context['file_debug_handler__name_project'] = \ file_debug_handler.name.project context['file_debug_handler__path_project'] = \ file_debug_handler.path.project context['file_debug_handler__name_server'] = \ file_debug_handler.name.server context['file_debug_handler__path_server'] = \ file_debug_handler.path.server copy_full_path = "{copy_path}/{build_name}".format( copy_path=copy_path, build_name=build_name) copy_cmd = "cp {build_path} {copy_full_path}".format( build_path=build_path, copy_full_path=copy_full_path) backup_cmd = "cp {copy_full_path} " \ "{copy_full_path}.bak".format(copy_full_path=copy_full_path) from .utils import upload_template as utils_upload_template if env.debug: logger.debug("template_name : %s" % template_name) logger.debug("project_branch : %s" % project_branch) logger.debug("project_name : %s" % project_name) logger.debug("build_path : %s" % build_path) logger.debug("files_path : %s" % files_path) logger.debug("files_name : %s" % files_name) logger.debug("copy_path : %s" % copy_path) logger.debug("copy_full_path : %s" % copy_full_path) logger.debug("build_name : %s" % build_name) upload_msg = utils_upload_template( filename=files_name, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=True) logger.debug("upload_msg : %s" % upload_msg) logger.debug("make_copy : %s" % make_copy) logger.debug("copy_cmd : %s" % copy_cmd) logger.debug("backup_cmd : %s" % backup_cmd) else: utils_upload_template( filename=files_name, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=False) if make_copy: if exists(copy_full_path): fabric_ops.run(backup_cmd) fabric_ops.run(copy_cmd) print("\n\n------------------------------") print("project_name : %s" % project_name) print("project_branch : %s" % project_branch) print("project_path : %s" % project_path) print("template_name : %s" % template_name) print("build_path : %s" % build_path) print("files_path : %s" % files_path) print("files_name : %s" % files_name) print("copy_path : %s" % copy_path) print("copy_full_path : %s" % copy_full_path) print("build_name : %s" % build_name) upload_msg = utils_upload_template( filename=files_name, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=True) print("upload_msg : %s" % upload_msg) print("make_copy : %s" % make_copy) print("copy_cmd : %s" % copy_cmd) print("backup_cmd : %s" % backup_cmd) print("------------------------------\n\n") @task def generate(param="help", make_copy=False): """ param can be one of settings, local, wsgi make_copy must be set to True if you want it to actually do anthing """ SCRIPT_LIST = ['settings', 'local', 'local.generated', 'wsgi'] PARAM_LIST = list(SCRIPT_LIST) PARAM_LIST.append('gunicorn') make_copy = booleanize(make_copy) if param: err_msg = None if param == "help": err_msg = """ fab django.generate:param="help",make_copy=False param - file to be generated can be of type: %s make_copy - determines whether or generate copies the generated file into it's practical location (otherwise doesn't do anything) """ % PARAM_LIST elif param not in PARAM_LIST: err_msg = "You asked to generate %s, that value isn't available" \ " possible script values available: %s" % (param, PARAM_LIST) if err_msg: import sys sys.exit(err_msg) print("django:generate make_copy : %s\n" % make_copy) print("django:generate script : %s" % param) if env.debug: print("django:generate script : %s" % param) print("django:generate make_copy : %s\n" % make_copy) else: pass # env.debug does not block the rest of the commands because this # function acts primarily as a wrapper for the following cmomands, in # those fucntion env.debug will be used to decide if anything should # happen or not if not param: # this is where script=None, generate all scripts for scriptkey in SCRIPT_LIST: generate_scripts(scriptkey, make_copy) generate_gunicorn(make_link=make_copy) elif param == 'gunicorn': generate_gunicorn(make_link=make_copy) else: generate_scripts(param, make_copy) def generate_gunicorn(make_link=True): """ create the gunicorn configuration script put it in the build folder and link it to the scripts directory """ configuration = env.config make_link = booleanize(make_link) if env.debug: logger = loggify("django", "generate_gunicorn") files_path = os.path.join( configuration.templates.gunicorn.path.local, 'files') build_path = os.path.join( configuration.templates.gunicorn.path.remote, 'build', configuration.templates.gunicorn.conf.dst) link_path = os.path.join( configuration.paths.server.scripts, configuration.templates.gunicorn.conf.dst ) context = dict() context['host'] = configuration.server.django.host context['port'] = configuration.server.django.port context['user'] = configuration.project.user context['group'] = configuration.project.group context['settings_module'] = configuration.imports.settings context['extended_name'] = configuration.project.extendedname context['logging_access'] = configuration.logging.gunicorn.access context['logging_error'] = configuration.logging.gunicorn.error msg_link_gunicorn = "ln -sf {gunicorn_root} {link_gunicorn}".format( gunicorn_root=build_path, link_gunicorn=link_path) print_run(msg_link_gunicorn) if env.debug: logger.debug("\n") logger.debug("--- in gunicorn ---\n") for key in context.keys(): logger.debug("%s\t: %s" % (key, context[key])) logger.debug('build_path\t: %s' % build_path) logger.debug('files_path\t: %s' % files_path) logger.debug('files config src: %s' % configuration.templates.gunicorn.conf.src) logger.debug('\n%s' % print_run(msg_link_gunicorn)) from .utils import upload_template as utils_upload_template upload_msg = utils_upload_template( filename=configuration.templates.gunicorn.conf.src, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=True) logger.debug("upload_msg : %s" % upload_msg) else: from fabric.contrib.files import upload_template upload_template( filename=configuration.templates.gunicorn.conf.src, destination=build_path, context=context, use_jinja=True, backup=True, template_dir=files_path) if make_link: print("\nlinking the generating gunicorn file in conf to " "the server diretory\n") fabric_ops.run(msg_link_gunicorn) else: print("\nNOTE: not linking the generated gunicorn file" "to the server directory\n") @task def edit(param='help'): """ calls up mvim on the gunicorn and django conf file """ from .maintenance import edit as maintenance_edit configuration = env.config link_path = os.path.join( configuration.paths.server.scripts, configuration.templates.gunicorn.conf.dst ) build_path = os.path.join( configuration.templates.gunicorn.path.remote, 'build', configuration.templates.gunicorn.conf.dst) project_branch = configuration.project.branch project_path = configuration.paths.django.root project_settings_dir = configuration.project.django.settings_folder django_path = "{project_path}/{project_settings_dir}".format( project_path=project_path, project_settings_dir=project_settings_dir ) settings_path = "{django_path}/settings.py".format( django_path=django_path) settings_local_path = "{django_path}/_settings/{project_branch}.py".format( django_path=django_path, project_branch=project_branch) settings_generated_path = "{django_path}/_settings/" \ "{project_branch}_generated.py".format( django_path=django_path, project_branch=project_branch) file_debug_handler = configuration.logging.django.handlers.file_debug # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', # 'settings', 'local'] locations = { 'gunicorn': { 'path': link_path, 'desc': 'gunicorn.conf file', }, 'gunicorn_build': { 'path': build_path, 'desc': "gunicorn.conf file in scripts/conf/gunicorn/build" }, 'gunicorn.log.error': { 'path': configuration.logging.gunicorn.error, 'desc': "gunicorn error log file", }, 'gunicorn.log.access': { 'path': configuration.logging.gunicorn.access, 'desc': "gunicorn error log file", }, 'settings': { 'path': settings_path, 'desc': 'main settings file for django project', }, 'local': { 'path': settings_local_path, 'desc': 'local settings file for django project', }, 'generated': { 'path': settings_generated_path, 'desc': 'generated settings file for django project', }, 'server_log': { 'path': file_debug_handler.path.server, 'desc': 'django server log - handler name: %s' % file_debug_handler.name.server, }, 'project_log': { 'path': file_debug_handler.path.project, 'desc': 'django project log - handler name: %s' % file_debug_handler.name.project, }, } if param in locations.keys(): remote_path = locations[param]['path'] maintenance_edit(remote_path=remote_path) else: # if param == 'help': print(""" "fab django.edit" automates editing files important to django whether locally or remotely to use this you must pass one of the editable locations in as a parameter currently editable locations are: """) for k_loc in locations.keys(): print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) return @task def clearmigrations(appname="help"): if appname == "help": print(""" "fab django.clearmigration:{appname}" clears out all migrations for the specified appname if no appname is given, or if you pass the "help" appnameter in place of an appname then this help message will appear. Note: if your appname is actually "help" you might want to go into this function and change it up a bit! """) return configuration = env.config import os app_path = os.path.join( configuration.paths.django.apps, appname) path_migrations = os.path.join( app_path, 'migrations') path_migrations_old = os.path.join( app_path, 'migrations.old') import fabric if fabric.contrib.files.exists(path_migrations): # get rid of any old migration backups if fabric.contrib.files.exists(path_migrations_old): cmd_rm_migration_old = "rm -Rf %s" % path_migrations_old fabric.operations.run(cmd_rm_migration_old) # move the original migrations folder to migrations.old cmd_migration = "mv %s %s" % ( path_migrations, path_migrations_old) fabric.operations.run(cmd_migration) manage("makemigrations --empty %s" % appname) manage("makemigrations") manage("migrate --fake %s 0002" % appname) @task def makemigrations_empty(param="help"): if param == "help": print("print this help message") return manage("makemigrations --empty %s" % param) @task def fixtures(appname=None, backup=False): """ @appname django app name for this fixture @backup default False, store in server backup dir? """ configuration = env.config # booleanize the backup parameter backup = booleanize(backup) print("debug - appname: %s" % appname) # from fabric.api import * path_data = configuration.paths.django.fixtures path_backup = configuration.paths.server.backups.fixtures print("debug - path_backup: %s" % path_backup) if appname is not None: path_data = os.path.join(path_data, appname) path_backup = os.path.join(path_backup, appname) fixture_name = "%s.json" % appname else: fixture_name = "all.json" if backup: fix_split = os.path.splitext(fixture_name) fixture_name = fix_split[0] + "_backup" + fix_split[1] path_fixture = os.path.join(path_backup, fixture_name) else: path_fixture = os.path.join(path_data, fixture_name) print("debug - path_fixture: %s" % path_fixture) from .utils import ensure_dir ensure_dir(path_data) ensure_dir(path_backup) from fabric.context_managers import hide with hide('running', 'stdout', 'stderr'): # get rid of things like "localhost: out" output = manage('dumpdata %s --indent 2' % appname) # Now go through the output and ignore all the extra # information there. import re # first get rid of everything until the last time # "[localhost] out:" is printed if configuration.project.host == 'localhost': p = re.compile("\[localhost\] out:") for match in p.finditer(output): pass try: pos = match.end() output = output[pos:] except: pass # now find the first occurence of [ on a newline, e.g. # [ # { # "model.auth": 40, # etc. p = re.compile(r"\n\[") m = p.search(output) pos = m.end() - 1 output = output[pos:] from StringIO import StringIO fixture_iostring = StringIO(output) fabric_ops.put(fixture_iostring, path_fixture) @task def loaddata(param=None): """ loads fixture data to the specified app from extras/data """ configuration = env.config if param is None: print("you must specify an appname to load the fixture to") return else: appname = param path_data = configuration.paths.django.fixtures if appname is not None: path_data = os.path.join(path_data, appname) path_fixture = os.path.join(path_data, "%s.json" % appname) else: path_fixture = os.path.join(path_data, "all.json") manage("loaddata %s" % path_fixture) @task def check_app(appname): """ check if is installed in settings @appname: the name of the app being checked for returns True/False """ from fabric.context_managers import hide with hide('running', 'stderr'): output = manage('listapps') import re p = re.compile(appname) match = p.search(output) if match: return True else: return False