diff --git a/bin/deploy_meta.py b/bin/deploy_meta.py index 01a42b1..03d02f3 100644 --- a/bin/deploy_meta.py +++ b/bin/deploy_meta.py @@ -1,7 +1,10 @@ import sys import yaml import os -from PyQt4 import QtGui, QtCore +from PyQt5.QtWidgets import QMainWindow, QLineEdit, QLabel, QAction +from PyQt5.QtWidgets import QFileDialog, QGridLayout, QWidget, QApplication +from PyQt5.QtWidgets import QMessageBox +from PyQt5.QtCore import QVariant class DictQuery(dict): @@ -74,7 +77,7 @@ def nested_set(dic, keys, value): dic[keys[-1]] = value -class DeployMeta(QtGui.QMainWindow): +class DeployMeta(QMainWindow): PATH_META = "../../../meta/project" PATH_TEMPLATES = "../share/templates/meta" @@ -263,17 +266,17 @@ class DeployMeta(QtGui.QMainWindow): def load_config(self, configname): if configname not in self.CONFIG_TYPES.keys(): - print "\nerror, load_config was called with parameter: {confname}," \ - "which is not a legitimate value in CONFIG TYPES." \ - "\nLegitimate values are {configvalues}".format( - confname=configname, - configvalues=self.CONFIG_TYPES.keys()) + print("\nerror, load_config was called with parameter: {confname}," + "which is not a legitimate value in CONFIG TYPES." + "\nLegitimate values are {configvalues}".format( + confname=configname, + configvalues=self.CONFIG_TYPES.keys())) sys.exit() path_config_full = os.path.join( self.path_templates, self.CONFIG_TYPES[configname]) - configuration_file = yaml.load(file(path_config_full, 'r')) + configuration_file = yaml.load(open(path_config_full, 'r')) return configuration_file def store_config(self): @@ -322,8 +325,8 @@ class DeployMeta(QtGui.QMainWindow): title = self.widgets[key]['title'] - label = QtGui.QLabel(title) - field = QtGui.QLineEdit() + label = QLabel(title) + field = QLineEdit() grid.addWidget(label, row, 0) grid.addWidget(field, row, 1) @@ -337,19 +340,23 @@ class DeployMeta(QtGui.QMainWindow): desc = "Load Configuration %s" % title - loadAction = QtGui.QAction(menuname, self) + loadAction = QAction(menuname, self) loadAction.setShortcut(shortcut) loadAction.setStatusTip(desc) loadAction.triggered.connect(self.action_loadconfig) - variant = QtCore.QVariant(key) + variant = QVariant(key) loadAction.setData(variant) return loadAction def action_loadconfig(self): sender = self.sender() - self.currentbranch = sender.data().toString().__str__() + + if type(sender.data()) is str: + self.currentbranch = sender.data() + else: + self.currentbranch = sender.data().toString().__str__() self.config_data = self.load_config(self.currentbranch) @@ -392,23 +399,22 @@ class DeployMeta(QtGui.QMainWindow): # # get the path value from the dialog box - path_newconfig = QtGui.QFileDialog.getSaveFileName( + path_newconfig = QFileDialog.getSaveFileName( self, dialog_title, path_default_save, - selectedFilter='*.yml') + filter='*.yml') # # if the user hit 'cancel' path_newconfig will be empty if path_newconfig: - stream = file(path_newconfig, 'w') + stream = open(path_newconfig[0], 'w') yaml.dump(self.config_data, stream) else: # # display message warning no configuration has been loaded - from PyQt4.QtGui import QMessageBox msg = QMessageBox() msg.setIcon(QMessageBox.Warning) msg.setText('Save Error') @@ -417,18 +423,18 @@ class DeployMeta(QtGui.QMainWindow): " default config types") retval = msg.exec_() - print "value of qmessagebox in action_save: %s" % retval + print("value of qmessagebox in action_save: %s" % retval) def setupMenu(self): menubar = self.menuBar() menubar.setNativeMenuBar(False) - exitAction = QtGui.QAction('&Exit', self) + exitAction = QAction('&Exit', self) exitAction.setShortcut('Ctrl+Q') exitAction.setStatusTip("Exit application") - exitAction.triggered.connect(QtGui.qApp.quit) + exitAction.triggered.connect(self.close) - saveAction = QtGui.QAction('&Gore', self) + saveAction = QAction('&Gore', self) saveAction.setShortcut('Ctrl+T') saveAction.setStatusTip("Save Config File") saveAction.triggered.connect(self.action_save_config) @@ -446,7 +452,7 @@ class DeployMeta(QtGui.QMainWindow): self.setupMenu() - grid = QtGui.QGridLayout() + grid = QGridLayout() grid.setSpacing(10) self.add_widgetrow('PROJECT_NAME', grid) @@ -466,7 +472,7 @@ class DeployMeta(QtGui.QMainWindow): self.add_widgetrow('DJANGO_PORT', grid) self.add_widgetrow('NGINX_PORT', grid) - central = QtGui.QWidget() + central = QWidget() central.setLayout(grid) self.setCentralWidget(central) @@ -477,12 +483,12 @@ class DeployMeta(QtGui.QMainWindow): def main(): - app = QtGui.QApplication(sys.argv) + app = QApplication(sys.argv) app.setStyle("cleanlooks") ex = DeployMeta() sys.exit(app.exec_()) - print ex + print(ex) if __name__ == '__main__': diff --git a/bin/test.py b/bin/test.py index efb3043..74b0592 100644 --- a/bin/test.py +++ b/bin/test.py @@ -10,8 +10,8 @@ parser.add_argument('--file2', default='test.yml') def findDiff(d1, d2, path=""): for k in d1.keys(): if not (k in d2): - print path, ":" - print k + " as key not in d2", "\n" + print(path, ":") + print(k + " as key not in d2", "\n") else: if type(d1[k]) is dict: if path == "": @@ -21,9 +21,9 @@ def findDiff(d1, d2, path=""): findDiff(d1[k], d2[k], path) else: if d1[k] != d2[k]: - print path, ":" - print " - ", k, " : ", d1[k] - print " + ", k, " : ", d2[k] + print(path, ":") + print(" - ", k, " : ", d1[k]) + print(" + ", k, " : ", d2[k]) def compare_dictionaries(dict_1, dict_2, dict_1_name, dict_2_name, path=""): @@ -85,6 +85,6 @@ def main(): yaml_test = ruamel.yaml.load(file(path_test, 'r')) a = compare_dictionaries(yaml_main, yaml_test, 'main', 'test') - print a + print(a) main() diff --git a/fabfile.py b/fabfile.py index f05fe7c..919d05d 100644 --- a/fabfile.py +++ b/fabfile.py @@ -17,7 +17,7 @@ branch = "development" def setup(): - print "setup fabric configuration files" + print("setup fabric configuration files") def all(): @@ -48,7 +48,7 @@ def stage(): global branch branch = "staging" - print "in staging" + print("in staging") all() @@ -62,7 +62,7 @@ def prod(): global branch branch = "production" - print "in production" + print("in production") all() diff --git a/modules/conf_setup.py b/modules/conf_setup.py index d402443..e1db674 100644 --- a/modules/conf_setup.py +++ b/modules/conf_setup.py @@ -3,7 +3,7 @@ from fabric.operations import run, put from fabric.api import env, task import os -from maintenance import _get_configuration_path, load_configuration +from .maintenance import _get_configuration_path, load_configuration @task @@ -11,7 +11,7 @@ def deploy(param=None): param_list = ['conf', 'readmes'] if not param: - print "this requires input param must be one of %s" % param_list + print("this requires input param must be one of %s" % param_list) import sys sys.exit() @@ -26,7 +26,7 @@ def backup(param=None): param_list = ['conf', 'meta', 'readmes'] if not param: - print "this requires input param must be one of %s" % param_list + print("this requires input param must be one of %s" % param_list) import sys sys.exit() @@ -257,3 +257,57 @@ def copy_directories(source_path, dest_path): fpath = os.path.join(source_path, fname) put(fpath, dest_path) + + +@task +def edit(param='help'): + """ + calls up mvim on the yaml project configuration files + """ + + from .maintenance import edit as maintenance_edit + from .maintenance import _get_configuration_path + + locations = { + 'development': { + 'path': _get_configuration_path("config", "development"), + 'desc': 'development project configuration file', + }, + + 'staging': { + 'path': _get_configuration_path("config", "staging"), + 'desc': 'staging project configuration file', + }, + + 'production': { + 'path': _get_configuration_path("config", "production"), + 'desc': 'production project configuration file', + }, + } + + if param in locations.keys(): + # it is unnecessary to give the host_string to edit + # because the host_string is determined when we call fab + # ie. "fab devel whatever" or "fab stage whatever" + # we want to edit all files on localhost. b/c we only call + # fab FROM localhost + + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab configuration.edit" automates opening up and editing project + configuration files + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return diff --git a/modules/configuration_setup.py b/modules/configuration_setup.py index 22dd99a..be3aef6 100644 --- a/modules/configuration_setup.py +++ b/modules/configuration_setup.py @@ -31,7 +31,7 @@ Error. maintenance.exists_dir_sub takes a 'section' parameter value. Valid options include:""" % section for key in configuration.templates.keys(): - print " %s" % key + print(" %s" % key) print """ Please run the command again, but this time enter a valid section value. @@ -54,8 +54,8 @@ def exists_dir_top(): # located files, since those files get rsync'd anyway. if env.debug: - print "maintenance.exists_dir_top -- checking for " \ - "directory:\n\t%s\n" % configuration.paths.conf.local + print("maintenance.exists_dir_top -- checking for " + "directory:\n\t%s\n" % configuration.paths.conf.local) return exists(configuration.paths.conf.local) diff --git a/modules/database.py b/modules/database.py index ecda262..1a5ee7d 100644 --- a/modules/database.py +++ b/modules/database.py @@ -1,8 +1,8 @@ from fabric.api import env, task # # from jinja2 import Environment import os -from utils import upload_template as utils_upload_template -from utils import loggify, print_console, booleanize +from .utils import upload_template as utils_upload_template +from .utils import loggify, print_console, booleanize # from utils import prompt_continue from getpass import getpass import fabric.operations as fabric_ops @@ -78,7 +78,7 @@ def generate_sql(script_name): # with open(build_path, "w") as output: # output.write(rendered) - print NOTE + print(NOTE) def execute_sql(script_name, add_dbname=True, is_admin=False): @@ -121,7 +121,7 @@ def execute_sql(script_name, add_dbname=True, is_admin=False): else: run_database_command(psql_command, user) - print NOTE + print(NOTE) @task @@ -138,7 +138,7 @@ def generate(): @task def clear_scripts(): - print "this does nothing, the code isn't here" + print("this does nothing, the code isn't here") @task @@ -200,11 +200,10 @@ def commandline(dbuser='default'): host = configuration.server.database.host port = configuration.server.database.port - print "debug -- dbuser: %s" % dbuser + print("debug -- dbuser: %s" % dbuser) if dbuser == 'admin': user = configuration.server.database.admin.user - db_name = configuration.server.database.admin.db_name elif dbuser == 'default': user = configuration.server.database.user @@ -222,7 +221,7 @@ def commandline(dbuser='default'): user=user ) - print "debug -- cmd_commandline: %s" % cmd_commandline + print("debug -- cmd_commandline: %s" % cmd_commandline) run_database_command(cmd_commandline, user) @@ -270,7 +269,7 @@ def backup(dbuser='default', backup_file=None, branch=None, timefilename = os.path.join( configuration.paths.server.backups.database, backup_file_time) - print "debug timefilename -- %s" % timefilename + print("debug timefilename -- %s" % timefilename) dumpfilename = os.path.join( configuration.paths.server.backups.database, @@ -278,7 +277,7 @@ def backup(dbuser='default', backup_file=None, branch=None, datadump_only = booleanize(datadump_only) - print "dumpfilename: %s" % dumpfilename + print("dumpfilename: %s" % dumpfilename) if not datadump_only: @@ -307,7 +306,7 @@ def backup(dbuser='default', backup_file=None, branch=None, hide_output = False - print "cmd_pg_dump: %s" % cmd_pg_dump + print("cmd_pg_dump: %s" % cmd_pg_dump) output = run_database_command(cmd_pg_dump, user, hide=hide_output) @@ -349,7 +348,7 @@ def restore(dbuser='default', backup_file=None, reinitialize=True): timefilename = os.path.join( configuration.paths.server.backups.database, backup_file_time) - print "debug -- timefilename: %s" % timefilename + print("debug -- timefilename: %s" % timefilename) dumpfilename = os.path.join( configuration.paths.server.backups.database, @@ -390,26 +389,26 @@ def sync(src): configuration_src = initialize.environment(branch_src) configuration_dst = configuration - print "branch_src: %s" % configuration_src.project.branch - print "branch_dst: %s" % configuration_dst.project.branch + print("branch_src: %s" % configuration_src.project.branch) + print("branch_dst: %s" % configuration_dst.project.branch) # backup all files with names that wont interfere output_name_dst = "output_sync_dst.sql" backup_name_dst = "backup_sync_dst.sql" backup_name_src = "backup_sync_src.sql" - print "output_src = backup(" - print "\tbackup_file=%s," % backup_name_src - print "\tbranch=%s)" % configuration_src.project.branch + print("output_src = backup(") + print("\tbackup_file=%s," % backup_name_src) + print("\tbranch=%s)" % configuration_src.project.branch) # dump the source database to our {destination}/backups/database - print "backup up src: %s" % configuration_src.project.branch + print("backup up src: %s" % configuration_src.project.branch) output_src = backup( backup_file=backup_name_src, branch=configuration_src.project.branch) - print "backing up dst: %s" % configuration_dst.project.branch + print("backing up dst: %s" % configuration_dst.project.branch) backup( backup_file=backup_name_dst, branch=configuration_dst.project.branch) @@ -456,6 +455,15 @@ def run_database_command(cmd_string, username, hide=False): cmd_string=cmd_string, ) + if env.debug: + + # print out the full command and return + + logger = loggify('database', 'run_database_command') + logger.debug("run_database_command = %s" % cmd_full) + + return + # NOTE # DO NOT USE "with hide" # one of the commands is to access to psql commandline @@ -492,8 +500,8 @@ def get_template_path(script_name, script_type): 'files', file_template) else: - print "Error, you passed the variable %s, must pass" \ - "either 'build' or 'template'" % script_type + print("Error, you passed the variable %s, must pass" + "either 'build' or 'template'" % script_type) import sys sys.exit() @@ -554,7 +562,7 @@ def edit(param='help'): remote_path = locations[param]['path'] maintenance_edit(remote_path=remote_path) else: - print """ + print(""" "fab database.edit" automates editing files important to django whether locally or remotely @@ -562,10 +570,11 @@ def edit(param='help'): parameter currently editable locations are: - """ + """) for k_loc in locations.keys(): - print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + print("\t{0: <20} - {1}".format( + k_loc, locations[k_loc]['desc'])) return diff --git a/modules/deploy.py b/modules/deploy.py index 5acd042..98574c0 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -1,19 +1,39 @@ from fabric.api import env, task -from fabric.operations import run, local +from fabric.operations import run +from fabric.operations import local # from fabric.contrib.files import upload_template # from utils import print_run -from utils import virtualenv_source, booleanize, loggify -from utils import print_console +from .utils import virtualenv_source, booleanize, loggify +from .utils import print_console +from .utils import link_create -from pip import setup_virtualenv -from pip import bootstrap_pip +from .nginx import upload as upload_nginx +from .nginx import remove as nginx_remove +from .supervisor import upload as upload_supervisor +from .supervisor import remove as supervisor_remove + + +from .pip import setup_virtualenv +from .pip import bootstrap_pip +from .pip import install as pip_install + +from .django import create_project +from .django import generate as django_generate +from .django import collectstatic as django_collectstatic + +from . import initialize import os - import logging +# NOTE: +# we have a bunch of imports scattered through the func definitions +# this is done b/c otherwise fabric will consider the modules we are +# importing from to be a SUB-MODULE of deploy. So leave them where +# they are + @task def setup_rootpath(): @@ -59,8 +79,7 @@ def setup_rootpath(): @task def bootstrap(): - import database as db - + from . import docker configuration = env.config if env.debug: @@ -81,7 +100,6 @@ def bootstrap(): bootstrap_pip() # create the django project - from django import create_project create_project() # @@ -108,8 +126,6 @@ def bootstrap(): # links before creating them, otherwise you get really weird errors # where the a link is recreated within the destination link - from utils import link_create - if env.debug: logger.debug("virtualenv.root : %s" % configuration.virtualenv.paths.root) @@ -153,10 +169,6 @@ def bootstrap(): # create and link the scripts that manage the server # e.g. nginx, supervisor, gunicorn - from nginx import upload as upload_nginx - from supervisor import upload as upload_supervisor - from django import generate as django_generate - print_console("creating gunicorn script") django_generate('gunicorn', True) django_generate('local', True) @@ -170,8 +182,6 @@ def bootstrap(): # # instantiate docker containers if any - import docker - print_console("check to see if docker containers are used") if hasattr(configuration, "docker"): @@ -185,6 +195,144 @@ def bootstrap(): # # create and initialize the database + from . import database as db + + print_console("in db.generate") + db.generate() + + print_console("in db.init") + db.init() + + +@task +def bootstrap_part1(): + # import database as db + + # configuration = env.config + + # if env.debug: + # logger = loggify('deploy', 'bootstrap_part1') + + # + # not doing a full sync, because we have to set up the rootpath, + # virtualenv, files, dir structure, etc. This means we aren't + # going to upload gunicorn and supervisor until after we've done + # everything else at the end of the bootstrapping process + + sync(full=False) + + # continue setting up the rootpath and virtualenv + setup_rootpath() + + setup_virtualenv() + + +@task +def bootstrap_part2(): + from . import docker + configuration = env.config + + if env.debug: + logger = loggify('deploy', 'bootstrap') + + # create the django project + create_project() + + # + # link virtualenv to rootpath/private/virtualenv + + src_virtual = configuration.virtualenv.paths.root + dst_virtual = configuration.paths.server.virtual + + # + # link templates to rootpath/private/templates + + src_templates = configuration.paths.django.templates + dst_templates = configuration.paths.server.django.templates + + # + # link the django code in the project directory to the appropriate location + # in the rootpath directory + + src_code = configuration.paths.django.root + dst_code = configuration.paths.server.django.code + + # + # I corrected the linking code so that it deletes already existing + # links before creating them, otherwise you get really weird errors + # where the a link is recreated within the destination link + + if env.debug: + logger.debug("virtualenv.root : %s" + % configuration.virtualenv.paths.root) + logger.debug("virtualenv.bin : %s\n" % + configuration.virtualenv.paths.bin) + + logger.debug("paths.server\n") + + logger.debug(" - root\t: %s" % configuration.paths.server.root) + + logger.debug(" - media\t: %s" % + configuration.paths.server.media.static) + + logger.debug(" - virtual\t: %s" % configuration.paths.server.virtual) + + logger.debug(" - django.code\t: %s\n" % + configuration.paths.server.django.code) + + logger.debug("django templates : %s" % + configuration.paths.django.templates) + + logger.debug("django root : %s" % + configuration.paths.django.root) + + logger.debug("django settings : %s" % + configuration.paths.django.settings.root) + + logger.debug("django local : %s" % + configuration.paths.django.settings.local) + + logger.debug(link_create(src_virtual, dst_virtual, debug=True)) + logger.debug(link_create(src_templates, dst_templates, debug=True)) + logger.debug(link_create(src_code, dst_code, debug=True)) + + else: + link_create(src_virtual, dst_virtual) + link_create(src_templates, dst_templates) + link_create(src_code, dst_code) + + # + # create and link the scripts that manage the server + # e.g. nginx, supervisor, gunicorn + + print_console("creating gunicorn script") + django_generate('gunicorn', True) + django_generate('local', True) + + print_console("creating supervisor script") + upload_supervisor() + + print_console("creating nginx script") + upload_nginx() + + # + # instantiate docker containers if any + + print_console("check to see if docker containers are used") + + if hasattr(configuration, "docker"): + print_console("generating docker configuration file") + docker.generate() + + print_console("creating and starting docker container") + docker.create() + else: + print_console("no docker containers are being used. pass") + + # + # create and initialize the database + + from . import database as db print_console("in db.generate") db.generate() @@ -229,34 +377,34 @@ def sync(full=True, extras=False): command_mkdir_remote = "mkdir -p {remote}".format(remote=remote_dir) excludeitems = (".git", "*.swp", "*.swo", ".DS_Store", "*.pyc", "*.bak", - "build/*", "/extras", "/opt") + "/usr/etc/*/build/*", "/extras", "/opt") if env.debug: - print "\n%s debug: %s" % (debug_prefix, env.debug) + print("\n%s debug: %s" % (debug_prefix, env.debug)) - print "\n%s project.name %s" \ - % (debug_prefix, configuration.project.name) + print("\n%s project.name %s" % + (debug_prefix, configuration.project.name)) - print "%s project.branch %s" \ - % (debug_prefix, configuration.project.branch) + print("%s project.branch %s" % + (debug_prefix, configuration.project.branch)) - print "%s path.project %s" \ - % (debug_prefix, configuration.paths.project.root) + print("%s path.project %s" % + (debug_prefix, configuration.paths.project.root)) - print "\n%s run(%s)" % (debug_prefix, command_mkdir_remote) + print("\n%s run(%s)" % (debug_prefix, command_mkdir_remote)) - print "\n{debug_prefix} rsync_project(\n\tremote_dir={remote_dir}," \ - "\n\tlocal_dir={local_dir},\n\texclude={excludeitems})".format( - debug_prefix=debug_prefix, - remote_dir=remote_dir, - local_dir=local_dir, - excludeitems=excludeitems) + print("\n{debug_prefix} rsync_project(\n\tremote_dir={remote_dir}," + "\n\tlocal_dir={local_dir},\n\texclude={excludeitems})".format( + debug_prefix=debug_prefix, + remote_dir=remote_dir, + local_dir=local_dir, + excludeitems=excludeitems)) - # print "\n%s override: %s " \ - # % (debug_prefix, configuration.overrides.keys()) + # print("\n%s override: %s " % + # (debug_prefix, configuration.overrides.keys())) - # print "%s has overrides? %s" \ - # % (debug_prefix, (len(configuration.overrides.keys()) > 0)) + # print("%s has overrides? %s" % + # (debug_prefix, (len(configuration.overrides.keys()) > 0))) else: @@ -264,7 +412,7 @@ def sync(full=True, extras=False): # either SYNCING TO OR FROM, then just exit at this point if configuration.project.branch == "development": - print """ + print(""" ------------------------------------ NOTE: this is a hack for the function deploy.sync() @@ -289,15 +437,15 @@ def sync(full=True, extras=False): Right after this message gets printed, the sync function is told to "return" without any arguments ------------------------------------\n - """ + """) # # exit the function without any arguments return - # print "remote_dir: %s" % remote_dir - # print "local_dir: %s" % local_dir + # print("remote_dir: %s" % remote_dir) + # print("local_dir: %s" % local_dir) # import sys # sys.exit() @@ -307,7 +455,6 @@ def sync(full=True, extras=False): exclude=excludeitems) if full: - from pip import install as pip_install pip_install('--all') # overrides() @@ -317,15 +464,10 @@ def sync(full=True, extras=False): # NOTE: if using full synch if full: - from nginx import upload as upload_nginx - from supervisor import upload as upload_supervisor - from django import generate as django_generate - upload_supervisor() django_generate('gunicorn', True) upload_nginx() - from django import collectstatic as django_collectstatic django_collectstatic() @@ -338,9 +480,7 @@ def media(source_branch): """ configuration = env.config - import initialize - - print "project.branch: %s" % configuration.project.branch + print("project.branch: %s" % configuration.project.branch) # # if we use initialize.environemnt, we will overwrite all project @@ -349,13 +489,14 @@ def media(source_branch): configuration_src = initialize.get_config(source_branch) configuration_dst = configuration - print "branch_src: %s" % configuration_src.project.branch - print "branch_dst: %s" % configuration_dst.project.branch + print("branch_src: %s" % configuration_src.project.branch) + print("branch_dst: %s" % configuration_dst.project.branch) - print "src - server_media_dynamic: %s" % \ - configuration_src.paths.server.media.dynamic - print "dst - server_media_dynamic: %s" % \ - configuration_dst.paths.server.media.dynamic + print("src - server_media_dynamic: %s" % + configuration_src.paths.server.media.dynamic) + + print("dst - server_media_dynamic: %s" % + configuration_dst.paths.server.media.dynamic) # add a trailing slash to the directories dynamic_src = "%s/" % configuration_src.paths.server.media.dynamic @@ -386,7 +527,7 @@ def media(source_branch): path_dst=dynamic_dst, ) - print cmd_rsync + print(cmd_rsync) upstream = True remote_dir = dynamic_dst @@ -402,30 +543,30 @@ def media(source_branch): # # rsync relies on env.host_string to determine what the username and # host is - # print "before: %s" % env.host_string + # print("before: %s" % env.host_string) # configuration_dst = initialize.environment(source_branch) - # print "after: %s" % env.host_string + # print("after: %s" % env.host_string) # import sys # sys.exit() - print "upstream: %s" % upstream - print "remote_dir: %s" % remote_dir - print "local_dir: %s" % local_dir - print "\ncopy from {src} to {dst}\n".format( + print("upstream: %s" % upstream) + print("remote_dir: %s" % remote_dir) + print("local_dir: %s" % local_dir) + print("\ncopy from {src} to {dst}\n".format( src=configuration_src.project.host, - dst=configuration_dst.project.host) + dst=configuration_dst.project.host)) elif configuration_dst.project.host == "localhost": remote_dir = dynamic_src local_dir = dynamic_dst upstream = False - print "upstream: %s" % upstream - print "remote_dir: %s" % remote_dir - print "local_dir: %s" % local_dir - print "\ncopy from {src} to {dst}\n".format( + print("upstream: %s" % upstream) + print("remote_dir: %s" % remote_dir) + print("local_dir: %s" % local_dir) + print("\ncopy from {src} to {dst}\n".format( src=configuration_src.project.host, - dst=configuration_dst.project.host) + dst=configuration_dst.project.host)) else: print_console("no moving media files from staging to production") return @@ -436,7 +577,7 @@ def media(source_branch): local_dir=local_dir, upstream=upstream) - print cmd_msg + print(cmd_msg) from fabric.contrib.project import rsync_project rsync_project(remote_dir=remote_dir, @@ -451,9 +592,9 @@ def test(): projectpath = configuration.paths.project.root local_dir = configuration.paths.project.local + "/" - print hasattr(configuration, 'docker') - print "project path : %s" % projectpath - print "local dir : %s" % local_dir + print(hasattr(configuration, 'docker')) + print("project path : %s" % projectpath) + print("local dir : %s" % local_dir) @task @@ -470,7 +611,7 @@ def remove(full=True): import sys if env.branch == "development" and not env.debug and full: - print """ + print(""" ------------------------------ WARNING: @@ -481,7 +622,7 @@ def remove(full=True): Exiting NOW. -------------------------------- - """ + """) sys.exit() # @@ -546,14 +687,11 @@ def remove(full=True): # because otherwise they interfere with this modules version # of "remove" - from nginx import remove as nginx_remove - from supervisor import remove as supervisor_remove - nginx_remove() supervisor_remove() - from database import drop_all as db_drop_all - db_drop_all() + from . import database as db + db.drop_all() # # check to see if the parent directory contains anything else @@ -582,6 +720,9 @@ def remove(full=True): def ssh(): configuration = env.config + if env.debug: + logger = loggify('deploy', 'ssh') + if configuration.project.ssh: cmd_ssh = "ssh {user}@{host} -i {sshkey}".format( user=configuration.project.user, @@ -592,4 +733,61 @@ def ssh(): user=configuration.project.user, host=configuration.project.host) - local(cmd_ssh) + if env.debug: + logger.debug("ssh command: %s" % cmd_ssh) + else: + local(cmd_ssh) + + +@task +def edit(param='help', branch=None): + """ + calls up mvim on the configuration files for the branch + """ + + from .maintenance import edit as maintenance_edit + from .maintenance import _get_configuration_path + + configuration = env.config + + if not branch: + project_branch = configuration.project.branch + else: + project_branch = branch + + meta_path = _get_configuration_path('config', project_branch) + layout_path = _get_configuration_path('layout', project_branch) + + # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', + # 'settings', 'local'] + locations = { + 'meta': { + 'path': meta_path, + 'desc': 'meta configuration file', + }, + + 'layout': { + 'path': layout_path, + 'desc': 'layout configuration file', + }, + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab deploy.edit" automates editing branch configuration files + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return diff --git a/modules/django.py b/modules/django.py index 3a08ebd..f9e1208 100644 --- a/modules/django.py +++ b/modules/django.py @@ -5,9 +5,10 @@ import fabric.operations as fabric_ops from fabric.contrib.files import exists -from utils import loggify, print_run, booleanize -from utils import generate_template_build_path -from utils import generate_template_files_path +from .utils import loggify, print_run, booleanize +from .utils import generate_template_build_path +from .utils import generate_template_files_path +from .utils import handle_help import os @@ -31,48 +32,81 @@ def generate_secret_key(): @task def test(args=None): - print "debug - testing checkapp(sorl.thumbnail): %s" % \ - check_app("sorl.thumbnail") + print("debug - testing checkapp(sorl.thumbnail): %s" % + check_app("sorl.thumbnail")) @task -def commandline(args=""): +def commandline(args="", workingdir=None): """ prints out what you need to run on the command line to invoke manage.py """ configuration = env.config - commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " \ - "--settings={djangosettings}".format( - djangoroot=configuration.paths.django.root, - args=args, - djangosettings=configuration.imports.settings,) + if workingdir: + commandline = "{djangoroot}/manage.py {args} " \ + "--pythonpath='{djangoroot}' " \ + "--settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings,) + else: + commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}'" \ + " --settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings,) - print commandline + print(commandline) @task -def manage(args=""): +def manage(args="", workingdir=None): configuration = env.config # changes the working directory to the djangoroot from fabric.context_managers import cd with virtualenv(): + if workingdir: + with cd(workingdir): + fabcommand = "{djangoroot}/manage.py {args} " \ + "--pythonpath='{djangoroot}' " \ + "--settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings, + ) + + if env.debug: + print("fabcommand: %s" % fabcommand) + print("workingdir: %s" % workingdir) + else: + output = fabric_ops.run( + fabcommand, + # MAKE SURE THIS IS ALWAYS HERE! + shell='/bin/bash') + with cd(configuration.paths.django.root): - output = fabric_ops.run( - "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " + cmd = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " \ "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, - djangosettings=configuration.imports.settings, - ), - # MAKE SURE THIS IS ALWAYS HERE! - shell='/bin/bash' - ) + djangosettings=configuration.imports.settings,) - # fabric.run has the ability to give me back the output - return output + if env.debug: + print("command: with cd(%s)" % configuration.paths.django.root) + print("command: fabric_ops.run(%s)" % cmd) + else: + + output = fabric_ops.run( + cmd, + # MAKE SURE THIS IS ALWAYS HERE! + shell='/bin/bash' + ) + + # fabric.run has the ability to give me back the output + return output # NOTE: # there was a major problem using fabric commands of "local" or "prefix" @@ -129,28 +163,32 @@ def run(args=None): @task -def startapp(args): +def startapp(appname='help'): """ wrapper for the django.startapp takes name of app and creates in in code/apps - args - name of app + appname - name of app """ configuration = env.config - destination = os.path.join(configuration.paths.django.apps, args) + msg_help = """ + django.startapp takes one of two values: + \thelp - this help message + \tappname - the name of the app you want to start + """ - cmd_mkdir = "mkdir {destination}".format( - destination=destination) + import sys - command = "startapp {appname} {destination}".format( - appname=args, - destination=destination) + if handle_help(appname, msg_help, 'help'): + sys.exit() - fabric_ops.run(cmd_mkdir) + command = "startapp {appname}".format( + appname=appname) + + manage(command, workingdir=configuration.paths.django.apps) - manage(command) # with lcd(configuration.paths.django.apps): # manage(command) @@ -162,8 +200,8 @@ def installed_apps(): """ configuration = env.config - printecho = "print '\\n'" - printcommand = "print '\\n'.join([ item for item" \ + printecho = "print('\\n')" + printcommand = "print('\\n').join([ item for item" \ " in {settings}.INSTALLED_APPS])".format( settings=configuration.imports.settings) @@ -291,11 +329,67 @@ def generate_scripts(template_name, make_copy=False): project_name=project_name) if template_name == 'local': + copy_path = "{project_path}/{project_name}/_settings".format( project_path=project_path, project_name=project_name) build_name = "%s.py" % project_branch + # generate the local generate scripts + generate('local.generated', make_copy) + + elif template_name == 'local.generated': + + copy_path = "{project_path}/{project_name}/_settings".format( + project_path=project_path, + project_name=project_name) + build_name = "%s_generated.py" % project_branch + + # add extra local specfic context variables + # NOTE: + # if you change project settings, you MUST run generate + + context['paths_django_templates'] = \ + configuration.paths.django.templates + + context['server_database_backend'] = \ + configuration.server.database.backend + + context['server_database_name'] = configuration.server.database.name + context['server_database_user'] = configuration.server.database.user + + context['server_database_password'] = \ + configuration.server.database.password + + context['server_database_host'] = configuration.server.database.host + context['server_database_port'] = configuration.server.database.port + + context['paths_server_media_static'] = \ + configuration.paths.server.media.static + + context['paths_server_media_dynamic'] = \ + configuration.paths.server.media.dynamic + + context['paths_project_root'] = configuration.paths.project.root + + context['project_django_allowedhosts'] = \ + configuration.project.django.allowedhosts + + # convenience variable naming, otherwise it's too long to deal with + file_debug_handler = configuration.logging.django.handlers.file_debug + + context['file_debug_handler__name_project'] = \ + file_debug_handler.name.project + + context['file_debug_handler__path_project'] = \ + file_debug_handler.path.project + + context['file_debug_handler__name_server'] = \ + file_debug_handler.name.server + + context['file_debug_handler__path_server'] = \ + file_debug_handler.path.server + copy_full_path = "{copy_path}/{build_name}".format( copy_path=copy_path, build_name=build_name) @@ -305,7 +399,7 @@ def generate_scripts(template_name, make_copy=False): backup_cmd = "cp {copy_full_path} " \ "{copy_full_path}.bak".format(copy_full_path=copy_full_path) - from utils import upload_template as utils_upload_template + from .utils import upload_template as utils_upload_template if env.debug: logger.debug("template_name : %s" % template_name) @@ -342,51 +436,73 @@ def generate_scripts(template_name, make_copy=False): fabric_ops.run(copy_cmd) - print "\n\n------------------------------" - print "project_name : %s" % project_name - print "project_branch : %s" % project_branch - print "project_path : %s" % project_path - print "template_name : %s" % template_name - print "build_path : %s" % build_path - print "files_path : %s" % files_path - print "files_name : %s" % files_name + print("\n\n------------------------------") + print("project_name : %s" % project_name) + print("project_branch : %s" % project_branch) + print("project_path : %s" % project_path) + print("template_name : %s" % template_name) + print("build_path : %s" % build_path) + print("files_path : %s" % files_path) + print("files_name : %s" % files_name) - print "copy_path : %s" % copy_path - print "copy_full_path : %s" % copy_full_path - print "build_name : %s" % build_name + print("copy_path : %s" % copy_path) + print("copy_full_path : %s" % copy_full_path) + print("build_name : %s" % build_name) upload_msg = utils_upload_template( filename=files_name, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=True) - print "upload_msg : %s" % upload_msg - print "make_copy : %s" % make_copy - print "copy_cmd : %s" % copy_cmd - print "backup_cmd : %s" % backup_cmd - print "------------------------------\n\n" + print("upload_msg : %s" % upload_msg) + print("make_copy : %s" % make_copy) + print("copy_cmd : %s" % copy_cmd) + print("backup_cmd : %s" % backup_cmd) + print("------------------------------\n\n") @task -def generate(param=None, make_copy=False): - SCRIPT_LIST = ['settings', 'local', 'wsgi'] +def generate(param="help", make_copy=False): + """ + param can be one of settings, local, wsgi + make_copy must be set to True if you want it to actually do anthing + """ + + SCRIPT_LIST = ['settings', 'local', 'local.generated', 'wsgi'] PARAM_LIST = list(SCRIPT_LIST) PARAM_LIST.append('gunicorn') make_copy = booleanize(make_copy) - if param and param not in PARAM_LIST: - err_msg = "You asked to generate %s, that value isn't available" \ - " possible script values available: %s" % (param, PARAM_LIST) + if param: - import sys - sys.exit(err_msg) + err_msg = None + + if param == "help": + err_msg = """ + + fab django.generate:param="help",make_copy=False + + param - file to be generated can be of type: %s + + make_copy - determines whether or generate copies the generated file + into it's practical location (otherwise doesn't do anything) + """ % PARAM_LIST + + elif param not in PARAM_LIST: + err_msg = "You asked to generate %s, that value isn't available" \ + " possible script values available: %s" % (param, PARAM_LIST) + + if err_msg: + import sys + sys.exit(err_msg) + + print("django:generate make_copy : %s\n" % make_copy) + print("django:generate script : %s" % param) - print "django:generate make_copy : %s\n" % make_copy - print "django:generate script : %s" % param if env.debug: - print "django:generate script : %s" % param - print "django:generate make_copy : %s\n" % make_copy + print("django:generate script : %s" % param) + print("django:generate make_copy : %s\n" % make_copy) else: pass # env.debug does not block the rest of the commands because this @@ -396,7 +512,6 @@ def generate(param=None, make_copy=False): if not param: # this is where script=None, generate all scripts - for scriptkey in SCRIPT_LIST: generate_scripts(scriptkey, make_copy) @@ -468,7 +583,7 @@ def generate_gunicorn(make_link=True): logger.debug('\n%s' % print_run(msg_link_gunicorn)) - from utils import upload_template as utils_upload_template + from .utils import upload_template as utils_upload_template upload_msg = utils_upload_template( filename=configuration.templates.gunicorn.conf.src, @@ -491,21 +606,21 @@ def generate_gunicorn(make_link=True): template_dir=files_path) if make_link: - print "\nlinking the generating gunicorn file in conf to " \ - "the server diretory\n" + print("\nlinking the generating gunicorn file in conf to " + "the server diretory\n") fabric_ops.run(msg_link_gunicorn) else: - print "\nNOTE: not linking the generated gunicorn file" \ - "to the server directory\n" + print("\nNOTE: not linking the generated gunicorn file" + "to the server directory\n") @task def edit(param='help'): """ - calls up mvim on the gunicorn conf file + calls up mvim on the gunicorn and django conf file """ - from maintenance import edit as maintenance_edit + from .maintenance import edit as maintenance_edit configuration = env.config @@ -536,6 +651,13 @@ def edit(param='help'): django_path=django_path, project_branch=project_branch) + settings_generated_path = "{django_path}/_settings/" \ + "{project_branch}_generated.py".format( + django_path=django_path, + project_branch=project_branch) + + file_debug_handler = configuration.logging.django.handlers.file_debug + # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', # 'settings', 'local'] locations = { @@ -549,6 +671,16 @@ def edit(param='help'): 'desc': "gunicorn.conf file in scripts/conf/gunicorn/build" }, + 'gunicorn.log.error': { + 'path': configuration.logging.gunicorn.error, + 'desc': "gunicorn error log file", + }, + + 'gunicorn.log.access': { + 'path': configuration.logging.gunicorn.access, + 'desc': "gunicorn error log file", + }, + 'settings': { 'path': settings_path, 'desc': 'main settings file for django project', @@ -557,7 +689,24 @@ def edit(param='help'): 'local': { 'path': settings_local_path, 'desc': 'local settings file for django project', - } + }, + + 'generated': { + 'path': settings_generated_path, + 'desc': 'generated settings file for django project', + }, + + 'server_log': { + 'path': file_debug_handler.path.server, + 'desc': 'django server log - handler name: %s' % + file_debug_handler.name.server, + }, + + 'project_log': { + 'path': file_debug_handler.path.project, + 'desc': 'django project log - handler name: %s' % + file_debug_handler.name.project, + }, } if param in locations.keys(): @@ -566,7 +715,7 @@ def edit(param='help'): else: # if param == 'help': - print """ + print(""" "fab django.edit" automates editing files important to django whether locally or remotely @@ -574,10 +723,10 @@ def edit(param='help'): parameter currently editable locations are: - """ + """) for k_loc in locations.keys(): - print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) return @@ -585,7 +734,7 @@ def edit(param='help'): @task def clearmigrations(appname="help"): if appname == "help": - print """ + print(""" "fab django.clearmigration:{appname}" clears out all migrations for the specified appname @@ -594,7 +743,7 @@ def clearmigrations(appname="help"): Note: if your appname is actually "help" you might want to go into this function and change it up a bit! - """ + """) return configuration = env.config @@ -636,7 +785,7 @@ def clearmigrations(appname="help"): @task def makemigrations_empty(param="help"): if param == "help": - print "print this help message" + print("print this help message") return manage("makemigrations --empty %s" % param) @@ -653,14 +802,14 @@ def fixtures(appname=None, backup=False): # booleanize the backup parameter backup = booleanize(backup) - print "debug - appname: %s" % appname + print("debug - appname: %s" % appname) # from fabric.api import * path_data = configuration.paths.django.fixtures path_backup = configuration.paths.server.backups.fixtures - print "debug - path_backup: %s" % path_backup + print("debug - path_backup: %s" % path_backup) if appname is not None: path_data = os.path.join(path_data, appname) @@ -677,9 +826,9 @@ def fixtures(appname=None, backup=False): else: path_fixture = os.path.join(path_data, fixture_name) - print "debug - path_fixture: %s" % path_fixture + print("debug - path_fixture: %s" % path_fixture) - from utils import ensure_dir + from .utils import ensure_dir ensure_dir(path_data) ensure_dir(path_backup) @@ -696,10 +845,15 @@ def fixtures(appname=None, backup=False): if configuration.project.host == 'localhost': p = re.compile("\[localhost\] out:") + for match in p.finditer(output): pass - pos = match.end() - output = output[pos:] + + try: + pos = match.end() + output = output[pos:] + except: + pass # now find the first occurence of [ on a newline, e.g. # [ @@ -727,7 +881,7 @@ def loaddata(param=None): configuration = env.config if param is None: - print "you must specify an appname to load the fixture to" + print("you must specify an appname to load the fixture to") return else: appname = param diff --git a/modules/docker.py b/modules/docker.py index 2a05d73..10ff6c2 100644 --- a/modules/docker.py +++ b/modules/docker.py @@ -4,8 +4,8 @@ from fabric.operations import run from fabric.contrib.files import upload_template -from utils import loggify, generate_template_files_path, booleanize -from utils import generate_template_build_path, print_console +from .utils import loggify, generate_template_files_path, booleanize +from .utils import generate_template_build_path, print_console @task @@ -13,17 +13,27 @@ def docker_ip(): configuration = env.config if configuration.docker.database.host == 'local': - docker_cmd = 'docker-machine ip default' + docker_cmd = 'docker-machine ip %s' % \ + configuration.docker.machine + return run(docker_cmd) + else: return configuration.docker.database.host def docker_run(cmd): from fabric.context_managers import prefix + configuration = env.config - docker_eval = "eval $(docker-machine env default)" - with prefix(docker_eval): + if configuration.docker.machine: + docker_eval = "eval $(docker-machine env %s)" % \ + configuration.docker.machine + + with prefix(docker_eval): + run(cmd) + + else: run(cmd) @@ -74,6 +84,7 @@ def generate(): context['docker_database_port_internal'] = \ configuration.docker.database.port + context['projectbranch'] = configuration.project.branch context['database_user'] = configuration.server.database.admin.user context['database_pass'] = configuration.server.database.admin.password context['database_name'] = configuration.server.database.name @@ -101,12 +112,13 @@ def generate(): import pprint pp = pprint.PrettyPrinter(indent=4) - print "context:\n" + print("context:\n") pp.pprint(context) else: config_src = configuration.templates.docker.database.src + upload_template( filename=config_src, destination=build_path, @@ -176,7 +188,7 @@ def start(create=False): configuration.docker.database.container_name if env.debug: - print "docker_run(%s)" % docker_start + print("docker_run(%s)" % docker_start) else: docker_run(docker_start) @@ -202,11 +214,82 @@ def stop(remove=False): docker_rm = 'docker rm %s' % configuration.docker.database.container_name if env.debug: - print "docker_run(%s)" % docker_stop - print "docker_run(%s)" % docker_rm + print("docker_run(%s)" % docker_stop) + print("docker_run(%s)" % docker_rm) else: docker_run(docker_stop) if remove: docker_run(docker_rm) + + +@task +def edit(param='help'): + """ + calls up mvim on the docker conf and build files + """ + + from .maintenance import edit as maintenance_edit + + configuration = env.config + print('keys: %s' % configuration.templates.docker.keys()) + print('path: %s' % configuration.templates.docker.path.keys()) + print('path.remote: %s' % configuration.templates.docker.path.remote) + print('path.local: %s' % configuration.templates.docker.path.local) + print('database: %s' % configuration.templates.docker.database.keys()) + print('database.src: %s' % configuration.templates.docker.database.src) + print('database.dst: %s' % configuration.templates.docker.database.dst) + + import os + + database_build_path = os.path.join( + configuration.templates.docker.path.remote, + 'build', + configuration.templates.docker.database.dst + ) + + database_template_path = os.path.join( + configuration.templates.docker.path.remote, + 'files', + configuration.templates.docker.database.src + ) + + print('database build path: %s' % database_build_path) + print('database template path: %s' % database_template_path) + + project_branch = configuration.project.branch + + locations = { + 'database': { + 'path': database_build_path, + 'desc': 'docker database file in scripts/conf/docker/build', + }, + + 'template': { + 'path': database_template_path, + 'desc': 'docker template file for database is' + 'scripts/conf/docker/files', + } + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab docker.edit" automates editing template configuration files + for docker database + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return diff --git a/modules/initialize.py b/modules/initialize.py index 2740a87..219ccc0 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -1,7 +1,7 @@ import os # import yaml import fabric.utils -import maintenance +from . import maintenance from fabric.api import env, task # from utils import loggify @@ -51,7 +51,7 @@ def environment(branchname): host = env.config.server.database.host - from docker import docker_ip + from .docker import docker_ip if host == 'docker': env.config.server.database.host = docker_ip() @@ -121,7 +121,8 @@ def add_template(dataobject, layout, config, section, template_name="conf"): # dataobject.templates.section.tname.dst # if env.debug: - # logger = loggify('intialize', 'add_template') + # # logger = loggify('intialize', 'add_template') + # debug_section = "DEBUG initialize.add_templates[%s]" % section if not hasattr(dataobject.templates, section): dataobject.templates.addbranch(section) @@ -142,12 +143,40 @@ def add_template(dataobject, layout, config, section, template_name="conf"): var_section_source = layout['templates'][section][template_name]['source'] var_section_output = layout['templates'][section][template_name]['output'] + # debug statements + # + # if env.debug and section == 'nginx': + # print("%s -- var_section_path: %s" % ( + # debug_section, var_section_path)) + # print("%s -- var_section_source: %s" % + # (debug_section, var_section_source)) + + # print("%s -- templates in config: %s" % + # (debug_section, 'templates' in config)) + + # print("%s -- nginx in config: %s" % + # (debug_section, 'nginx' in config)) + + # if 'nginx' in config: + # print("%s -- ssl in config[nginx]: %s" % + # (debug_section, 'ssl' in config['nginx'])) + # print("%s -- port in config[nginx]: %s" % + # (debug_section, 'port' in config['nginx'])) + + # if 'templates' in config: + # print("%s -- templates in config" % debug_section) + if 'templates' in config and section in config['templates']: _config = config['templates'][section] var_section_path = _config['path'] var_section_source = _config[template_name]['source'] var_section_output = _config[template_name]['output'] + # debug statements + # + # if env.debug: + # print("%s -- breakpoint 2" % debug_section) + # define the local, and dest paths _template.path.local = os.path.join( dataobject.paths.project.local, @@ -163,9 +192,17 @@ def add_template(dataobject, layout, config, section, template_name="conf"): conf_template = getattr(_template, template_name) conf_template.src = var_section_source - conf_template.dst = var_section_output + # debug statements + # + # if env.debug: + # print("%s -- template_name: %s" % (debug_section, template_name)) + # print("%s -- conf_template.src: %s" % + # (debug_section, conf_template.src)) + # print("%s -- conf_template.dst: %s" % + # (debug_section, conf_template.dst)) + def get_config(branchname): @@ -225,6 +262,19 @@ def get_config(branchname): dataobject.project.django.settings_folder = \ config['django']['settings_folder'] + # + # allowed_hosts are the ip addresses and hostnames + # that this instance is allowed to run on + + dataobject.project.django.allowedhosts = list() + _allowed = getattr(dataobject.project.django, 'allowedhosts') + _allowed.append(dataobject.project.extendedname) + _allowed.append(dataobject.project.host) + + if 'allowed_hosts' in config['django']: + for allowed in config['django']['allowed_hosts']: + _allowed.append(allowed) + dataobject.addbranch('paths') dataobject.paths.addbranch('project') @@ -349,7 +399,27 @@ def get_config(branchname): dataobject.server.nginx.port = config['nginx']['port'] - if config['project']['host'] == "localhost": + dataobject.server.nginx.socket = config['nginx']['socket'] + print("DEBUG -- socket: %s" % dataobject.server.nginx.socket) + + if 'host' in config['nginx']: + + # if we specificed a host name in the + # configuration file under 'nginx' + + # dataobject.server.nginx.addbranch('ssl') + + # dataobject.server.nginx.ssl.host = \ + # config['nginx']['ssl']['host'] + + dataobject.server.nginx.host = \ + config['nginx']['host'] + + elif config['project']['host'] == "localhost": + + # if localhost, then create a nginx appropriate name based on + # the project name and the extension + dataobject.server.nginx.host = "{projectname}.{ext}".format( ext=dataobject.project.extension, projectname=dataobject.project.name) @@ -380,6 +450,7 @@ def get_config(branchname): add_template(dataobject, layout, config, "django", "settings") add_template(dataobject, layout, config, "django", "local") + add_template(dataobject, layout, config, "django", "local.generated") add_template(dataobject, layout, config, "django", "wsgi") # @@ -414,9 +485,9 @@ def get_config(branchname): if 'database' in config['docker']: add_template(dataobject, layout, config, "docker", "database") else: - print "NOTE: docker.database does not exist for this branch" + print("NOTE: docker.database does not exist for this branch") else: - print "NOTE: docker information does not exist for this branch" + print("NOTE: docker information does not exist for this branch") # # nginx information @@ -846,7 +917,7 @@ def _init_docker(configuration, layout, config): project_branch=configuration.project.branch) if 'name' in config['docker'] and config['docker']['name']: - configuration.docker.name = config['docker']['name'] + configuration.docker.name = config['docker']['name'] # # host information @@ -854,6 +925,15 @@ def _init_docker(configuration, layout, config): if 'host' in config['docker']: configuration.docker.host = config['docker']['host'] + # + # get machine name for docker-machine if exists + + configuration.docker.machine = None + + if 'machine' in config['docker']: + configuration.docker.machine = config['docker']['machine'] + + # # # configuration info for docker database diff --git a/modules/maintenance.py b/modules/maintenance.py index 03b35cc..02e6c17 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -3,8 +3,8 @@ from fabric.api import local import os import sys -import utils -from utils import executize, virtualenv +from . import utils +from .utils import executize, virtualenv, loggify def command(program=None, cmd=None, extra_param=None): @@ -22,19 +22,19 @@ def command(program=None, cmd=None, extra_param=None): # logger = loggify('maintenance', 'command') if program is None: - print "Error: You have not given a legitimate program" + print("Error: You have not given a legitimate program") - print "permissable programs : %s" \ - % configuration.maintenance.keys() + print("permissable programs : %s" % + configuration.maintenance.keys()) sys.exit() configuration_program = getattr(configuration.maintenance, program) if cmd is None: - print "Error: You have not given a legitimate command" - print "permissable commands : %s" \ - % configuration_program.commands.keys() + print("Error: You have not given a legitimate command") + print("permissable commands : %s" % + configuration_program.commands.keys()) sys.exit() @@ -65,14 +65,24 @@ def command(program=None, cmd=None, extra_param=None): _execute(_command) -def edit(remote_path): +def edit(remote_path, host_string=None): """ calls up mvim or vim on the file remote_path - path to file we want to edit + host_string - what machine the file is located on + + host_string is necessary b/c in the case where I want to edit + configuration files, it is essential to do so only on the localhost + + that is, whether I'm editing staging, development or production project + yml files, the place to do that editing is ALWAYS on localhost, this is + because all fabric commands are executed from localhost and will check for + the necessary configuration files on local (I'm not running fabric from an + ssh command line on the aws server, I'm running it on my laptop) """ - # logger = loggify('maintenance', 'edit') + logger = loggify('maintenance', 'edit') # configuration = env.config @@ -104,12 +114,26 @@ def edit(remote_path): remote_path=remote_path) if env.debug: - # logger.debug("remote_path : %s" % remote_path) - # logger.debug("env.host_string : %s" % env.host_string) - # logger.debug("sys.platform : %s" % sys.platform) - print "maintenance.edit - cmd_edit: %s" % cmd_edit - print "env.key_filename: %s" % env.key_filename + logger.debug("remote_path : %s" % remote_path) + logger.debug("env.host_string : %s" % env.host_string) + logger.debug("sys.platform : %s" % sys.platform) + pass else: + + if sys.platform == "darwin": + editor = "mvim" + else: + editor = "vim" + + if not host_string: + host_string = env.host_string + + cmd_edit = "{editor} sftp://{user}@{host_string}/{remote_path}".format( + editor=editor, + user=env.user, + host_string=host_string, + remote_path=remote_path) + local(cmd_edit) @@ -168,8 +192,8 @@ def _get_configuration_path(name, branch): param_list = ['config', 'layout'] if name not in param_list: - print "value %s was not legit. _get_configuration_path requires" \ - "value such from %s" % (name, param_list) + print("value %s was not legit. _get_configuration_path requires" + "value such from %s" % (name, param_list)) META_DIR = os.path.join( get_project_root(), 'usr', 'meta', 'project') @@ -213,7 +237,7 @@ def load_configuration(name, branch): elif name == "layout": file_path = _get_configuration_path('layout', branch) - configuration_file = yaml.load(file(file_path, 'r')) + configuration_file = yaml.load(open(file_path, 'r')) return configuration_file diff --git a/modules/nginx.py b/modules/nginx.py index c62f341..6cd72cd 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -6,10 +6,10 @@ from fabric.operations import sudo import os # import sys -from maintenance import command as maintenance_command -from maintenance import edit as maintenance_edit +from .maintenance import command as maintenance_command +from .maintenance import edit as maintenance_edit -from utils import loggify +from .utils import loggify @task @@ -58,25 +58,77 @@ def restart(): @task -def edit(location='conf'): +def edit(param='help'): """ calls up mvim on the Nginx conf file """ configuration = env.config - conf_path = os.path.join( + config_file_path = os.path.join( configuration.nginx.sites_available, configuration.nginx.conf.name) - if location == 'conf': - remote_path = conf_path - elif location == 'log.error' or location == 'log': - remote_path = configuration.logging.nginx.error - elif location == 'log.access': - remote_path = configuration.logging.nginx.access + backup_file_path = config_file_path + '.bak' - maintenance_edit(remote_path=remote_path) + locations = { + 'conf': { + 'path': config_file_path, + 'desc': "nginx configuration file", + }, + + 'backup': { + 'path': backup_file_path, + 'desc': "backup configuration file", + }, + + 'log.error': { + 'path': configuration.logging.nginx.error, + 'desc': "nginx error log file", + }, + + 'log.access': { + 'path': configuration.logging.nginx.access, + 'desc': "nginx access log file", + }, + 'log.nginx.error': { + 'path': '/var/log/nginx/error.log', + 'desc': "nginx main error file" + }, + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + + if env.debug: + logger = loggify('nginx', 'edit') + logger.debug("param: %s\n" % param) + + logger.debug("locations:") + for key, value in locations.items(): + logger.debug("%s [path]: %s" % (key, value['path'])) + logger.debug("%s [desc]: %s\n" % (key, value['desc'])) + + logger.debug("remote_path: %s" % remote_path) + else: + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab nginx.edit" automates editing files important to nginx whether + locally or remotely + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return @task @@ -98,10 +150,12 @@ def upload(): context['error_log'] = configuration.logging.nginx.error context['port'] = configuration.server.nginx.port + context['socket'] = configuration.server.nginx.socket context['django_host'] = configuration.server.django.host context['django_port'] = configuration.server.django.port + context['project_name'] = configuration.project.name context['extended_name'] = configuration.project.extendedname context['virtualenv_sitepackages'] = \ diff --git a/modules/pip.py b/modules/pip.py index 60e4b38..44e707e 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -5,15 +5,15 @@ import sys import os import logging -from utils import virtualenv_source, virtualenv -from utils import print_console, printerr +from .utils import virtualenv_source, virtualenv +from .utils import print_console, printerr ERROR_BAD_BRANCH_PARAM = -3 ERROR_BAD_PARAM = -2 @task -def setup_virtualenv(): +def setup_virtualenv(python3=True): configuration = env.config if env.debug: @@ -21,11 +21,18 @@ def setup_virtualenv(): format='\n%(levelname)s: deploy.setup_virtualenv %(message)s', level=logging.DEBUG) - mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " \ - "{virtualenv_name}".format( - virtualenv_name=configuration.virtualenv.name) + mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " + + if python3: + mkvirtualenv_cmd += "--python=python3 " + + mkvirtualenv_cmd += "{virtualenv_name}".format( + virtualenv_name=configuration.virtualenv.name) if env.debug: + + logging.debug("python3 install: %s" % python3) + logging.debug("virtualenv.workon : %s" % configuration.virtualenv.workon) @@ -41,6 +48,8 @@ def setup_virtualenv(): logging.debug("virtualenv.paths.root : %s" % configuration.virtualenv.paths.root) + logging.debug("mkvirtualenv_cmd: %s" % mkvirtualenv_cmd) + logging.debug("with virtualenv_source(): run(\"\n\t%s\n\t\")".format( mkvirtualenv_cmd)) else: @@ -58,7 +67,7 @@ def bootstrap_pip(): """ bootstraps pip """ - upgrade() + # upgrade() setup() @@ -221,8 +230,8 @@ def copyto(branch): configuration.virtualenv.requirements.local, branch_config.virtualenv.requirements.filename) - print "current_local_path: %s" % current_local_path - print "branch_local_path: %s" % branch_local_path + print("current_local_path: %s" % current_local_path) + print("branch_local_path: %s" % branch_local_path) message = "Copying file from current branch '{branch_src}' to " \ "destination branch '{branch_dst}'. Continue? Y/n ".format( diff --git a/modules/supervisor.py b/modules/supervisor.py index b3f0a22..1ad3827 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -5,10 +5,10 @@ from fabric.operations import sudo, run import os -from maintenance import command as maintenance_command -from maintenance import edit as maintenance_edit +from .maintenance import command as maintenance_command +from .maintenance import edit as maintenance_edit -from utils import loggify +from .utils import loggify def _initialize(configuration): @@ -47,7 +47,7 @@ def command(cmd=None): param = _initialize(configuration) if cmd == "update": - print "in update" + print("in update") # we don't need to specify the supervisor configuration file name, because # supervisor can figure it out. ie whatever.conf is referred to by @@ -108,24 +108,56 @@ def restart(): @task -def edit(): +def edit(param='help'): """ calls up mvim on the Supervisor conf file """ configuration = env.config - param = _initialize(configuration) if env.debug: logger = loggify('supervisor', 'edit') + logger.debug() - conf_path = param['conf_path'] + configuration_path = _initialize(configuration)['conf_path'] - if env.debug: - logger.debug("conf path : %s" % conf_path) + locations = { + 'configuration': { + 'path': configuration_path, + 'desc': "supervisor configuration path", + }, + 'log.out': { + 'path': configuration.logging.supervisor.out, + 'desc': "out logging file", + }, + + 'log.err': { + 'path': configuration.logging.supervisor.err, + 'desc': "err logging file", + }, + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) else: - maintenance_edit(remote_path=conf_path) + # if param == 'help': + + print(""" + "fab django.edit" automates editing files important to django whether + locally or remotely + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return @task @@ -224,7 +256,7 @@ def upload(): logger.debug("run(%s)" % copy_command) else: - import utils + from . import utils utils.printvar("files_path", files_path) upload_template( diff --git a/modules/utils.py b/modules/utils.py index d34d51c..b1aa820 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -11,7 +11,7 @@ from fabric.operations import run, sudo def printvar(name, value, exit=False): - print "%s : %s" % (name, value) + print("%s : %s" % (name, value)) if exit: sys.exit() @@ -29,7 +29,7 @@ def printerr(message="", errcode=-2, exit=True): message=message, errcode=errcode) print - print message + print(message) print sys.exit(errcode) @@ -74,22 +74,21 @@ def print_console(string, prepend="\n\n", append="\n\n", sep="-", numsep=44): numsep - number of times the separator is printed out on a line """ - print prepend + print(prepend) if sep: - print sep * numsep + print(sep * numsep) - print string + print(string) if sep: - print sep * numsep + print(sep * numsep) - print append + print(append) def print_debug(debugstr, module, function): - print "%s:%s:%s" \ - % (module, function, debugstr) + print("%s:%s:%s" % (module, function, debugstr)) def executize(config_execute): @@ -146,12 +145,13 @@ def ensure_dir(directory): """ try: if not os.path.exists(directory): - print "creating directory: %s" % directory + print("creating directory: %s" % directory) os.makedirs(directory) - except OSError, e: + except OSError as e: if e.errno != errno.EEXIST: - print "Error occurred while creating directory: %s" % directory + print("Error occurred while creating directory: %s" + % directory) raise @@ -319,7 +319,7 @@ def handle_help(param, message, values=None): if isinstance(param, str): if param.lower() in values: - print message + print(message) return True return False diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..b04fe95 --- /dev/null +++ b/readme.md @@ -0,0 +1,21 @@ +## adding fabric + +`git add submodule git@bitbucket.org:ronnyabraham/fabric.git fabric` + +the second "fabric" puts the submogule in a directory called 'fabric" + +workon a virtualenv that has PyQt5 + +load up bin/deploy_meta.py + +check django.numbers to make sure your port numbers aren't already being used +- database +- django + +after you create the dev file, go to usr/meta/projects and copy the layout.yml file from another project + +now go to usr/meta/virtualenv, copy bootstrap.txt to development.txt +replace pycrypto with pycryptodome +replace psycopg2 with psycopg2-binary + +add allowed_hosts = [127.0.0.1] to the django section in development.yml diff --git a/share/templates/conf/database/files/db.init.sql.jinja2 b/share/templates/conf/database/files/db.init.sql.jinja2 index dd8ee81..c8bc93a 100644 --- a/share/templates/conf/database/files/db.init.sql.jinja2 +++ b/share/templates/conf/database/files/db.init.sql.jinja2 @@ -1,5 +1,5 @@ CREATE USER {{db_user}} WITH PASSWORD '{{db_password}}'; ALTER USER {{db_user}} CREATEDB; -CREATE DATABASE {{db_name}}; +CREATE DATABASE {{db_name}} WITH ENCODING='UTF8'; ALTER DATABASE {{db_name}} OWNER TO {{db_user}}; GRANT ALL PRIVILEGES ON DATABASE {{db_name}} TO {{db_user}}; diff --git a/share/templates/conf/database/files/db.re_init.sql.jinja2 b/share/templates/conf/database/files/db.re_init.sql.jinja2 index d9e25c6..e1b4261 100644 --- a/share/templates/conf/database/files/db.re_init.sql.jinja2 +++ b/share/templates/conf/database/files/db.re_init.sql.jinja2 @@ -1,4 +1,4 @@ DROP DATABASE {{db_name}}; -CREATE DATABASE {{db_name}}; +CREATE DATABASE {{db_name}} WITH ENCODING='UTF8'; ALTER DATABASE {{db_name}} OWNER TO {{db_user}}; GRANT ALL PRIVILEGES ON DATABASE {{db_name}} TO {{db_user}}; diff --git a/share/templates/conf/django/files/local.generated.jinja2 b/share/templates/conf/django/files/local.generated.jinja2 new file mode 100644 index 0000000..e23c128 --- /dev/null +++ b/share/templates/conf/django/files/local.generated.jinja2 @@ -0,0 +1,135 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + +LOCAL_SETTINGS = True # avoid recursive imports +BRANCH = '{{ project_branch }}' + +from {{ project_name }}.settings import * + +import logging + +LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH +logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) + +SITE_ID = 1 + +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") + +# +# NOTE: a lot of the code in these local settings files are automated and you +# might be inclined to take them out and move them into the main settings.py +# file. That would be a mistake. These automatic files AT THE VERY LEAST have +# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. +# Once I set that file, THEN all the rest of the information can be automated. +# So all these automated info needs to be here. + +DATABASES = { + 'default': { + 'ENGINE': '{{server_database_backend}}', + 'NAME': '{{server_database_name}}', + 'USER': '{{server_database_user}}', + 'PASSWORD': '{{server_database_password}}', + 'HOST': '{{server_database_host}}', + 'PORT': '{{server_database_port}}', + } +} + + +# +# directory from which we serve static files +# +# NOTE: both STATIC and MEDIA roots are getting their values from the +# initialization files that are set up above. Also, MEDIA_ROOT is being set +# to something called "paths.server.media.dynamic" - the names are different, +# but it means the same thing. +# +# MEDIA_ROOT is the dynamic media information that the web server, user or +# admin # will be adding and taking out. It's why I call it "dynamic" + +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' + +# directories from which we search for static files to place in STATIC_ROOT +# these static files are located within the project root as opposed to the +# server root location + +STATICFILES_DIRS = ( + os.path.join("{{paths_project_root}}", "share", "media"), +) + +# debug and debug toolbar settings +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +USE_DEBUG_TOOLBAR = DEBUG + +# allow template debug outputs on {{ project_branch }} environment +INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] + +ALLOWED_HOSTS = {{ project_django_allowedhosts }} + +# ----------------------------------------- +# Debug logging to the console + +LOGGING = { + 'version': 1, + 'formatters': { + 'verbose': { + 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" + " %(thread)d %(message)s" + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + + 'code': { + 'format': "%(module)s:%(funcName)s - %(message)s" + }, + }, + 'handlers': { + '{{ file_debug_handler__name_project }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_project}}', + + 'formatter': 'code' + }, + + '{{ file_debug_handler__name_server }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_server}}', + + 'formatter': 'code' + }, + + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'null': { + 'level': 'DEBUG', + 'class': 'logging.NullHandler', + } + }, + 'loggers': { + 'django.debug': { + # use the console for logging + 'handlers': + [ + 'console', + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', + ], + + 'level': 'DEBUG', + 'propagate': True, + }, + } +} + diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index 980b10f..95267b3 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -1,136 +1,9 @@ LOCAL_SETTINGS = True # avoid recursive imports -BRANCH = '{{ project_branch }}' -from {{ project_name }}.settings import * +from {{ project_name }}._settings.{{ project_branch }}_generated import * -import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -configuration = initialize.environment(BRANCH) - -SITE_ID = 1 - -TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) - -# -# NOTE: a lot of the code in these local settings files are automated and you -# might be inclined to take them out and move them into the main settings.py -# file. That would be a mistake. These automatic files AT THE VERY LEAST have -# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. -# Once I set that file, THEN all the rest of the information can be automated. -# So all these automated info needs to be here. - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend=configuration.server.database.backend), - - 'NAME': '{db_name}'.format(db_name=configuration.server.database.name), - 'USER': '{db_user}'.format(db_user=configuration.server.database.user), - - 'PASSWORD': '{db_pass}'.format( - db_pass=configuration.server.database.password), - - 'HOST': '{db_host}'.format(db_host=configuration.server.database.host), - 'PORT': '{db_port}'.format(db_port=configuration.server.database.port), - } -} - - -# -# directory from which we serve static files -# -# NOTE: both STATIC and MEDIA roots are getting their values from the -# initialization files that are set up above. Also, MEDIA_ROOT is being set -# to something called "paths.server.media.dynamic" - the names are different, -# but it means the same thing. -# -# MEDIA_ROOT is the dynamic media information that the web server, user or -# admin # will be adding and taking out. It's why I call it "dynamic" - -STATIC_ROOT = configuration.paths.server.media.static -MEDIA_ROOT = configuration.paths.server.media.dynamic - -# directories from which we search for static files to place in STATIC_ROOT -# these static files are located within the project root as opposed to the -# server root location - -STATICFILES_DIRS = ( - os.path.join(configuration.paths.project.root, "share", "media"), -) - -# debug and debug toolbar settings -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -USE_DEBUG_TOOLBAR = DEBUG - -# allow template debug outputs on {{ project_branch }} environment -INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] - -ALLOWED_HOSTS = [configuration.project.extendedname, ] - -# ----------------------------------------- -# Debug logging to the console - -# convenience variable naming, otherwise it's too long to deal with -file_debug_handler = configuration.logging.django.handlers.file_debug - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" - " %(thread)d %(message)s" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - - 'code': { - 'format': "%(module)s:%(funcName)s - %(message)s" - }, - }, - 'handlers': { - file_debug_handler.name.project: { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.project, - 'formatter': 'code' - }, - - file_debug_handler.name.server: { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.server, - 'formatter': 'code' - }, - - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - }, - 'null': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', - } - }, - 'loggers': { - 'django.debug': { - # use the console for logging - 'handlers': - [ - 'console', - file_debug_handler.name.project, - file_debug_handler.name.server - ], - - 'level': 'DEBUG', - 'propagate': True, - }, - } -} - diff --git a/share/templates/conf/django/files/settings.1.9.jinja2 b/share/templates/conf/django/files/settings.1.9.jinja2 deleted file mode 100644 index be1fd74..0000000 --- a/share/templates/conf/django/files/settings.1.9.jinja2 +++ /dev/null @@ -1,138 +0,0 @@ -""" -Django settings for {{ project_name }} project. - -Generated by 'django-admin startproject' using Django 1.9.4. - -For more information on this file, see -https://docs.djangoproject.com/en/1.9/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.9/ref/settings/ -""" - -import os -import sys - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) - -# -# add the directory that has the intialize.py module which contains the -# get_config function definition which will pull out all the configuraiton -# infomration for the settings file - -sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) - -# -# add an "apps" directory to this project, which is where all the apps -# ought to be in the first place. - -sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '{{ secret_key }}' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -] - -MIDDLEWARE_CLASSES = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = '{{ project_name }}.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(PROJECT_ROOT, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = '{{ project_name }}.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.9/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - - -# Password validation -# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/1.9/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.9/howto/static-files/ - -STATIC_URL = '/static/' diff --git a/share/templates/conf/django/files/settings.jinja2 b/share/templates/conf/django/files/settings.jinja2 index 587a1f5..a771b3c 100644 --- a/share/templates/conf/django/files/settings.jinja2 +++ b/share/templates/conf/django/files/settings.jinja2 @@ -1,13 +1,13 @@ """ Django settings for {{ project_name }} project. -Generated by 'django-admin startproject' using Django 1.10.1. +Generated by 'django-admin startproject' using Django 2.1.2. For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ +https://docs.djangoproject.com/en/2.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ +https://docs.djangoproject.com/en/2.1/ref/settings/ """ import os @@ -23,7 +23,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) # infomration for the settings file sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) + PROJECT_ROOT, 'usr', 'bin', 'fabric')) # # add an "apps" directory to this project, which is where all the apps @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.join( sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '{{ secret_key }}' @@ -60,7 +60,6 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] @@ -87,7 +86,7 @@ WSGI_APPLICATION = '{{ project_name }}.wsgi.application' # Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASES = { 'default': { @@ -98,7 +97,7 @@ DATABASES = { # Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { @@ -121,7 +120,7 @@ AUTH_PASSWORD_VALIDATORS = [ # Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ +# https://docs.djangoproject.com/en/2.1/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -135,7 +134,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ +# https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_URL = '/static/' MEDIA_URL = '/media/' diff --git a/share/templates/conf/django/files/settings18.jinja2 b/share/templates/conf/django/files/settings18.jinja2 deleted file mode 100644 index 75c5cdb..0000000 --- a/share/templates/conf/django/files/settings18.jinja2 +++ /dev/null @@ -1,125 +0,0 @@ -""" -Django settings for {{ project_name }} project. - -Generated by 'django-admin startproject' using Django 1.8.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.8/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.8/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os -import sys - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) - -# -# add the directory that has the intialize.py module which contains the -# get_config function definition which will pull out all the configuraiton -# infomration for the settings file - -sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'scripts', 'tools', 'fabric', 'modules')) - -# -# add an "apps" directory to this project, which is where all the apps -# ought to be in the first place. - -sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) - -# -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ - -# -# SECURITY WARNING: keep the secret key used in production secret! - -SECRET_KEY = '{{ secret_key }}' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.sites', - 'django.contrib.staticfiles', - - 'polls', -) - - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = '{{ project_name }}.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(PROJECT_ROOT, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] -WSGI_APPLICATION = '{{ project_name }}.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - - -# Internationalization -# https://docs.djangoproject.com/en/1.8/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.8/howto/static-files/ - -STATIC_URL = '/static/' -MEDIA_URL = '/media/' - diff --git a/share/templates/conf/gunicorn/files/local.generated.jinja2 b/share/templates/conf/gunicorn/files/local.generated.jinja2 new file mode 100644 index 0000000..e23c128 --- /dev/null +++ b/share/templates/conf/gunicorn/files/local.generated.jinja2 @@ -0,0 +1,135 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + +LOCAL_SETTINGS = True # avoid recursive imports +BRANCH = '{{ project_branch }}' + +from {{ project_name }}.settings import * + +import logging + +LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH +logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) + +SITE_ID = 1 + +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") + +# +# NOTE: a lot of the code in these local settings files are automated and you +# might be inclined to take them out and move them into the main settings.py +# file. That would be a mistake. These automatic files AT THE VERY LEAST have +# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. +# Once I set that file, THEN all the rest of the information can be automated. +# So all these automated info needs to be here. + +DATABASES = { + 'default': { + 'ENGINE': '{{server_database_backend}}', + 'NAME': '{{server_database_name}}', + 'USER': '{{server_database_user}}', + 'PASSWORD': '{{server_database_password}}', + 'HOST': '{{server_database_host}}', + 'PORT': '{{server_database_port}}', + } +} + + +# +# directory from which we serve static files +# +# NOTE: both STATIC and MEDIA roots are getting their values from the +# initialization files that are set up above. Also, MEDIA_ROOT is being set +# to something called "paths.server.media.dynamic" - the names are different, +# but it means the same thing. +# +# MEDIA_ROOT is the dynamic media information that the web server, user or +# admin # will be adding and taking out. It's why I call it "dynamic" + +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' + +# directories from which we search for static files to place in STATIC_ROOT +# these static files are located within the project root as opposed to the +# server root location + +STATICFILES_DIRS = ( + os.path.join("{{paths_project_root}}", "share", "media"), +) + +# debug and debug toolbar settings +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +USE_DEBUG_TOOLBAR = DEBUG + +# allow template debug outputs on {{ project_branch }} environment +INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] + +ALLOWED_HOSTS = {{ project_django_allowedhosts }} + +# ----------------------------------------- +# Debug logging to the console + +LOGGING = { + 'version': 1, + 'formatters': { + 'verbose': { + 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" + " %(thread)d %(message)s" + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + + 'code': { + 'format': "%(module)s:%(funcName)s - %(message)s" + }, + }, + 'handlers': { + '{{ file_debug_handler__name_project }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_project}}', + + 'formatter': 'code' + }, + + '{{ file_debug_handler__name_server }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_server}}', + + 'formatter': 'code' + }, + + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'null': { + 'level': 'DEBUG', + 'class': 'logging.NullHandler', + } + }, + 'loggers': { + 'django.debug': { + # use the console for logging + 'handlers': + [ + 'console', + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', + ], + + 'level': 'DEBUG', + 'propagate': True, + }, + } +} + diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index 980b10f..95267b3 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -1,136 +1,9 @@ LOCAL_SETTINGS = True # avoid recursive imports -BRANCH = '{{ project_branch }}' -from {{ project_name }}.settings import * +from {{ project_name }}._settings.{{ project_branch }}_generated import * -import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -configuration = initialize.environment(BRANCH) - -SITE_ID = 1 - -TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) - -# -# NOTE: a lot of the code in these local settings files are automated and you -# might be inclined to take them out and move them into the main settings.py -# file. That would be a mistake. These automatic files AT THE VERY LEAST have -# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. -# Once I set that file, THEN all the rest of the information can be automated. -# So all these automated info needs to be here. - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend=configuration.server.database.backend), - - 'NAME': '{db_name}'.format(db_name=configuration.server.database.name), - 'USER': '{db_user}'.format(db_user=configuration.server.database.user), - - 'PASSWORD': '{db_pass}'.format( - db_pass=configuration.server.database.password), - - 'HOST': '{db_host}'.format(db_host=configuration.server.database.host), - 'PORT': '{db_port}'.format(db_port=configuration.server.database.port), - } -} - - -# -# directory from which we serve static files -# -# NOTE: both STATIC and MEDIA roots are getting their values from the -# initialization files that are set up above. Also, MEDIA_ROOT is being set -# to something called "paths.server.media.dynamic" - the names are different, -# but it means the same thing. -# -# MEDIA_ROOT is the dynamic media information that the web server, user or -# admin # will be adding and taking out. It's why I call it "dynamic" - -STATIC_ROOT = configuration.paths.server.media.static -MEDIA_ROOT = configuration.paths.server.media.dynamic - -# directories from which we search for static files to place in STATIC_ROOT -# these static files are located within the project root as opposed to the -# server root location - -STATICFILES_DIRS = ( - os.path.join(configuration.paths.project.root, "share", "media"), -) - -# debug and debug toolbar settings -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -USE_DEBUG_TOOLBAR = DEBUG - -# allow template debug outputs on {{ project_branch }} environment -INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] - -ALLOWED_HOSTS = [configuration.project.extendedname, ] - -# ----------------------------------------- -# Debug logging to the console - -# convenience variable naming, otherwise it's too long to deal with -file_debug_handler = configuration.logging.django.handlers.file_debug - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" - " %(thread)d %(message)s" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - - 'code': { - 'format': "%(module)s:%(funcName)s - %(message)s" - }, - }, - 'handlers': { - file_debug_handler.name.project: { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.project, - 'formatter': 'code' - }, - - file_debug_handler.name.server: { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.server, - 'formatter': 'code' - }, - - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - }, - 'null': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', - } - }, - 'loggers': { - 'django.debug': { - # use the console for logging - 'handlers': - [ - 'console', - file_debug_handler.name.project, - file_debug_handler.name.server - ], - - 'level': 'DEBUG', - 'propagate': True, - }, - } -} - diff --git a/share/templates/conf/gunicorn/files/settings.jinja2 b/share/templates/conf/gunicorn/files/settings.jinja2 index 587a1f5..a771b3c 100644 --- a/share/templates/conf/gunicorn/files/settings.jinja2 +++ b/share/templates/conf/gunicorn/files/settings.jinja2 @@ -1,13 +1,13 @@ """ Django settings for {{ project_name }} project. -Generated by 'django-admin startproject' using Django 1.10.1. +Generated by 'django-admin startproject' using Django 2.1.2. For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ +https://docs.djangoproject.com/en/2.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ +https://docs.djangoproject.com/en/2.1/ref/settings/ """ import os @@ -23,7 +23,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) # infomration for the settings file sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) + PROJECT_ROOT, 'usr', 'bin', 'fabric')) # # add an "apps" directory to this project, which is where all the apps @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.join( sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '{{ secret_key }}' @@ -60,7 +60,6 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] @@ -87,7 +86,7 @@ WSGI_APPLICATION = '{{ project_name }}.wsgi.application' # Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASES = { 'default': { @@ -98,7 +97,7 @@ DATABASES = { # Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { @@ -121,7 +120,7 @@ AUTH_PASSWORD_VALIDATORS = [ # Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ +# https://docs.djangoproject.com/en/2.1/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -135,7 +134,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ +# https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_URL = '/static/' MEDIA_URL = '/media/' diff --git a/share/templates/meta/development.yml b/share/templates/meta/development.yml index 077788f..b6db090 100644 --- a/share/templates/meta/development.yml +++ b/share/templates/meta/development.yml @@ -6,7 +6,10 @@ database: users: admin: {name: DATABASE_ADMIN_NAME, pass: DATABASE_ADMIN_PASS} default: {name: DATABASE_USER_NAME, pass: DATABASE_USER_PASS} -django: {host: DJANGO_IP, port: DJANGO_PORT} +django: + host: DJANGO_IP + port: DJANGO_PORT + allowed_hosts: [127.0.0.1, 'PROJECT_NAME.BRANCH_EXT',] docker: database: env: {name: POSTGRES_DB, pass: POSTGRES_PASSWORD, user: POSTGRES_USER} @@ -32,6 +35,7 @@ media: nginx: paths: {available: null, enabled: servers, root: /usr/local/etc/nginx} port: NGINX_PORT + socket: False overrides: [null] project: branch: development diff --git a/share/templates/meta/staging.yml b/share/templates/meta/staging.yml index 593ee01..418299c 100644 --- a/share/templates/meta/staging.yml +++ b/share/templates/meta/staging.yml @@ -24,6 +24,7 @@ media: nginx: paths: {available: sites-available, enabled: sites-enabled, root: /etc/nginx} port: NGINX_PORT + socket: True overrides: [null] project: branch: BRANCH_NAME diff --git a/share/templates/readmes/aws.md b/share/templates/readmes/aws.md index 64582ea..f5cbb66 100644 --- a/share/templates/readmes/aws.md +++ b/share/templates/readmes/aws.md @@ -161,9 +161,15 @@ replace it with: mkdir .ssh touch .ssh/authorized_keys vim ~/.ssh/authorized_keys + +6. change the permissions -6. now paste in the public_key you got in step 1 -7. log out and test with the username + chown -R username:username_gropus .ssh + chmod 700 .ssh + chmod 600 .ssh/authorized_keys + +7. now paste in the public_key you got in step 1 +8. log out and test with the username ssh username@instance.domain @@ -228,6 +234,7 @@ AWS has a sudo group that allows a user sudo priveleges ### install python packages sudo apt-get install python-dev + sudo apt-get install python3-dev sudo apt-get install libjpeg-dev diff --git a/share/templates/readmes/certbot.md b/share/templates/readmes/certbot.md new file mode 100644 index 0000000..b2462cd --- /dev/null +++ b/share/templates/readmes/certbot.md @@ -0,0 +1 @@ +certbot looks through all configuration files and modifies them for ssl diff --git a/share/templates/readmes/fontawesome.md b/share/templates/readmes/fontawesome.md new file mode 100644 index 0000000..556e0c9 --- /dev/null +++ b/share/templates/readmes/fontawesome.md @@ -0,0 +1,35 @@ +###specify the font path to the "webfonts directory" + +$fa-font-path: "../node_modules/@fortawesome/fontawesome-free/webfonts" !default; + +###when importing from an app-local installation: + +in this case "coffesshop" is the name of the app + +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/fontawesome'` +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/solid'` +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/brands'` + +### when importing from a shared directory + +#### set the location in static and sass + +`STATICFILES_DIRS = [ + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/@fortawesome'), +]` + +`SASS_PROCESSOR_INCLUDE_DIRS = [ + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/@fortawesome'), +]` + +#### now import directly from where you set loc above + +`@import 'fontawesome-free/scss/fontawesome'` +`@import 'fontawesome-free/scss/solid'` +`@import 'fontawesome-free/scss/brands'` + + diff --git a/share/templates/readmes/foundation.md b/share/templates/readmes/foundation.md new file mode 100644 index 0000000..a08e959 --- /dev/null +++ b/share/templates/readmes/foundation.md @@ -0,0 +1,120 @@ +# How to install Foundation Zurb + +## Resources +* [node js](https://nodejs.org/en/download/) +* [foundation zurb](foundation.zurb.com) + + +## install nodeenv + +[how to install nodeenv](https://calvinx.com/2013/07/11/python-virtualenv-with-node-environment-via-nodeenv/) + +1. mkvirtualenv myproject1 +2. pip install nodeenv +3. nodeenv -p +4. deactivate; workon myproject1 +5. npm install -g foundation-cli +6. npm install -g autoprefixer +7. npm install -g generate +8. npm install -g postcss-cli + +## install django sass processor + +github url [django sass processor](https://github.com/jrief/django-sass-processor) + + pip install django-sass-processor + +### add to installed apps + + sass_processor + +### additional configuration settings + +#### set extensions to look for + +sass processor searches template files to find links to sass stylesheets. If using templates with a .django extension, you have to set the ext types you want the processor to search + + SASS_TEMPLATE_EXTS = ['.django', '.html'] + +#### file search root directory + +set the root value for which sass processor will search for sass files (set to the same value as static root) + + SASS_PROCESSOR_ROOT = STATIC_ROOT + +#### add cssfinder to static file finders + +we need to tell the static finders thingie to use sass_processor + + STATICFILES_FINDERS = [ + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'sass_processor.finders.CssFinder', + ] + +#### directories to look through + +you must tell django wher eto find static files that you want to look through if you want to use them in a template + +you must also tell the sass processor where those files are if you want the sass processor to be able to do anything with it + + STATICFILES_DIRS = [ + ('node_modules', + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/')), + + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/foundation-sites/scss'), + ] + + SASS_PROCESSOR_INCLUDE_DIRS = [ + os.path.join(PROJECT_ROOT, 'share/media/node_modules'), + os.path.join(PROJECT_ROOT, + 'share/media/node_modules/foundation-sites/scss'), + ] + +Notice how "node modules" is attached without a prefix. + +also, I add direct links to the scss of the underlying packages so I can import them with a direct command + + @import foundation + +_instead of_ + + @import foundation-sites/scss/foundation + +When you want to import sass modules underneath the "node_modules" directory you must refer to the directory structure _underneath_ the node modules structure: + + @import foundation-sites/scss/foundation + +however, to directly import the foundation sass library, you must do so through the "assets" directory referenced in static. + +so foundation-sites/assets refers to what will be used in static, and foundation-sites/scss/foundation refers to compiling + +### Usage + + {% load sass_tags %} + + + + +## installing Zurb Foundation Template + +[how to install foundation with django](https://www.physics.utoronto.ca/~icom/workshop/django-install/foundation.html) + +either open up the virtualenv that contains foundation-cli or create one, or just install foundation-cli + +create a new template + + foundation new + +## installing via foundation-sites + + npm install foundation-sites + + + + diff --git a/share/templates/readmes/geo.md b/share/templates/readmes/geo.md new file mode 100644 index 0000000..9984965 --- /dev/null +++ b/share/templates/readmes/geo.md @@ -0,0 +1,25 @@ +##if you get the following error: + +django.core.exceptions.ImproperlyConfigured: Could not find the GDAL library (tried "gdal", "GDAL", "gdal2.2.0", "gdal2.1.0", "gdal2.0.0", "gdal1.11.0", "gdal1.10.0", "gdal1.9.0"). Is GDAL installed? If it is, try setting GDAL_LIBRARY_PATH in your settings. + +you must install on the host machine the following packages: +`sudo apt-get install binutils libproj-dev gdal-bin` + +next you must update POstgres to use POSTGIS + +[https://docs.djangoproject.com/en/2.1/ref/contrib/gis/install/postgis/](https://docs.djangoproject.com/en/2.1/ref/contrib/gis/install/postgis/) + +## update postgres + +### the following packages must be installed: +(x.x matching the PostgreSQL version you want to install) + +postgresql-x.x +postgresql-x.x-postgis +postgresql-server-dev-x.x +python-psycopg2 + +find out the postgres server number: + +`apt list --installed | grep 'post'` + diff --git a/share/templates/readmes/paramiko.md b/share/templates/readmes/paramiko.md new file mode 100644 index 0000000..919e08f --- /dev/null +++ b/share/templates/readmes/paramiko.md @@ -0,0 +1 @@ +env LDFLAGS="-L$(brew --prefix openssl)/lib" CFLAGS="-I$(brew --prefix openssl)/include" pip install cryptography diff --git a/share/templates/readmes/regex.md b/share/templates/readmes/regex.md new file mode 100644 index 0000000..379a19d --- /dev/null +++ b/share/templates/readmes/regex.md @@ -0,0 +1,9 @@ +# useful regex commands + +## convert href to static + +`.s/href="\([^"]\+\)"/href="{% static '\1' %}"` + +use when **href="/some/link/file.css"** if the + +quotes are single quotes, then convert them to double \ No newline at end of file diff --git a/share/templates/readmes/sass.md b/share/templates/readmes/sass.md index 8841319..ce517ae 100644 --- a/share/templates/readmes/sass.md +++ b/share/templates/readmes/sass.md @@ -15,6 +15,8 @@ let g:syntastic_sass_sass_args=" -I /Full/share/media/projectwide_sass_files" ```` +you can also add multiple directories to the -I directive. Separate them with a space + [fixing syntastic check args](http://stackoverflow.com/questions/29041876/fixing-syntasticcheck-bootstrap-error-vim-syntastic-plugin) [syntastic manual](https://github.com/vim-syntastic/syntastic/blob/master/doc/syntastic.txt) \ No newline at end of file diff --git a/share/templates/readmes/set_the_key_file_on_local.md b/share/templates/readmes/set_the_key_file_on_local.md new file mode 100644 index 0000000..2af0f1a --- /dev/null +++ b/share/templates/readmes/set_the_key_file_on_local.md @@ -0,0 +1,15 @@ +`Host **my-remote-server**` *a symbolic name of your choice* + + +`Hostname **server-IP-or-DNS**` *the real name* + +`User **username**` *the username you use for login* + +`IdentityFile ~/.ssh/a_suitable_ssh_key_if_any` *for authentication purposes* + +`ControlMaster auto` + +`ControlPath ~/.ssh/%C` +*or* `~/.ssh/%h-%p-%r` *for older versions* + +`ControlPersist 5m` *or* `yes` *for never expiring persistent connection* \ No newline at end of file diff --git a/share/templates/readmes/setup_foundation.md b/share/templates/readmes/setup_foundation.md new file mode 100644 index 0000000..c0903dd --- /dev/null +++ b/share/templates/readmes/setup_foundation.md @@ -0,0 +1,2 @@ +node.js +https://lincolnloop.com/blog/installing-nodejs-and-npm-python-virtualenv/ diff --git a/share/templates/readmes/social_auth.md b/share/templates/readmes/social_auth.md new file mode 100644 index 0000000..defab85 --- /dev/null +++ b/share/templates/readmes/social_auth.md @@ -0,0 +1,96 @@ +# Social Auth notes + +## basic python_social_auth + +### How to Add Social Login to Django +[How to Add Social Login to Django](https://simpleisbetterthancomplex.com/tutorial/2016/10/24/how-to-add-social-login-to-django.html) + +#### pre test notes +1. disable 'test_social.middleware.ForwardHQMiddleware' +2. reset LOGIN_REDIRECT_URL = '/tsoc' or to wherever you want it to land +3. set SOCIAL_AUTH_URL_NAMESPACE = 'social' +4. DON'T use localhost if you plan on using twitter, twitter doesn't like it + +#### error in login +[stackoverflow desc](https://stackoverflow.com/questions/46975657/attributeerror-module-django-contrib-auth-views-has-no-attribute-login) + +change: +`path('login/', views.login, name='login')` +`path('login/', views.LoginView.as_view(template_name="registration/login.django"), name='login')` + +#### when making callback urls +- you can use localhost +- you can also specify a port on localhost, e.g. http://localhost:8026 +- you can also use minyanfinder.stg or any .stg + +- make sure to specify the "u" letter in oauth +- make sure to add the final forward slash "/" at the end of the callback url + +`minyanfinder.stg/oauth/complete/twitter/` + + + +## global variables +global variables should be in settings.py + +local info like the specific client id/secret key, etc, should be in the local setting sfiles + +test_social should only be activated in the local setting files + + +### twitter notes +1. you want to either use http or https, be specific +2. twitter does not use localhost and forwardhq is buggy +3. twitter DOES register 127.0.0.1:8026 as a callback + + +### facebook notes + +1. can use localhost +2. minaynfinder.stg is not working, need a certificate of some kind +3. forwardhq also not working + +#### using a domain + +##### create registered users + +Go to Roles->Test Users + +##### create the ssl +1. set a registered domain name to point at the ip address (ie ronnyabraham.com) +2. create a [https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs#generating-ssl-certificates](crt certificate) + +3. `Settings->Basic-> App Domains` add `ronnyabraham.com` +4. `Settings->Basic->Website/Site URL` add `https://ronnyabraham.com` +5. in Products->FacebookLogin->Settings add the following to "VAlid OAuth Redirect URIs" + +- https://ronnyabraham.com +- https://ronnyabraham.com/oauth/complete/facebook/ + +6. use letsencrypt to create ssl keys + + +### setting up facebook + +1. if used for testing make sure you set the app FOR TESTING you have to do this in the create app stage + +2. [select create test app from the app you like](https://imgur.com/RzK3pS3) + +3. go to settings->basic get the app-id and app-secret + +4. go to Settings-Basic-App Domains and add your FQDN (ie. mycrustyass.com) + +5. go to Settings-Basic-Website (it's near the bottom, add website if it isn't there), add the site URL with it's prefix ie http or https. + +Step 5 is VERY IMPORTANT b/c facebook doesn't recognize http prefix. +You must have SSL set up. But this is NOT the caes with localhost! + +6. in Settings-Products add Facebook Login then set Web OAuth Login to true +7. Settings-Products-Facebook Login add Valid OAuth redirect calls. + +Important! You must find out what the specific redirect call looks like. Once again, facebook doesn't require this for localhost, but for a named host it does + +e.g. https://mycrustyass.com/oauth/complete/facebook/ + +8. test this URL in the Settings-Products-FAcebook Login-Redirect URL Validator +