From 152b5658a92db467f1b1781e7d7a342654f9750f Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 20 Jul 2017 23:43:33 +0300 Subject: [PATCH 01/65] fix bug in deployment where pip gets updated during initial bootstrap,but then screws up the rest of the deployment including pip install. Break up the deployment into two parts. NOTE: must find a way to have deployment as one part alone Changes to be committed: modified: modules/deploy.py modified: modules/pip.py --- modules/deploy.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++ modules/pip.py | 2 +- 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/modules/deploy.py b/modules/deploy.py index 5acd042..6b9971b 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -193,6 +193,151 @@ def bootstrap(): db.init() +@task +def bootstrap_part1(): + # import database as db + + # configuration = env.config + + # if env.debug: + # logger = loggify('deploy', 'bootstrap_part1') + + # + # not doing a full sync, because we have to set up the rootpath, + # virtualenv, files, dir structure, etc. This means we aren't + # going to upload gunicorn and supervisor until after we've done + # everything else at the end of the bootstrapping process + + sync(full=False) + + # continue setting up the rootpath and virtualenv + setup_rootpath() + + setup_virtualenv() + + +@task +def bootstrap_part2(): + import database as db + + configuration = env.config + + if env.debug: + logger = loggify('deploy', 'bootstrap') + + # create the django project + from django import create_project + create_project() + + # + # link virtualenv to rootpath/private/virtualenv + + src_virtual = configuration.virtualenv.paths.root + dst_virtual = configuration.paths.server.virtual + + # + # link templates to rootpath/private/templates + + src_templates = configuration.paths.django.templates + dst_templates = configuration.paths.server.django.templates + + # + # link the django code in the project directory to the appropriate location + # in the rootpath directory + + src_code = configuration.paths.django.root + dst_code = configuration.paths.server.django.code + + # + # I corrected the linking code so that it deletes already existing + # links before creating them, otherwise you get really weird errors + # where the a link is recreated within the destination link + + from utils import link_create + + if env.debug: + logger.debug("virtualenv.root : %s" + % configuration.virtualenv.paths.root) + logger.debug("virtualenv.bin : %s\n" % + configuration.virtualenv.paths.bin) + + logger.debug("paths.server\n") + + logger.debug(" - root\t: %s" % configuration.paths.server.root) + + logger.debug(" - media\t: %s" % + configuration.paths.server.media.static) + + logger.debug(" - virtual\t: %s" % configuration.paths.server.virtual) + + logger.debug(" - django.code\t: %s\n" % + configuration.paths.server.django.code) + + logger.debug("django templates : %s" % + configuration.paths.django.templates) + + logger.debug("django root : %s" % + configuration.paths.django.root) + + logger.debug("django settings : %s" % + configuration.paths.django.settings.root) + + logger.debug("django local : %s" % + configuration.paths.django.settings.local) + + logger.debug(link_create(src_virtual, dst_virtual, debug=True)) + logger.debug(link_create(src_templates, dst_templates, debug=True)) + logger.debug(link_create(src_code, dst_code, debug=True)) + + else: + link_create(src_virtual, dst_virtual) + link_create(src_templates, dst_templates) + link_create(src_code, dst_code) + + # + # create and link the scripts that manage the server + # e.g. nginx, supervisor, gunicorn + + from nginx import upload as upload_nginx + from supervisor import upload as upload_supervisor + from django import generate as django_generate + + print_console("creating gunicorn script") + django_generate('gunicorn', True) + django_generate('local', True) + + print_console("creating supervisor script") + upload_supervisor() + + print_console("creating nginx script") + upload_nginx() + + # + # instantiate docker containers if any + + import docker + + print_console("check to see if docker containers are used") + + if hasattr(configuration, "docker"): + print_console("generating docker configuration file") + docker.generate() + + print_console("creating and starting docker container") + docker.create() + else: + print_console("no docker containers are being used. pass") + + # + # create and initialize the database + + print_console("in db.generate") + db.generate() + + print_console("in db.init") + db.init() + + @task def sync(full=True, extras=False): diff --git a/modules/pip.py b/modules/pip.py index 60e4b38..88b5ba1 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -58,7 +58,7 @@ def bootstrap_pip(): """ bootstraps pip """ - upgrade() + # upgrade() setup() From 6c757d0343cf5228e236532dacc8258d170583c1 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 3 Aug 2017 07:50:08 +0300 Subject: [PATCH 02/65] add regex.md file for useful regex commands Changes to be committed: new file: share/templates/readmes/regex.md --- share/templates/readmes/regex.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 share/templates/readmes/regex.md diff --git a/share/templates/readmes/regex.md b/share/templates/readmes/regex.md new file mode 100644 index 0000000..379a19d --- /dev/null +++ b/share/templates/readmes/regex.md @@ -0,0 +1,9 @@ +# useful regex commands + +## convert href to static + +`.s/href="\([^"]\+\)"/href="{% static '\1' %}"` + +use when **href="/some/link/file.css"** if the + +quotes are single quotes, then convert them to double \ No newline at end of file From 6277075e18b1c40c11f3ac95a2952b9bd2f04f90 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 8 Aug 2017 17:26:35 +0300 Subject: [PATCH 03/65] added some options for nginx Changes to be committed: modified: modules/initialize.py modified: modules/nginx.py new file: share/templates/readmes/paramiko.md --- modules/initialize.py | 3 +++ modules/nginx.py | 1 + share/templates/readmes/paramiko.md | 1 + 3 files changed, 5 insertions(+) create mode 100644 share/templates/readmes/paramiko.md diff --git a/modules/initialize.py b/modules/initialize.py index 2740a87..080af04 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -348,6 +348,9 @@ def get_config(branchname): dataobject.server.addbranch('nginx') dataobject.server.nginx.port = config['nginx']['port'] + dataobject.server.nginx.socket = config['nginx']['socket'] + + print "DEBUG -- socket: %s" % dataobject.server.nginx.socket if config['project']['host'] == "localhost": dataobject.server.nginx.host = "{projectname}.{ext}".format( diff --git a/modules/nginx.py b/modules/nginx.py index c62f341..f90fc8f 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -98,6 +98,7 @@ def upload(): context['error_log'] = configuration.logging.nginx.error context['port'] = configuration.server.nginx.port + context['socket'] = configuration.server.nginx.socket context['django_host'] = configuration.server.django.host context['django_port'] = configuration.server.django.port diff --git a/share/templates/readmes/paramiko.md b/share/templates/readmes/paramiko.md new file mode 100644 index 0000000..919e08f --- /dev/null +++ b/share/templates/readmes/paramiko.md @@ -0,0 +1 @@ +env LDFLAGS="-L$(brew --prefix openssl)/lib" CFLAGS="-I$(brew --prefix openssl)/include" pip install cryptography From 2c19ba589b039988d7b60ba40b02df2f60e7a5e3 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 28 Aug 2017 23:53:26 +0300 Subject: [PATCH 04/65] updated help message modified: modules/django.py --- modules/django.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/django.py b/modules/django.py index 3a08ebd..deb2969 100644 --- a/modules/django.py +++ b/modules/django.py @@ -369,6 +369,11 @@ def generate_scripts(template_name, make_copy=False): @task def generate(param=None, make_copy=False): + """ + param can be one of settings, local, wsgi + make_copy must be set to True if you want it to actually do anthing + """ + SCRIPT_LIST = ['settings', 'local', 'wsgi'] PARAM_LIST = list(SCRIPT_LIST) PARAM_LIST.append('gunicorn') @@ -384,6 +389,7 @@ def generate(param=None, make_copy=False): print "django:generate make_copy : %s\n" % make_copy print "django:generate script : %s" % param + if env.debug: print "django:generate script : %s" % param print "django:generate make_copy : %s\n" % make_copy @@ -396,7 +402,6 @@ def generate(param=None, make_copy=False): if not param: # this is where script=None, generate all scripts - for scriptkey in SCRIPT_LIST: generate_scripts(scriptkey, make_copy) From 6691cd1af8689038482bf2bb616d7663f8efafbc Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 28 Aug 2017 23:53:42 +0300 Subject: [PATCH 05/65] changed ALLOWED_SITES to include whatever we call the host site Changes to be committed: modified: share/templates/conf/django/files/local.jinja2 --- share/templates/conf/django/files/local.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index 980b10f..bc06c78 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = [configuration.project.extendedname, ] +ALLOWED_HOSTS = [configuration.project.extendedname, configuration.project.host] # ----------------------------------------- # Debug logging to the console From 200924b6d8ec143fc97be76d2235d2accec2a3d3 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 29 Aug 2017 00:07:47 +0300 Subject: [PATCH 06/65] add check to see if the 'socket' or 'port' options have been set in the configuration file before trying to load it in initilaize.py --- modules/initialize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index 080af04..38ba3f8 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -347,10 +347,12 @@ def get_config(branchname): dataobject.server.addbranch('nginx') - dataobject.server.nginx.port = config['nginx']['port'] - dataobject.server.nginx.socket = config['nginx']['socket'] + if 'port' in config['nginx']: + dataobject.server.nginx.port = config['nginx']['port'] - print "DEBUG -- socket: %s" % dataobject.server.nginx.socket + if 'socket' in config['nginx']: + dataobject.server.nginx.socket = config['nginx']['socket'] + print "DEBUG -- socket: %s" % dataobject.server.nginx.socket if config['project']['host'] == "localhost": dataobject.server.nginx.host = "{projectname}.{ext}".format( From 9c8f83cc38d53406ba6b479f4a06ed545580207b Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 29 Aug 2017 02:32:23 +0300 Subject: [PATCH 07/65] go back to original settings for config/nginx/static modified: modules/initialize.py --- modules/initialize.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index 38ba3f8..edfe21f 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -347,12 +347,10 @@ def get_config(branchname): dataobject.server.addbranch('nginx') - if 'port' in config['nginx']: - dataobject.server.nginx.port = config['nginx']['port'] + dataobject.server.nginx.port = config['nginx']['port'] - if 'socket' in config['nginx']: - dataobject.server.nginx.socket = config['nginx']['socket'] - print "DEBUG -- socket: %s" % dataobject.server.nginx.socket + dataobject.server.nginx.socket = config['nginx']['socket'] + print "DEBUG -- socket: %s" % dataobject.server.nginx.socket if config['project']['host'] == "localhost": dataobject.server.nginx.host = "{projectname}.{ext}".format( From 2237dd48560beaa17ca504e66dd95b2274b18b70 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 9 Nov 2017 00:33:30 +0200 Subject: [PATCH 08/65] added help message for django.startapp --- modules/django.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/django.py b/modules/django.py index 3a08ebd..b9cc655 100644 --- a/modules/django.py +++ b/modules/django.py @@ -129,23 +129,35 @@ def run(args=None): @task -def startapp(args): +def startapp(appname='help'): """ wrapper for the django.startapp takes name of app and creates in in code/apps - args - name of app + appname - name of app """ configuration = env.config - destination = os.path.join(configuration.paths.django.apps, args) + msg_help = """ + django.startapp takes one of two values: + \thelp - this help message + \tappname - the name of the app you want to start + """ + + from utils import handle_help + import sys + + if handle_help(appname, msg_help, 'help'): + sys.exit() + + destination = os.path.join(configuration.paths.django.apps, appname) cmd_mkdir = "mkdir {destination}".format( destination=destination) command = "startapp {appname} {destination}".format( - appname=args, + appname=appname, destination=destination) fabric_ops.run(cmd_mkdir) From 9c31d26156185ab6bab00847333817385c47cdcb Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 17 Sep 2018 23:47:52 +0300 Subject: [PATCH 09/65] added socket variable to nginx for the development and staging templates --- share/templates/meta/development.yml | 1 + share/templates/meta/staging.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/share/templates/meta/development.yml b/share/templates/meta/development.yml index 077788f..1ea9a1e 100644 --- a/share/templates/meta/development.yml +++ b/share/templates/meta/development.yml @@ -32,6 +32,7 @@ media: nginx: paths: {available: null, enabled: servers, root: /usr/local/etc/nginx} port: NGINX_PORT + socket: False overrides: [null] project: branch: development diff --git a/share/templates/meta/staging.yml b/share/templates/meta/staging.yml index 593ee01..418299c 100644 --- a/share/templates/meta/staging.yml +++ b/share/templates/meta/staging.yml @@ -24,6 +24,7 @@ media: nginx: paths: {available: sites-available, enabled: sites-enabled, root: /etc/nginx} port: NGINX_PORT + socket: True overrides: [null] project: branch: BRANCH_NAME From ba9b5875f2c981fe25b9b4af9e60a11dff75c63a Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 10 Oct 2018 13:05:17 +0300 Subject: [PATCH 10/65] added debugging to deploy.ssh --- modules/deploy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/deploy.py b/modules/deploy.py index 6b9971b..d2fad01 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -727,6 +727,9 @@ def remove(full=True): def ssh(): configuration = env.config + if env.debug: + logger = loggify('deploy', 'ssh') + if configuration.project.ssh: cmd_ssh = "ssh {user}@{host} -i {sshkey}".format( user=configuration.project.user, @@ -737,4 +740,7 @@ def ssh(): user=configuration.project.user, host=configuration.project.host) - local(cmd_ssh) + if env.debug: + logger.debug("ssh command: %s" % cmd_ssh) + else: + local(cmd_ssh) From 35cdb1fe13d6446919e2279e04a0acfc625207b9 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 10 Oct 2018 13:06:02 +0300 Subject: [PATCH 11/65] added the ability to specify what machine we want to edit files on (probably should be removed) --- modules/maintenance.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/modules/maintenance.py b/modules/maintenance.py index 2d1fc94..18eea22 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -4,7 +4,7 @@ import os import sys import utils -from utils import executize, virtualenv +from utils import executize, virtualenv, loggify def command(program=None, cmd=None, extra_param=None): @@ -65,21 +65,31 @@ def command(program=None, cmd=None, extra_param=None): _execute(_command) -def edit(remote_path): +def edit(remote_path, host_string=None): """ calls up mvim or vim on the file remote_path - path to file we want to edit + host_string - what machine the file is located on + + host_string is necessary b/c in the case where I want to edit + configuration files, it is essential to do so only on the localhost + + that is, whether I'm editing staging, development or production project + yml files, the place to do that editing is ALWAYS on localhost, this is + because all fabric commands are executed from localhost and will check for + the necessary configuration files on local (I'm not running fabric from an + ssh command line on the aws server, I'm running it on my laptop) """ - # logger = loggify('maintenance', 'edit') + logger = loggify('maintenance', 'edit') # configuration = env.config if env.debug: - # logger.debug("remote_path : %s" % remote_path) - # logger.debug("env.host_string : %s" % env.host_string) - # logger.debug("sys.platform : %s" % sys.platform) + logger.debug("remote_path : %s" % remote_path) + logger.debug("env.host_string : %s" % env.host_string) + logger.debug("sys.platform : %s" % sys.platform) pass else: @@ -88,10 +98,13 @@ def edit(remote_path): else: editor = "vim" + if not host_string: + host_string = env.host_string + cmd_edit = "{editor} sftp://{user}@{host_string}/{remote_path}".format( editor=editor, user=env.user, - host_string=env.host_string, + host_string=host_string, remote_path=remote_path) local(cmd_edit) From 0fd0bbe7250930c98f3c79a2851c71eac8b0abe3 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 10 Oct 2018 13:06:19 +0300 Subject: [PATCH 12/65] added edit project configuration files capability --- modules/conf_setup.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/modules/conf_setup.py b/modules/conf_setup.py index d402443..2fb8eac 100644 --- a/modules/conf_setup.py +++ b/modules/conf_setup.py @@ -257,3 +257,57 @@ def copy_directories(source_path, dest_path): fpath = os.path.join(source_path, fname) put(fpath, dest_path) + + +@task +def edit(param='help'): + """ + calls up mvim on the yaml project configuration files + """ + + from maintenance import edit as maintenance_edit + from maintenance import _get_configuration_path + + locations = { + 'development': { + 'path': _get_configuration_path("config", "development"), + 'desc': 'development project configuration file', + }, + + 'staging': { + 'path': _get_configuration_path("config", "staging"), + 'desc': 'staging project configuration file', + }, + + 'production': { + 'path': _get_configuration_path("config", "production"), + 'desc': 'production project configuration file', + }, + } + + if param in locations.keys(): + # it is unnecessary to give the host_string to edit + # because the host_string is determined when we call fab + # ie. "fab devel whatever" or "fab stage whatever" + # we want to edit all files on localhost. b/c we only call + # fab FROM localhost + + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print """ + "fab configuration.edit" automates opening up and editing project + configuration files + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """ + + for k_loc in locations.keys(): + print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + + return From 954d67fa32e52c0c2e99bb63037fc3ae8d1d1e87 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 10 Oct 2018 13:33:46 +0300 Subject: [PATCH 13/65] changed deploy.ssh from using sudo to local b/c sudo runs on the remote machine and I want ssh to be run off the local to connect to the remote --- modules/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/deploy.py b/modules/deploy.py index d2fad01..0d1a102 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -1,5 +1,6 @@ from fabric.api import env, task -from fabric.operations import run, local +from fabric.operations import run +from fabric.operations import local # from fabric.contrib.files import upload_template # from utils import print_run From ef95ecfebcff5b06f6909c4dc0383226d3b2b777 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 10 Oct 2018 13:35:29 +0300 Subject: [PATCH 14/65] updated readme on aws to show how to pipe info into osx clipboard when using ssh-keygen --- share/templates/readmes/aws.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/templates/readmes/aws.md b/share/templates/readmes/aws.md index 64582ea..dde69de 100644 --- a/share/templates/readmes/aws.md +++ b/share/templates/readmes/aws.md @@ -147,8 +147,9 @@ replace it with: ssh-keygen -y -f /path/to/myinstance.pem -2. copy the above results +2. copy the above results. on osx this would look like: + ssh-keygen -y -f /path/to/myinstance.pem | pbcopy 3. log in to the ubuntu instance using default ubuntu user 4. change users to the custom username From d9e5f5edd13cc72bb96e24eeeaa92f86c2647989 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 16 Oct 2018 03:03:20 +0300 Subject: [PATCH 15/65] added code to check if the allowed_hosts param was set in the configuration file, if it was, then add the extra hosts to allowed hosts --- modules/initialize.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/initialize.py b/modules/initialize.py index edfe21f..26aecda 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -197,6 +197,15 @@ def get_config(branchname): dataobject.project.user = config['project']['user'] dataobject.project.group = config['project']['group'] + dataobject.project.allowedhosts = list() + _allowed = getattr(dataobject.project, 'allowedhosts') + _allowed.append(dataobject.project.extendedname) + _allowed.append(dataobject.project.host) + + if 'allowed_hosts' in config['project']: + for allowed in config['project']['allowed_hosts']: + _allowed.append(allowed) + dataobject.project.sudo = config['project']['sudo'] # From d57781436b39f3f220d81fcba99528b786b539db Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 16 Oct 2018 03:04:13 +0300 Subject: [PATCH 16/65] use the allowedhosts variable in place of the other two. even if allowed_hosts is not set in conf, the host extended host name will be used anyway --- share/templates/conf/django/files/local.jinja2 | 2 +- share/templates/conf/gunicorn/files/local.jinja2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index bc06c78..96c3f20 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = [configuration.project.extendedname, configuration.project.host] +ALLOWED_HOSTS = [configuration.project.allowedhosts] # ----------------------------------------- # Debug logging to the console diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index 980b10f..96c3f20 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = [configuration.project.extendedname, ] +ALLOWED_HOSTS = [configuration.project.allowedhosts] # ----------------------------------------- # Debug logging to the console From b93ea8e2591c39aab73dbc615f1e99c2e93a1577 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 16 Oct 2018 03:14:10 +0300 Subject: [PATCH 17/65] fixed some print statement to be python3 compatible --- modules/django.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/django.py b/modules/django.py index 55d3737..b9290ad 100644 --- a/modules/django.py +++ b/modules/django.py @@ -399,12 +399,12 @@ def generate(param=None, make_copy=False): import sys sys.exit(err_msg) - print "django:generate make_copy : %s\n" % make_copy - print "django:generate script : %s" % param + print("django:generate make_copy : %s\n" % make_copy) + print("django:generate script : %s" % param) if env.debug: - print "django:generate script : %s" % param - print "django:generate make_copy : %s\n" % make_copy + print("django:generate script : %s" % param) + print("django:generate make_copy : %s\n" % make_copy) else: pass # env.debug does not block the rest of the commands because this From 4ccbe25f251f12c34d274e099e548d6effa13f81 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 16 Oct 2018 03:26:03 +0300 Subject: [PATCH 18/65] the allowedhosts value was a list, and I was putting it inside another list [ ] set to ALLOWED_HOSTS --- share/templates/conf/django/files/local.jinja2 | 2 +- share/templates/conf/gunicorn/files/local.jinja2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index 96c3f20..4454550 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = [configuration.project.allowedhosts] +ALLOWED_HOSTS = configuration.project.allowedhosts # ----------------------------------------- # Debug logging to the console diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index 96c3f20..4454550 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = [configuration.project.allowedhosts] +ALLOWED_HOSTS = configuration.project.allowedhosts # ----------------------------------------- # Debug logging to the console From 33b11f3df845be911238ee95a3b6d698c2f93601 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 16 Oct 2018 03:42:26 +0300 Subject: [PATCH 19/65] added help message to django.generate --- modules/django.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/modules/django.py b/modules/django.py index b9290ad..4d9af9b 100644 --- a/modules/django.py +++ b/modules/django.py @@ -380,7 +380,7 @@ def generate_scripts(template_name, make_copy=False): @task -def generate(param=None, make_copy=False): +def generate(param="help", make_copy=False): """ param can be one of settings, local, wsgi make_copy must be set to True if you want it to actually do anthing @@ -392,12 +392,28 @@ def generate(param=None, make_copy=False): make_copy = booleanize(make_copy) - if param and param not in PARAM_LIST: - err_msg = "You asked to generate %s, that value isn't available" \ - " possible script values available: %s" % (param, PARAM_LIST) + if param: - import sys - sys.exit(err_msg) + err_msg = None + + if param == "help": + err_msg = """ + + fab django.generate:param="help",make_copy=False + + param - file to be generated can be of type: %s + + make_copy - determines whether or generate copies the generated file + into it's practical location (otherwise doesn't do anything) + """ % PARAM_LIST + + elif param not in PARAM_LIST: + err_msg = "You asked to generate %s, that value isn't available" \ + " possible script values available: %s" % (param, PARAM_LIST) + + if err_msg: + import sys + sys.exit(err_msg) print("django:generate make_copy : %s\n" % make_copy) print("django:generate script : %s" % param) From a97698ae9a81391bd98402d178f89fea95eca18f Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 22:48:54 +0300 Subject: [PATCH 20/65] fixed print statements to be in python3 --- bin/deploy_meta.py | 14 +++--- bin/test.py | 12 ++--- fabfile.py | 6 +-- modules/conf_setup.py | 6 +-- modules/configuration_setup.py | 6 +-- modules/database.py | 39 +++++++-------- modules/deploy.py | 89 +++++++++++++++++----------------- modules/django.py | 62 +++++++++++------------ modules/docker.py | 8 +-- modules/initialize.py | 6 +-- modules/maintenance.py | 16 +++--- modules/pip.py | 4 +- modules/supervisor.py | 2 +- modules/utils.py | 10 ++-- 14 files changed, 141 insertions(+), 139 deletions(-) diff --git a/bin/deploy_meta.py b/bin/deploy_meta.py index 01a42b1..c4d2c3c 100644 --- a/bin/deploy_meta.py +++ b/bin/deploy_meta.py @@ -263,11 +263,11 @@ class DeployMeta(QtGui.QMainWindow): def load_config(self, configname): if configname not in self.CONFIG_TYPES.keys(): - print "\nerror, load_config was called with parameter: {confname}," \ - "which is not a legitimate value in CONFIG TYPES." \ - "\nLegitimate values are {configvalues}".format( - confname=configname, - configvalues=self.CONFIG_TYPES.keys()) + print("\nerror, load_config was called with parameter: {confname}," + "which is not a legitimate value in CONFIG TYPES." + "\nLegitimate values are {configvalues}".format( + confname=configname, + configvalues=self.CONFIG_TYPES.keys())) sys.exit() path_config_full = os.path.join( @@ -417,7 +417,7 @@ class DeployMeta(QtGui.QMainWindow): " default config types") retval = msg.exec_() - print "value of qmessagebox in action_save: %s" % retval + print("value of qmessagebox in action_save: %s" % retval) def setupMenu(self): menubar = self.menuBar() @@ -482,7 +482,7 @@ def main(): ex = DeployMeta() sys.exit(app.exec_()) - print ex + print(ex) if __name__ == '__main__': diff --git a/bin/test.py b/bin/test.py index efb3043..74b0592 100644 --- a/bin/test.py +++ b/bin/test.py @@ -10,8 +10,8 @@ parser.add_argument('--file2', default='test.yml') def findDiff(d1, d2, path=""): for k in d1.keys(): if not (k in d2): - print path, ":" - print k + " as key not in d2", "\n" + print(path, ":") + print(k + " as key not in d2", "\n") else: if type(d1[k]) is dict: if path == "": @@ -21,9 +21,9 @@ def findDiff(d1, d2, path=""): findDiff(d1[k], d2[k], path) else: if d1[k] != d2[k]: - print path, ":" - print " - ", k, " : ", d1[k] - print " + ", k, " : ", d2[k] + print(path, ":") + print(" - ", k, " : ", d1[k]) + print(" + ", k, " : ", d2[k]) def compare_dictionaries(dict_1, dict_2, dict_1_name, dict_2_name, path=""): @@ -85,6 +85,6 @@ def main(): yaml_test = ruamel.yaml.load(file(path_test, 'r')) a = compare_dictionaries(yaml_main, yaml_test, 'main', 'test') - print a + print(a) main() diff --git a/fabfile.py b/fabfile.py index f05fe7c..919d05d 100644 --- a/fabfile.py +++ b/fabfile.py @@ -17,7 +17,7 @@ branch = "development" def setup(): - print "setup fabric configuration files" + print("setup fabric configuration files") def all(): @@ -48,7 +48,7 @@ def stage(): global branch branch = "staging" - print "in staging" + print("in staging") all() @@ -62,7 +62,7 @@ def prod(): global branch branch = "production" - print "in production" + print("in production") all() diff --git a/modules/conf_setup.py b/modules/conf_setup.py index 2fb8eac..1212c2e 100644 --- a/modules/conf_setup.py +++ b/modules/conf_setup.py @@ -11,7 +11,7 @@ def deploy(param=None): param_list = ['conf', 'readmes'] if not param: - print "this requires input param must be one of %s" % param_list + print("this requires input param must be one of %s" % param_list) import sys sys.exit() @@ -26,7 +26,7 @@ def backup(param=None): param_list = ['conf', 'meta', 'readmes'] if not param: - print "this requires input param must be one of %s" % param_list + print("this requires input param must be one of %s" % param_list) import sys sys.exit() @@ -308,6 +308,6 @@ def edit(param='help'): """ for k_loc in locations.keys(): - print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) return diff --git a/modules/configuration_setup.py b/modules/configuration_setup.py index 22dd99a..be3aef6 100644 --- a/modules/configuration_setup.py +++ b/modules/configuration_setup.py @@ -31,7 +31,7 @@ Error. maintenance.exists_dir_sub takes a 'section' parameter value. Valid options include:""" % section for key in configuration.templates.keys(): - print " %s" % key + print(" %s" % key) print """ Please run the command again, but this time enter a valid section value. @@ -54,8 +54,8 @@ def exists_dir_top(): # located files, since those files get rsync'd anyway. if env.debug: - print "maintenance.exists_dir_top -- checking for " \ - "directory:\n\t%s\n" % configuration.paths.conf.local + print("maintenance.exists_dir_top -- checking for " + "directory:\n\t%s\n" % configuration.paths.conf.local) return exists(configuration.paths.conf.local) diff --git a/modules/database.py b/modules/database.py index ecda262..3ae06a2 100644 --- a/modules/database.py +++ b/modules/database.py @@ -78,7 +78,7 @@ def generate_sql(script_name): # with open(build_path, "w") as output: # output.write(rendered) - print NOTE + print(NOTE) def execute_sql(script_name, add_dbname=True, is_admin=False): @@ -121,7 +121,7 @@ def execute_sql(script_name, add_dbname=True, is_admin=False): else: run_database_command(psql_command, user) - print NOTE + print(NOTE) @task @@ -138,7 +138,7 @@ def generate(): @task def clear_scripts(): - print "this does nothing, the code isn't here" + print("this does nothing, the code isn't here") @task @@ -200,7 +200,7 @@ def commandline(dbuser='default'): host = configuration.server.database.host port = configuration.server.database.port - print "debug -- dbuser: %s" % dbuser + print("debug -- dbuser: %s" % dbuser) if dbuser == 'admin': user = configuration.server.database.admin.user @@ -222,7 +222,7 @@ def commandline(dbuser='default'): user=user ) - print "debug -- cmd_commandline: %s" % cmd_commandline + print("debug -- cmd_commandline: %s" % cmd_commandline) run_database_command(cmd_commandline, user) @@ -270,7 +270,7 @@ def backup(dbuser='default', backup_file=None, branch=None, timefilename = os.path.join( configuration.paths.server.backups.database, backup_file_time) - print "debug timefilename -- %s" % timefilename + print("debug timefilename -- %s" % timefilename) dumpfilename = os.path.join( configuration.paths.server.backups.database, @@ -278,7 +278,7 @@ def backup(dbuser='default', backup_file=None, branch=None, datadump_only = booleanize(datadump_only) - print "dumpfilename: %s" % dumpfilename + print("dumpfilename: %s" % dumpfilename) if not datadump_only: @@ -307,7 +307,7 @@ def backup(dbuser='default', backup_file=None, branch=None, hide_output = False - print "cmd_pg_dump: %s" % cmd_pg_dump + print("cmd_pg_dump: %s" % cmd_pg_dump) output = run_database_command(cmd_pg_dump, user, hide=hide_output) @@ -349,7 +349,7 @@ def restore(dbuser='default', backup_file=None, reinitialize=True): timefilename = os.path.join( configuration.paths.server.backups.database, backup_file_time) - print "debug -- timefilename: %s" % timefilename + print("debug -- timefilename: %s" % timefilename) dumpfilename = os.path.join( configuration.paths.server.backups.database, @@ -390,26 +390,26 @@ def sync(src): configuration_src = initialize.environment(branch_src) configuration_dst = configuration - print "branch_src: %s" % configuration_src.project.branch - print "branch_dst: %s" % configuration_dst.project.branch + print("branch_src: %s" % configuration_src.project.branch) + print("branch_dst: %s" % configuration_dst.project.branch) # backup all files with names that wont interfere output_name_dst = "output_sync_dst.sql" backup_name_dst = "backup_sync_dst.sql" backup_name_src = "backup_sync_src.sql" - print "output_src = backup(" - print "\tbackup_file=%s," % backup_name_src - print "\tbranch=%s)" % configuration_src.project.branch + print("output_src = backup(") + print("\tbackup_file=%s," % backup_name_src) + print("\tbranch=%s)" % configuration_src.project.branch) # dump the source database to our {destination}/backups/database - print "backup up src: %s" % configuration_src.project.branch + print("backup up src: %s" % configuration_src.project.branch) output_src = backup( backup_file=backup_name_src, branch=configuration_src.project.branch) - print "backing up dst: %s" % configuration_dst.project.branch + print("backing up dst: %s" % configuration_dst.project.branch) backup( backup_file=backup_name_dst, branch=configuration_dst.project.branch) @@ -492,8 +492,8 @@ def get_template_path(script_name, script_type): 'files', file_template) else: - print "Error, you passed the variable %s, must pass" \ - "either 'build' or 'template'" % script_type + print("Error, you passed the variable %s, must pass" + "either 'build' or 'template'" % script_type) import sys sys.exit() @@ -565,7 +565,8 @@ def edit(param='help'): """ for k_loc in locations.keys(): - print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + print("\t{0: <20} - {1}".format( + k_loc, locations[k_loc]['desc'])) return diff --git a/modules/deploy.py b/modules/deploy.py index 0d1a102..0a8c5d6 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -378,31 +378,31 @@ def sync(full=True, extras=False): "build/*", "/extras", "/opt") if env.debug: - print "\n%s debug: %s" % (debug_prefix, env.debug) + print("\n%s debug: %s" % (debug_prefix, env.debug)) - print "\n%s project.name %s" \ - % (debug_prefix, configuration.project.name) + print("\n%s project.name %s" % + (debug_prefix, configuration.project.name)) - print "%s project.branch %s" \ - % (debug_prefix, configuration.project.branch) + print("%s project.branch %s" % + (debug_prefix, configuration.project.branch)) - print "%s path.project %s" \ - % (debug_prefix, configuration.paths.project.root) + print("%s path.project %s" % + (debug_prefix, configuration.paths.project.root)) - print "\n%s run(%s)" % (debug_prefix, command_mkdir_remote) + print("\n%s run(%s)" % (debug_prefix, command_mkdir_remote)) - print "\n{debug_prefix} rsync_project(\n\tremote_dir={remote_dir}," \ - "\n\tlocal_dir={local_dir},\n\texclude={excludeitems})".format( - debug_prefix=debug_prefix, - remote_dir=remote_dir, - local_dir=local_dir, - excludeitems=excludeitems) + print("\n{debug_prefix} rsync_project(\n\tremote_dir={remote_dir}," + "\n\tlocal_dir={local_dir},\n\texclude={excludeitems})".format( + debug_prefix=debug_prefix, + remote_dir=remote_dir, + local_dir=local_dir, + excludeitems=excludeitems)) - # print "\n%s override: %s " \ - # % (debug_prefix, configuration.overrides.keys()) + # print("\n%s override: %s " % + # (debug_prefix, configuration.overrides.keys())) - # print "%s has overrides? %s" \ - # % (debug_prefix, (len(configuration.overrides.keys()) > 0)) + # print("%s has overrides? %s" % + # (debug_prefix, (len(configuration.overrides.keys()) > 0))) else: @@ -442,8 +442,8 @@ def sync(full=True, extras=False): return - # print "remote_dir: %s" % remote_dir - # print "local_dir: %s" % local_dir + # print("remote_dir: %s" % remote_dir) + # print("local_dir: %s" % local_dir) # import sys # sys.exit() @@ -486,7 +486,7 @@ def media(source_branch): import initialize - print "project.branch: %s" % configuration.project.branch + print("project.branch: %s" % configuration.project.branch) # # if we use initialize.environemnt, we will overwrite all project @@ -495,13 +495,14 @@ def media(source_branch): configuration_src = initialize.get_config(source_branch) configuration_dst = configuration - print "branch_src: %s" % configuration_src.project.branch - print "branch_dst: %s" % configuration_dst.project.branch + print("branch_src: %s" % configuration_src.project.branch) + print("branch_dst: %s" % configuration_dst.project.branch) - print "src - server_media_dynamic: %s" % \ - configuration_src.paths.server.media.dynamic - print "dst - server_media_dynamic: %s" % \ - configuration_dst.paths.server.media.dynamic + print("src - server_media_dynamic: %s" % + configuration_src.paths.server.media.dynamic) + + print("dst - server_media_dynamic: %s" % + configuration_dst.paths.server.media.dynamic) # add a trailing slash to the directories dynamic_src = "%s/" % configuration_src.paths.server.media.dynamic @@ -532,7 +533,7 @@ def media(source_branch): path_dst=dynamic_dst, ) - print cmd_rsync + print(cmd_rsync) upstream = True remote_dir = dynamic_dst @@ -548,30 +549,30 @@ def media(source_branch): # # rsync relies on env.host_string to determine what the username and # host is - # print "before: %s" % env.host_string + # print("before: %s" % env.host_string) # configuration_dst = initialize.environment(source_branch) - # print "after: %s" % env.host_string + # print("after: %s" % env.host_string) # import sys # sys.exit() - print "upstream: %s" % upstream - print "remote_dir: %s" % remote_dir - print "local_dir: %s" % local_dir - print "\ncopy from {src} to {dst}\n".format( + print("upstream: %s" % upstream) + print("remote_dir: %s" % remote_dir) + print("local_dir: %s" % local_dir) + print("\ncopy from {src} to {dst}\n".format( src=configuration_src.project.host, - dst=configuration_dst.project.host) + dst=configuration_dst.project.host)) elif configuration_dst.project.host == "localhost": remote_dir = dynamic_src local_dir = dynamic_dst upstream = False - print "upstream: %s" % upstream - print "remote_dir: %s" % remote_dir - print "local_dir: %s" % local_dir - print "\ncopy from {src} to {dst}\n".format( + print("upstream: %s" % upstream) + print("remote_dir: %s" % remote_dir) + print("local_dir: %s" % local_dir) + print("\ncopy from {src} to {dst}\n".format( src=configuration_src.project.host, - dst=configuration_dst.project.host) + dst=configuration_dst.project.host)) else: print_console("no moving media files from staging to production") return @@ -582,7 +583,7 @@ def media(source_branch): local_dir=local_dir, upstream=upstream) - print cmd_msg + print(cmd_msg) from fabric.contrib.project import rsync_project rsync_project(remote_dir=remote_dir, @@ -597,9 +598,9 @@ def test(): projectpath = configuration.paths.project.root local_dir = configuration.paths.project.local + "/" - print hasattr(configuration, 'docker') - print "project path : %s" % projectpath - print "local dir : %s" % local_dir + print(hasattr(configuration, 'docker')) + print("project path : %s" % projectpath) + print("local dir : %s" % local_dir) @task diff --git a/modules/django.py b/modules/django.py index 4d9af9b..1d697a9 100644 --- a/modules/django.py +++ b/modules/django.py @@ -31,8 +31,8 @@ def generate_secret_key(): @task def test(args=None): - print "debug - testing checkapp(sorl.thumbnail): %s" % \ - check_app("sorl.thumbnail") + print("debug - testing checkapp(sorl.thumbnail): %s" % + check_app("sorl.thumbnail")) @task @@ -48,7 +48,7 @@ def commandline(args=""): args=args, djangosettings=configuration.imports.settings,) - print commandline + print(commandline) @task @@ -174,8 +174,8 @@ def installed_apps(): """ configuration = env.config - printecho = "print '\\n'" - printcommand = "print '\\n'.join([ item for item" \ + printecho = "print('\\n')" + printcommand = "print('\\n').join([ item for item" \ " in {settings}.INSTALLED_APPS])".format( settings=configuration.imports.settings) @@ -354,29 +354,29 @@ def generate_scripts(template_name, make_copy=False): fabric_ops.run(copy_cmd) - print "\n\n------------------------------" - print "project_name : %s" % project_name - print "project_branch : %s" % project_branch - print "project_path : %s" % project_path - print "template_name : %s" % template_name - print "build_path : %s" % build_path - print "files_path : %s" % files_path - print "files_name : %s" % files_name + print("\n\n------------------------------") + print("project_name : %s" % project_name) + print("project_branch : %s" % project_branch) + print("project_path : %s" % project_path) + print("template_name : %s" % template_name) + print("build_path : %s" % build_path) + print("files_path : %s" % files_path) + print("files_name : %s" % files_name) - print "copy_path : %s" % copy_path - print "copy_full_path : %s" % copy_full_path - print "build_name : %s" % build_name + print("copy_path : %s" % copy_path) + print("copy_full_path : %s" % copy_full_path) + print("build_name : %s" % build_name) upload_msg = utils_upload_template( filename=files_name, destination=build_path, context=context, use_jinja=True, use_sudo=False, backup=True, template_dir=files_path, debug=True) - print "upload_msg : %s" % upload_msg - print "make_copy : %s" % make_copy - print "copy_cmd : %s" % copy_cmd - print "backup_cmd : %s" % backup_cmd - print "------------------------------\n\n" + print("upload_msg : %s" % upload_msg) + print("make_copy : %s" % make_copy) + print("copy_cmd : %s" % copy_cmd) + print("backup_cmd : %s" % backup_cmd) + print("------------------------------\n\n") @task @@ -524,12 +524,12 @@ def generate_gunicorn(make_link=True): template_dir=files_path) if make_link: - print "\nlinking the generating gunicorn file in conf to " \ - "the server diretory\n" + print("\nlinking the generating gunicorn file in conf to " + "the server diretory\n") fabric_ops.run(msg_link_gunicorn) else: - print "\nNOTE: not linking the generated gunicorn file" \ - "to the server directory\n" + print("\nNOTE: not linking the generated gunicorn file" + "to the server directory\n") @task @@ -610,7 +610,7 @@ def edit(param='help'): """ for k_loc in locations.keys(): - print "\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc']) + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) return @@ -669,7 +669,7 @@ def clearmigrations(appname="help"): @task def makemigrations_empty(param="help"): if param == "help": - print "print this help message" + print("print this help message") return manage("makemigrations --empty %s" % param) @@ -686,14 +686,14 @@ def fixtures(appname=None, backup=False): # booleanize the backup parameter backup = booleanize(backup) - print "debug - appname: %s" % appname + print("debug - appname: %s" % appname) # from fabric.api import * path_data = configuration.paths.django.fixtures path_backup = configuration.paths.server.backups.fixtures - print "debug - path_backup: %s" % path_backup + print("debug - path_backup: %s" % path_backup) if appname is not None: path_data = os.path.join(path_data, appname) @@ -710,7 +710,7 @@ def fixtures(appname=None, backup=False): else: path_fixture = os.path.join(path_data, fixture_name) - print "debug - path_fixture: %s" % path_fixture + print("debug - path_fixture: %s" % path_fixture) from utils import ensure_dir ensure_dir(path_data) @@ -760,7 +760,7 @@ def loaddata(param=None): configuration = env.config if param is None: - print "you must specify an appname to load the fixture to" + print("you must specify an appname to load the fixture to") return else: appname = param diff --git a/modules/docker.py b/modules/docker.py index 2a05d73..d99951e 100644 --- a/modules/docker.py +++ b/modules/docker.py @@ -101,7 +101,7 @@ def generate(): import pprint pp = pprint.PrettyPrinter(indent=4) - print "context:\n" + print("context:\n") pp.pprint(context) else: @@ -176,7 +176,7 @@ def start(create=False): configuration.docker.database.container_name if env.debug: - print "docker_run(%s)" % docker_start + print("docker_run(%s)" % docker_start) else: docker_run(docker_start) @@ -202,8 +202,8 @@ def stop(remove=False): docker_rm = 'docker rm %s' % configuration.docker.database.container_name if env.debug: - print "docker_run(%s)" % docker_stop - print "docker_run(%s)" % docker_rm + print("docker_run(%s)" % docker_stop) + print("docker_run(%s)" % docker_rm) else: docker_run(docker_stop) diff --git a/modules/initialize.py b/modules/initialize.py index 26aecda..5e7e2e4 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -359,7 +359,7 @@ def get_config(branchname): dataobject.server.nginx.port = config['nginx']['port'] dataobject.server.nginx.socket = config['nginx']['socket'] - print "DEBUG -- socket: %s" % dataobject.server.nginx.socket + print("DEBUG -- socket: %s" % dataobject.server.nginx.socket) if config['project']['host'] == "localhost": dataobject.server.nginx.host = "{projectname}.{ext}".format( @@ -426,9 +426,9 @@ def get_config(branchname): if 'database' in config['docker']: add_template(dataobject, layout, config, "docker", "database") else: - print "NOTE: docker.database does not exist for this branch" + print("NOTE: docker.database does not exist for this branch") else: - print "NOTE: docker information does not exist for this branch" + print("NOTE: docker information does not exist for this branch") # # nginx information diff --git a/modules/maintenance.py b/modules/maintenance.py index 18eea22..b799d31 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -22,19 +22,19 @@ def command(program=None, cmd=None, extra_param=None): # logger = loggify('maintenance', 'command') if program is None: - print "Error: You have not given a legitimate program" + print("Error: You have not given a legitimate program") - print "permissable programs : %s" \ - % configuration.maintenance.keys() + print("permissable programs : %s" % + configuration.maintenance.keys()) sys.exit() configuration_program = getattr(configuration.maintenance, program) if cmd is None: - print "Error: You have not given a legitimate command" - print "permissable commands : %s" \ - % configuration_program.commands.keys() + print("Error: You have not given a legitimate command") + print("permissable commands : %s" % + configuration_program.commands.keys()) sys.exit() @@ -165,8 +165,8 @@ def _get_configuration_path(name, branch): param_list = ['config', 'layout'] if name not in param_list: - print "value %s was not legit. _get_configuration_path requires" \ - "value such from %s" % (name, param_list) + print("value %s was not legit. _get_configuration_path requires" + "value such from %s" % (name, param_list)) META_DIR = os.path.join( get_project_root(), 'usr', 'meta', 'project') diff --git a/modules/pip.py b/modules/pip.py index 88b5ba1..e57ef80 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -221,8 +221,8 @@ def copyto(branch): configuration.virtualenv.requirements.local, branch_config.virtualenv.requirements.filename) - print "current_local_path: %s" % current_local_path - print "branch_local_path: %s" % branch_local_path + print("current_local_path: %s" % current_local_path) + print("branch_local_path: %s" % branch_local_path) message = "Copying file from current branch '{branch_src}' to " \ "destination branch '{branch_dst}'. Continue? Y/n ".format( diff --git a/modules/supervisor.py b/modules/supervisor.py index b3f0a22..bf938ba 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -47,7 +47,7 @@ def command(cmd=None): param = _initialize(configuration) if cmd == "update": - print "in update" + print("in update") # we don't need to specify the supervisor configuration file name, because # supervisor can figure it out. ie whatever.conf is referred to by diff --git a/modules/utils.py b/modules/utils.py index d34d51c..5c01122 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -11,7 +11,7 @@ from fabric.operations import run, sudo def printvar(name, value, exit=False): - print "%s : %s" % (name, value) + print("%s : %s" % (name, value)) if exit: sys.exit() @@ -88,8 +88,7 @@ def print_console(string, prepend="\n\n", append="\n\n", sep="-", numsep=44): def print_debug(debugstr, module, function): - print "%s:%s:%s" \ - % (module, function, debugstr) + print("%s:%s:%s" % (module, function, debugstr)) def executize(config_execute): @@ -146,12 +145,13 @@ def ensure_dir(directory): """ try: if not os.path.exists(directory): - print "creating directory: %s" % directory + print("creating directory: %s" % directory) os.makedirs(directory) except OSError, e: if e.errno != errno.EEXIST: - print "Error occurred while creating directory: %s" % directory + print("Error occurred while creating directory: %s" + % directory) raise From d75e93f2353621bcd1c074fdaa2c641f378416bb Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 22:52:00 +0300 Subject: [PATCH 21/65] fixed print statements --- modules/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 5c01122..d242c13 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -29,7 +29,7 @@ def printerr(message="", errcode=-2, exit=True): message=message, errcode=errcode) print - print message + print(message) print sys.exit(errcode) @@ -74,17 +74,17 @@ def print_console(string, prepend="\n\n", append="\n\n", sep="-", numsep=44): numsep - number of times the separator is printed out on a line """ - print prepend + print(prepend) if sep: - print sep * numsep + print(sep * numsep) - print string + print(string) if sep: - print sep * numsep + print(sep * numsep) - print append + print(append) def print_debug(debugstr, module, function): @@ -319,7 +319,7 @@ def handle_help(param, message, values=None): if isinstance(param, str): if param.lower() in values: - print message + print(message) return True return False From dc02488e8b1006836212f5b3ed7483d29ec7f2dc Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 22:52:15 +0300 Subject: [PATCH 22/65] fix import statements --- modules/initialize.py | 2 +- modules/maintenance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index 5e7e2e4..4ae3250 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -1,7 +1,7 @@ import os # import yaml import fabric.utils -import maintenance +from . import maintenance from fabric.api import env, task # from utils import loggify diff --git a/modules/maintenance.py b/modules/maintenance.py index b799d31..454c73f 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -3,7 +3,7 @@ from fabric.api import local import os import sys -import utils +from . import utils from utils import executize, virtualenv, loggify From a5677a96a00e306daea0ba1d6a2b2e1361e3ccc6 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 23:08:51 +0300 Subject: [PATCH 23/65] updated print funcs --- modules/conf_setup.py | 6 +++--- modules/database.py | 8 ++++---- modules/deploy.py | 16 ++++++++-------- modules/django.py | 24 ++++++++++++------------ modules/docker.py | 4 ++-- modules/initialize.py | 2 +- modules/maintenance.py | 4 ++-- modules/nginx.py | 6 +++--- modules/pip.py | 4 ++-- modules/supervisor.py | 6 +++--- modules/utils.py | 2 +- 11 files changed, 41 insertions(+), 41 deletions(-) diff --git a/modules/conf_setup.py b/modules/conf_setup.py index 1212c2e..508ad0d 100644 --- a/modules/conf_setup.py +++ b/modules/conf_setup.py @@ -3,7 +3,7 @@ from fabric.operations import run, put from fabric.api import env, task import os -from maintenance import _get_configuration_path, load_configuration +from .maintenance import _get_configuration_path, load_configuration @task @@ -297,7 +297,7 @@ def edit(param='help'): else: # if param == 'help': - print """ + print(""" "fab configuration.edit" automates opening up and editing project configuration files @@ -305,7 +305,7 @@ def edit(param='help'): parameter currently editable locations are: - """ + """) for k_loc in locations.keys(): print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) diff --git a/modules/database.py b/modules/database.py index 3ae06a2..e56dbf1 100644 --- a/modules/database.py +++ b/modules/database.py @@ -1,8 +1,8 @@ from fabric.api import env, task # # from jinja2 import Environment import os -from utils import upload_template as utils_upload_template -from utils import loggify, print_console, booleanize +from .utils import upload_template as utils_upload_template +from .utils import loggify, print_console, booleanize # from utils import prompt_continue from getpass import getpass import fabric.operations as fabric_ops @@ -554,7 +554,7 @@ def edit(param='help'): remote_path = locations[param]['path'] maintenance_edit(remote_path=remote_path) else: - print """ + print(""" "fab database.edit" automates editing files important to django whether locally or remotely @@ -562,7 +562,7 @@ def edit(param='help'): parameter currently editable locations are: - """ + """) for k_loc in locations.keys(): print("\t{0: <20} - {1}".format( diff --git a/modules/deploy.py b/modules/deploy.py index 0a8c5d6..4996c05 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -5,11 +5,11 @@ from fabric.operations import local # from fabric.contrib.files import upload_template # from utils import print_run -from utils import virtualenv_source, booleanize, loggify -from utils import print_console +from .utils import virtualenv_source, booleanize, loggify +from .utils import print_console -from pip import setup_virtualenv -from pip import bootstrap_pip +from .pip import setup_virtualenv +from .pip import bootstrap_pip import os @@ -410,7 +410,7 @@ def sync(full=True, extras=False): # either SYNCING TO OR FROM, then just exit at this point if configuration.project.branch == "development": - print """ + print(""" ------------------------------------ NOTE: this is a hack for the function deploy.sync() @@ -435,7 +435,7 @@ def sync(full=True, extras=False): Right after this message gets printed, the sync function is told to "return" without any arguments ------------------------------------\n - """ + """) # # exit the function without any arguments @@ -617,7 +617,7 @@ def remove(full=True): import sys if env.branch == "development" and not env.debug and full: - print """ + print(""" ------------------------------ WARNING: @@ -628,7 +628,7 @@ def remove(full=True): Exiting NOW. -------------------------------- - """ + """) sys.exit() # diff --git a/modules/django.py b/modules/django.py index 1d697a9..66e31fd 100644 --- a/modules/django.py +++ b/modules/django.py @@ -5,9 +5,9 @@ import fabric.operations as fabric_ops from fabric.contrib.files import exists -from utils import loggify, print_run, booleanize -from utils import generate_template_build_path -from utils import generate_template_files_path +from .utils import loggify, print_run, booleanize +from .utils import generate_template_build_path +from .utils import generate_template_files_path import os @@ -42,11 +42,11 @@ def commandline(args=""): """ configuration = env.config - commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " \ - "--settings={djangosettings}".format( - djangoroot=configuration.paths.django.root, - args=args, - djangosettings=configuration.imports.settings,) + commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}'" \ + " --settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings,) print(commandline) @@ -599,7 +599,7 @@ def edit(param='help'): else: # if param == 'help': - print """ + print(""" "fab django.edit" automates editing files important to django whether locally or remotely @@ -607,7 +607,7 @@ def edit(param='help'): parameter currently editable locations are: - """ + """) for k_loc in locations.keys(): print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) @@ -618,7 +618,7 @@ def edit(param='help'): @task def clearmigrations(appname="help"): if appname == "help": - print """ + print(""" "fab django.clearmigration:{appname}" clears out all migrations for the specified appname @@ -627,7 +627,7 @@ def clearmigrations(appname="help"): Note: if your appname is actually "help" you might want to go into this function and change it up a bit! - """ + """) return configuration = env.config diff --git a/modules/docker.py b/modules/docker.py index d99951e..e64f277 100644 --- a/modules/docker.py +++ b/modules/docker.py @@ -4,8 +4,8 @@ from fabric.operations import run from fabric.contrib.files import upload_template -from utils import loggify, generate_template_files_path, booleanize -from utils import generate_template_build_path, print_console +from .utils import loggify, generate_template_files_path, booleanize +from .utils import generate_template_build_path, print_console @task diff --git a/modules/initialize.py b/modules/initialize.py index 4ae3250..c870284 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -51,7 +51,7 @@ def environment(branchname): host = env.config.server.database.host - from docker import docker_ip + from .docker import docker_ip if host == 'docker': env.config.server.database.host = docker_ip() diff --git a/modules/maintenance.py b/modules/maintenance.py index 454c73f..8a0702d 100644 --- a/modules/maintenance.py +++ b/modules/maintenance.py @@ -4,7 +4,7 @@ import os import sys from . import utils -from utils import executize, virtualenv, loggify +from .utils import executize, virtualenv, loggify def command(program=None, cmd=None, extra_param=None): @@ -210,7 +210,7 @@ def load_configuration(name, branch): elif name == "layout": file_path = _get_configuration_path('layout', branch) - configuration_file = yaml.load(file(file_path, 'r')) + configuration_file = yaml.load(open(file_path, 'r')) return configuration_file diff --git a/modules/nginx.py b/modules/nginx.py index f90fc8f..a31c590 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -6,10 +6,10 @@ from fabric.operations import sudo import os # import sys -from maintenance import command as maintenance_command -from maintenance import edit as maintenance_edit +from .maintenance import command as maintenance_command +from .maintenance import edit as maintenance_edit -from utils import loggify +from .utils import loggify @task diff --git a/modules/pip.py b/modules/pip.py index e57ef80..7e60165 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -5,8 +5,8 @@ import sys import os import logging -from utils import virtualenv_source, virtualenv -from utils import print_console, printerr +from .utils import virtualenv_source, virtualenv +from .utils import print_console, printerr ERROR_BAD_BRANCH_PARAM = -3 ERROR_BAD_PARAM = -2 diff --git a/modules/supervisor.py b/modules/supervisor.py index bf938ba..53da1f7 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -5,10 +5,10 @@ from fabric.operations import sudo, run import os -from maintenance import command as maintenance_command -from maintenance import edit as maintenance_edit +from .maintenance import command as maintenance_command +from .maintenance import edit as maintenance_edit -from utils import loggify +from .utils import loggify def _initialize(configuration): diff --git a/modules/utils.py b/modules/utils.py index d242c13..b1aa820 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -148,7 +148,7 @@ def ensure_dir(directory): print("creating directory: %s" % directory) os.makedirs(directory) - except OSError, e: + except OSError as e: if e.errno != errno.EEXIST: print("Error occurred while creating directory: %s" % directory) From df846a1275ec8eebe61116404468d30b34cb8185 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 23:24:05 +0300 Subject: [PATCH 24/65] removing unnecessary old django settings files --- .../conf/django/files/settings.1.9.jinja2 | 138 ------------------ .../conf/django/files/settings18.jinja2 | 125 ---------------- 2 files changed, 263 deletions(-) delete mode 100644 share/templates/conf/django/files/settings.1.9.jinja2 delete mode 100644 share/templates/conf/django/files/settings18.jinja2 diff --git a/share/templates/conf/django/files/settings.1.9.jinja2 b/share/templates/conf/django/files/settings.1.9.jinja2 deleted file mode 100644 index be1fd74..0000000 --- a/share/templates/conf/django/files/settings.1.9.jinja2 +++ /dev/null @@ -1,138 +0,0 @@ -""" -Django settings for {{ project_name }} project. - -Generated by 'django-admin startproject' using Django 1.9.4. - -For more information on this file, see -https://docs.djangoproject.com/en/1.9/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.9/ref/settings/ -""" - -import os -import sys - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) - -# -# add the directory that has the intialize.py module which contains the -# get_config function definition which will pull out all the configuraiton -# infomration for the settings file - -sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) - -# -# add an "apps" directory to this project, which is where all the apps -# ought to be in the first place. - -sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '{{ secret_key }}' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -] - -MIDDLEWARE_CLASSES = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = '{{ project_name }}.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(PROJECT_ROOT, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = '{{ project_name }}.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.9/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - - -# Password validation -# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/1.9/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.9/howto/static-files/ - -STATIC_URL = '/static/' diff --git a/share/templates/conf/django/files/settings18.jinja2 b/share/templates/conf/django/files/settings18.jinja2 deleted file mode 100644 index 75c5cdb..0000000 --- a/share/templates/conf/django/files/settings18.jinja2 +++ /dev/null @@ -1,125 +0,0 @@ -""" -Django settings for {{ project_name }} project. - -Generated by 'django-admin startproject' using Django 1.8.3. - -For more information on this file, see -https://docs.djangoproject.com/en/1.8/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.8/ref/settings/ -""" - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os -import sys - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) - -# -# add the directory that has the intialize.py module which contains the -# get_config function definition which will pull out all the configuraiton -# infomration for the settings file - -sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'scripts', 'tools', 'fabric', 'modules')) - -# -# add an "apps" directory to this project, which is where all the apps -# ought to be in the first place. - -sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) - -# -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ - -# -# SECURITY WARNING: keep the secret key used in production secret! - -SECRET_KEY = '{{ secret_key }}' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.sites', - 'django.contrib.staticfiles', - - 'polls', -) - - -MIDDLEWARE_CLASSES = ( - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django.middleware.security.SecurityMiddleware', -) - -ROOT_URLCONF = '{{ project_name }}.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(PROJECT_ROOT, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] -WSGI_APPLICATION = '{{ project_name }}.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - - -# Internationalization -# https://docs.djangoproject.com/en/1.8/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.8/howto/static-files/ - -STATIC_URL = '/static/' -MEDIA_URL = '/media/' - From acc2392182fd175c452a9553234f1514aa82c617 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Oct 2018 23:25:26 +0300 Subject: [PATCH 25/65] updated settings for python3/django2.0 --- .../templates/conf/django/files/settings.jinja2 | 16 ++++++++-------- .../conf/gunicorn/files/settings.jinja2 | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/share/templates/conf/django/files/settings.jinja2 b/share/templates/conf/django/files/settings.jinja2 index 587a1f5..6086845 100644 --- a/share/templates/conf/django/files/settings.jinja2 +++ b/share/templates/conf/django/files/settings.jinja2 @@ -1,13 +1,13 @@ """ Django settings for {{ project_name }} project. -Generated by 'django-admin startproject' using Django 1.10.1. +Generated by 'django-admin startproject' using Django 2.1.2. For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ +https://docs.djangoproject.com/en/2.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ +https://docs.djangoproject.com/en/2.1/ref/settings/ """ import os @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.join( sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '{{ secret_key }}' @@ -87,7 +87,7 @@ WSGI_APPLICATION = '{{ project_name }}.wsgi.application' # Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASES = { 'default': { @@ -98,7 +98,7 @@ DATABASES = { # Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { @@ -121,7 +121,7 @@ AUTH_PASSWORD_VALIDATORS = [ # Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ +# https://docs.djangoproject.com/en/2.1/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -135,7 +135,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ +# https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_URL = '/static/' MEDIA_URL = '/media/' diff --git a/share/templates/conf/gunicorn/files/settings.jinja2 b/share/templates/conf/gunicorn/files/settings.jinja2 index 587a1f5..6086845 100644 --- a/share/templates/conf/gunicorn/files/settings.jinja2 +++ b/share/templates/conf/gunicorn/files/settings.jinja2 @@ -1,13 +1,13 @@ """ Django settings for {{ project_name }} project. -Generated by 'django-admin startproject' using Django 1.10.1. +Generated by 'django-admin startproject' using Django 2.1.2. For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ +https://docs.djangoproject.com/en/2.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ +https://docs.djangoproject.com/en/2.1/ref/settings/ """ import os @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.join( sys.path.insert(0, os.path.join(BASE_DIR, 'apps')) # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = '{{ secret_key }}' @@ -87,7 +87,7 @@ WSGI_APPLICATION = '{{ project_name }}.wsgi.application' # Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASES = { 'default': { @@ -98,7 +98,7 @@ DATABASES = { # Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { @@ -121,7 +121,7 @@ AUTH_PASSWORD_VALIDATORS = [ # Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ +# https://docs.djangoproject.com/en/2.1/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -135,7 +135,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ +# https://docs.djangoproject.com/en/2.1/howto/static-files/ STATIC_URL = '/static/' MEDIA_URL = '/media/' From a98f1635f45b40ebcef927c9aa1de00ef3f12eaa Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 18 Oct 2018 06:33:10 +0300 Subject: [PATCH 26/65] update so that mkvirtualenv will install python3 --- modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pip.py b/modules/pip.py index 7e60165..6cc8438 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -21,7 +21,7 @@ def setup_virtualenv(): format='\n%(levelname)s: deploy.setup_virtualenv %(message)s', level=logging.DEBUG) - mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " \ + mkvirtualenv_cmd = "mkvirtualenv --no-site-packages --python=`which python3` " \ "{virtualenv_name}".format( virtualenv_name=configuration.virtualenv.name) From 15ffc7ac8c9c6c6cdae840fa49444d8640dc0730 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 18 Oct 2018 06:33:23 +0300 Subject: [PATCH 27/65] update deploy so packages are found --- modules/deploy.py | 52 ++++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/modules/deploy.py b/modules/deploy.py index 4996c05..8862511 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -7,12 +7,28 @@ from fabric.operations import local from .utils import virtualenv_source, booleanize, loggify from .utils import print_console +from .utils import link_create + +from .nginx import upload as upload_nginx +from .nginx import remove as nginx_remove +from .supervisor import upload as upload_supervisor +from .supervisor import remove as supervisor_remove + from .pip import setup_virtualenv from .pip import bootstrap_pip +from .pip import install as pip_install + +from .django import create_project +from .django import generate as django_generate +from .django import collectstatic as django_collectstatic + +from . import initialize +from . import docker +from . import database as db + import os - import logging @@ -82,7 +98,6 @@ def bootstrap(): bootstrap_pip() # create the django project - from django import create_project create_project() # @@ -109,8 +124,6 @@ def bootstrap(): # links before creating them, otherwise you get really weird errors # where the a link is recreated within the destination link - from utils import link_create - if env.debug: logger.debug("virtualenv.root : %s" % configuration.virtualenv.paths.root) @@ -154,10 +167,6 @@ def bootstrap(): # create and link the scripts that manage the server # e.g. nginx, supervisor, gunicorn - from nginx import upload as upload_nginx - from supervisor import upload as upload_supervisor - from django import generate as django_generate - print_console("creating gunicorn script") django_generate('gunicorn', True) django_generate('local', True) @@ -171,8 +180,6 @@ def bootstrap(): # # instantiate docker containers if any - import docker - print_console("check to see if docker containers are used") if hasattr(configuration, "docker"): @@ -219,15 +226,12 @@ def bootstrap_part1(): @task def bootstrap_part2(): - import database as db - configuration = env.config if env.debug: logger = loggify('deploy', 'bootstrap') # create the django project - from django import create_project create_project() # @@ -254,8 +258,6 @@ def bootstrap_part2(): # links before creating them, otherwise you get really weird errors # where the a link is recreated within the destination link - from utils import link_create - if env.debug: logger.debug("virtualenv.root : %s" % configuration.virtualenv.paths.root) @@ -299,10 +301,6 @@ def bootstrap_part2(): # create and link the scripts that manage the server # e.g. nginx, supervisor, gunicorn - from nginx import upload as upload_nginx - from supervisor import upload as upload_supervisor - from django import generate as django_generate - print_console("creating gunicorn script") django_generate('gunicorn', True) django_generate('local', True) @@ -316,8 +314,6 @@ def bootstrap_part2(): # # instantiate docker containers if any - import docker - print_console("check to see if docker containers are used") if hasattr(configuration, "docker"): @@ -453,7 +449,6 @@ def sync(full=True, extras=False): exclude=excludeitems) if full: - from pip import install as pip_install pip_install('--all') # overrides() @@ -463,15 +458,10 @@ def sync(full=True, extras=False): # NOTE: if using full synch if full: - from nginx import upload as upload_nginx - from supervisor import upload as upload_supervisor - from django import generate as django_generate - upload_supervisor() django_generate('gunicorn', True) upload_nginx() - from django import collectstatic as django_collectstatic django_collectstatic() @@ -484,8 +474,6 @@ def media(source_branch): """ configuration = env.config - import initialize - print("project.branch: %s" % configuration.project.branch) # @@ -693,14 +681,10 @@ def remove(full=True): # because otherwise they interfere with this modules version # of "remove" - from nginx import remove as nginx_remove - from supervisor import remove as supervisor_remove - nginx_remove() supervisor_remove() - from database import drop_all as db_drop_all - db_drop_all() + db.drop_all() # # check to see if the parent directory contains anything else From 4eba2cbf5a364464749fe42c113af7e004bd735c Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 18 Oct 2018 12:45:07 +0300 Subject: [PATCH 28/65] updating for python3 --- modules/deploy.py | 2 -- modules/django.py | 10 +++++----- modules/pip.py | 12 ++++++------ modules/supervisor.py | 2 +- share/templates/conf/django/files/local.jinja2 | 2 +- share/templates/conf/django/files/settings.jinja2 | 3 +-- share/templates/conf/gunicorn/files/local.jinja2 | 2 +- share/templates/conf/gunicorn/files/settings.jinja2 | 3 +-- 8 files changed, 16 insertions(+), 20 deletions(-) diff --git a/modules/deploy.py b/modules/deploy.py index 8862511..18bcfb5 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -76,8 +76,6 @@ def setup_rootpath(): @task def bootstrap(): - import database as db - configuration = env.config if env.debug: diff --git a/modules/django.py b/modules/django.py index 66e31fd..ffbc1bf 100644 --- a/modules/django.py +++ b/modules/django.py @@ -8,6 +8,7 @@ from fabric.contrib.files import exists from .utils import loggify, print_run, booleanize from .utils import generate_template_build_path from .utils import generate_template_files_path +from .utils import handle_help import os @@ -145,7 +146,6 @@ def startapp(appname='help'): \tappname - the name of the app you want to start """ - from utils import handle_help import sys if handle_help(appname, msg_help, 'help'): @@ -317,7 +317,7 @@ def generate_scripts(template_name, make_copy=False): backup_cmd = "cp {copy_full_path} " \ "{copy_full_path}.bak".format(copy_full_path=copy_full_path) - from utils import upload_template as utils_upload_template + from .utils import upload_template as utils_upload_template if env.debug: logger.debug("template_name : %s" % template_name) @@ -501,7 +501,7 @@ def generate_gunicorn(make_link=True): logger.debug('\n%s' % print_run(msg_link_gunicorn)) - from utils import upload_template as utils_upload_template + from .utils import upload_template as utils_upload_template upload_msg = utils_upload_template( filename=configuration.templates.gunicorn.conf.src, @@ -538,7 +538,7 @@ def edit(param='help'): calls up mvim on the gunicorn conf file """ - from maintenance import edit as maintenance_edit + from .maintenance import edit as maintenance_edit configuration = env.config @@ -712,7 +712,7 @@ def fixtures(appname=None, backup=False): print("debug - path_fixture: %s" % path_fixture) - from utils import ensure_dir + from .utils import ensure_dir ensure_dir(path_data) ensure_dir(path_backup) diff --git a/modules/pip.py b/modules/pip.py index 6cc8438..69a774e 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -21,8 +21,8 @@ def setup_virtualenv(): format='\n%(levelname)s: deploy.setup_virtualenv %(message)s', level=logging.DEBUG) - mkvirtualenv_cmd = "mkvirtualenv --no-site-packages --python=`which python3` " \ - "{virtualenv_name}".format( + mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " \ + "--python=`which python3` {virtualenv_name}".format( virtualenv_name=configuration.virtualenv.name) if env.debug: @@ -147,7 +147,7 @@ def freeze(param='help'): \tFalse (default) - print the freeze output to the console """ - from utils import booleanize, handle_help + from .utils import booleanize, handle_help if handle_help(param, msg_help, 'help'): sys.exit() @@ -209,7 +209,7 @@ def copyto(branch): "This will simply copy over \n\tthe requirements file for " "this branch with itself", ERROR_BAD_BRANCH_PARAM) - from initialize import get_config + from .initialize import get_config branch_config = get_config(branch) @@ -228,10 +228,10 @@ def copyto(branch): "destination branch '{branch_dst}'. Continue? Y/n ".format( branch_src=configuration.project.branch, branch_dst=branch) - from utils import prompt_continue + from .utils import prompt_continue prompt_continue(message=message) - from utils import upload_template + from .utils import upload_template upload_template( filename=configuration.virtualenv.requirements.filename, destination=branch_local_path, diff --git a/modules/supervisor.py b/modules/supervisor.py index 53da1f7..f2ef637 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -224,7 +224,7 @@ def upload(): logger.debug("run(%s)" % copy_command) else: - import utils + from . import utils utils.printvar("files_path", files_path) upload_template( diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index 4454550..76255b9 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -3,7 +3,7 @@ BRANCH = '{{ project_branch }}' from {{ project_name }}.settings import * -import initialize +from modules import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH diff --git a/share/templates/conf/django/files/settings.jinja2 b/share/templates/conf/django/files/settings.jinja2 index 6086845..a771b3c 100644 --- a/share/templates/conf/django/files/settings.jinja2 +++ b/share/templates/conf/django/files/settings.jinja2 @@ -23,7 +23,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) # infomration for the settings file sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) + PROJECT_ROOT, 'usr', 'bin', 'fabric')) # # add an "apps" directory to this project, which is where all the apps @@ -60,7 +60,6 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index 4454550..76255b9 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -3,7 +3,7 @@ BRANCH = '{{ project_branch }}' from {{ project_name }}.settings import * -import initialize +from modules import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH diff --git a/share/templates/conf/gunicorn/files/settings.jinja2 b/share/templates/conf/gunicorn/files/settings.jinja2 index 6086845..a771b3c 100644 --- a/share/templates/conf/gunicorn/files/settings.jinja2 +++ b/share/templates/conf/gunicorn/files/settings.jinja2 @@ -23,7 +23,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BASE_DIR, os.pardir)) # infomration for the settings file sys.path.insert(0, os.path.join( - PROJECT_ROOT, 'usr', 'bin', 'fabric', 'modules')) + PROJECT_ROOT, 'usr', 'bin', 'fabric')) # # add an "apps" directory to this project, which is where all the apps @@ -60,7 +60,6 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] From b9d252a1fc3afc5db31a4459a3c296b80a0999db Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 24 Oct 2018 02:42:27 +0300 Subject: [PATCH 29/65] modified import statements in deploy to resolve conflict that led to docker and database modules being considered a submodule of deploy --- modules/deploy.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/deploy.py b/modules/deploy.py index 18bcfb5..f7f9df1 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -24,13 +24,16 @@ from .django import generate as django_generate from .django import collectstatic as django_collectstatic from . import initialize -from . import docker -from . import database as db - import os import logging +# NOTE: +# we have a bunch of imports scattered through the func definitions +# this is done b/c otherwise fabric will consider the modules we are +# importing from to be a SUB-MODULE of deploy. So leave them where +# they are + @task def setup_rootpath(): @@ -76,6 +79,7 @@ def setup_rootpath(): @task def bootstrap(): + from . import docker configuration = env.config if env.debug: @@ -191,6 +195,7 @@ def bootstrap(): # # create and initialize the database + from . import database as db print_console("in db.generate") db.generate() @@ -224,6 +229,7 @@ def bootstrap_part1(): @task def bootstrap_part2(): + from . import docker configuration = env.config if env.debug: @@ -326,6 +332,8 @@ def bootstrap_part2(): # # create and initialize the database + from . import database as db + print_console("in db.generate") db.generate() @@ -682,6 +690,7 @@ def remove(full=True): nginx_remove() supervisor_remove() + from . import database as db db.drop_all() # From 17468643b6ee51442a34f5099f976da2953a5ecc Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 25 Oct 2018 20:36:53 +0300 Subject: [PATCH 30/65] modify import statements --- modules/conf_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/conf_setup.py b/modules/conf_setup.py index 508ad0d..e1db674 100644 --- a/modules/conf_setup.py +++ b/modules/conf_setup.py @@ -265,8 +265,8 @@ def edit(param='help'): calls up mvim on the yaml project configuration files """ - from maintenance import edit as maintenance_edit - from maintenance import _get_configuration_path + from .maintenance import edit as maintenance_edit + from .maintenance import _get_configuration_path locations = { 'development': { From ebb0e6fdbe65320b19dc0bcf81bcad7caa015fb6 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 25 Oct 2018 20:37:15 +0300 Subject: [PATCH 31/65] updated edit to use log files --- modules/django.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/modules/django.py b/modules/django.py index ffbc1bf..c2891d5 100644 --- a/modules/django.py +++ b/modules/django.py @@ -535,7 +535,7 @@ def generate_gunicorn(make_link=True): @task def edit(param='help'): """ - calls up mvim on the gunicorn conf file + calls up mvim on the gunicorn and django conf file """ from .maintenance import edit as maintenance_edit @@ -569,6 +569,8 @@ def edit(param='help'): django_path=django_path, project_branch=project_branch) + file_debug_handler = configuration.logging.django.handlers.file_debug + # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', # 'settings', 'local'] locations = { @@ -590,7 +592,19 @@ def edit(param='help'): 'local': { 'path': settings_local_path, 'desc': 'local settings file for django project', - } + }, + + 'server_log': { + 'path': file_debug_handler.path.server, + 'desc': 'django server log - handler name: %s' % + file_debug_handler.name.server, + }, + + 'project_log': { + 'path': file_debug_handler.path.project, + 'desc': 'django project log - handler name: %s' % + file_debug_handler.name.project, + }, } if param in locations.keys(): From a3fcded4891cb932bfa8e1f7a48607b67d53f2eb Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 25 Oct 2018 20:37:36 +0300 Subject: [PATCH 32/65] add logs to edit --- modules/supervisor.py | 48 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/modules/supervisor.py b/modules/supervisor.py index f2ef637..7453a66 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -108,24 +108,60 @@ def restart(): @task -def edit(): +def edit(param='help'): """ calls up mvim on the Supervisor conf file """ configuration = env.config - param = _initialize(configuration) if env.debug: logger = loggify('supervisor', 'edit') + logger.debug() - conf_path = param['conf_path'] + configuration_path = _initialize(configuration)['conf_path'] - if env.debug: - logger.debug("conf path : %s" % conf_path) + locations = { + 'configuration': { + 'path': configuration_path, + 'desc': "supervisor configuration path", + }, + 'log.out': { + 'path': configuration.logging.supervisor.out, + 'desc': "out logging file", + }, + + 'log.err': { + 'path': configuration.logging.supervisor.err, + 'desc': "err logging file", + }, + } + + print("\n\n\nlocations: %s\n\n\n" % locations) + print("\n\n\nlocations.keys: %s\n\n\n" % locations.keys()) + print("\n\n\nparam: %s\n\n\n" % param) + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) else: - maintenance_edit(remote_path=conf_path) + # if param == 'help': + + print(""" + "fab django.edit" automates editing files important to django whether + locally or remotely + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return @task From 257de6483da3f89dcbafbd4229463336f659b30b Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 25 Oct 2018 20:45:11 +0300 Subject: [PATCH 33/65] got rid of comments in supervisor, updated edit method in nginx --- modules/nginx.py | 47 ++++++++++++++++++++++++++++++++++--------- modules/supervisor.py | 4 ---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/modules/nginx.py b/modules/nginx.py index a31c590..3e25acb 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -58,25 +58,54 @@ def restart(): @task -def edit(location='conf'): +def edit(param='help'): """ calls up mvim on the Nginx conf file """ configuration = env.config - conf_path = os.path.join( + config_file_path = os.path.join( configuration.nginx.sites_available, configuration.nginx.conf.name) - if location == 'conf': - remote_path = conf_path - elif location == 'log.error' or location == 'log': - remote_path = configuration.logging.nginx.error - elif location == 'log.access': - remote_path = configuration.logging.nginx.access + locations = { + 'conf': { + 'path': config_file_path, + 'desc': "nginx configuration file", + }, - maintenance_edit(remote_path=remote_path) + 'log.error': { + 'path': configuration.logging.nginx.error, + 'desc': "nginx error log file", + }, + + 'log.access': { + 'path': configuration.logging.nxing.access, + 'desc': "nginx access log file", + }, + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab nginx.edit" automates editing files important to nginx whether + locally or remotely + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return @task diff --git a/modules/supervisor.py b/modules/supervisor.py index 7453a66..1ad3827 100644 --- a/modules/supervisor.py +++ b/modules/supervisor.py @@ -138,10 +138,6 @@ def edit(param='help'): }, } - print("\n\n\nlocations: %s\n\n\n" % locations) - print("\n\n\nlocations.keys: %s\n\n\n" % locations.keys()) - print("\n\n\nparam: %s\n\n\n" % param) - if param in locations.keys(): remote_path = locations[param]['path'] maintenance_edit(remote_path=remote_path) From 8dab351ce6278eb361f3fd84e396445e3b84a46c Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 1 Nov 2018 01:34:30 +0200 Subject: [PATCH 34/65] add logging --- modules/django.py | 10 ++++++++++ modules/nginx.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/django.py b/modules/django.py index c2891d5..2e1f6c0 100644 --- a/modules/django.py +++ b/modules/django.py @@ -584,6 +584,16 @@ def edit(param='help'): 'desc': "gunicorn.conf file in scripts/conf/gunicorn/build" }, + 'gunicorn.log.error': { + 'path': configuration.logging.gunicorn.error, + 'desc': "gunicorn error log file", + }, + + 'gunicorn.log.access': { + 'path': configuration.logging.gunicorn.access, + 'desc': "gunicorn error log file", + }, + 'settings': { 'path': settings_path, 'desc': 'main settings file for django project', diff --git a/modules/nginx.py b/modules/nginx.py index 3e25acb..6f3aaff 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -81,7 +81,7 @@ def edit(param='help'): }, 'log.access': { - 'path': configuration.logging.nxing.access, + 'path': configuration.logging.nginx.access, 'desc': "nginx access log file", }, } From e270402ac9692a4b24f992cbeaec21a00ff99cbd Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Fri, 9 Nov 2018 10:16:13 +0200 Subject: [PATCH 35/65] updated fabric allowed hosts --- modules/initialize.py | 22 +++++++++++-------- .../templates/conf/django/files/local.jinja2 | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index c870284..7db1b83 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -197,15 +197,6 @@ def get_config(branchname): dataobject.project.user = config['project']['user'] dataobject.project.group = config['project']['group'] - dataobject.project.allowedhosts = list() - _allowed = getattr(dataobject.project, 'allowedhosts') - _allowed.append(dataobject.project.extendedname) - _allowed.append(dataobject.project.host) - - if 'allowed_hosts' in config['project']: - for allowed in config['project']['allowed_hosts']: - _allowed.append(allowed) - dataobject.project.sudo = config['project']['sudo'] # @@ -234,6 +225,19 @@ def get_config(branchname): dataobject.project.django.settings_folder = \ config['django']['settings_folder'] + # + # allowed_hosts are the ip addresses and hostnames + # that this instance is allowed to run on + + dataobject.project.django.allowedhosts = list() + _allowed = getattr(dataobject.project.django, 'allowedhosts') + _allowed.append(dataobject.project.extendedname) + _allowed.append(dataobject.project.host) + + if 'allowed_hosts' in config['django']: + for allowed in config['django']['allowed_hosts']: + _allowed.append(allowed) + dataobject.addbranch('paths') dataobject.paths.addbranch('project') diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index 76255b9..d59b8e1 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -70,7 +70,7 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = configuration.project.allowedhosts +ALLOWED_HOSTS = configuration.project.django.allowedhosts # ----------------------------------------- # Debug logging to the console From 2dffeaa1b99c77b953da66fb6aa24445a2fcdc89 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Fri, 9 Nov 2018 11:27:28 +0200 Subject: [PATCH 36/65] fixed the startapp and manage so that startapp does not create a directory (leads to an error) and instead added arg to manage to specify working directory where the manage command would be run in --- modules/django.py | 54 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/modules/django.py b/modules/django.py index 2e1f6c0..820c035 100644 --- a/modules/django.py +++ b/modules/django.py @@ -37,29 +37,56 @@ def test(args=None): @task -def commandline(args=""): +def commandline(args="", workingdir=None): """ prints out what you need to run on the command line to invoke manage.py """ configuration = env.config - commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}'" \ - " --settings={djangosettings}".format( - djangoroot=configuration.paths.django.root, - args=args, - djangosettings=configuration.imports.settings,) + if workingdir: + commandline = "{djangoroot}/manage.py {args} " \ + "--pythonpath='{djangoroot}' " \ + "--settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings,) + else: + commandline = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}'" \ + " --settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings,) print(commandline) @task -def manage(args=""): +def manage(args="", workingdir=None): configuration = env.config # changes the working directory to the djangoroot from fabric.context_managers import cd with virtualenv(): + if workingdir: + with cd(workingdir): + fabcommand = "{djangoroot}/manage.py {args} " \ + "--pythonpath='{djangoroot}' " \ + "--settings={djangosettings}".format( + djangoroot=configuration.paths.django.root, + args=args, + djangosettings=configuration.imports.settings, + ) + + if env.debug: + print("fabcommand: %s" % fabcommand) + print("workingdir: %s" % workingdir) + else: + output = fabric_ops.run( + fabcommand, + # MAKE SURE THIS IS ALWAYS HERE! + shell='/bin/bash') + with cd(configuration.paths.django.root): output = fabric_ops.run( "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " @@ -151,18 +178,11 @@ def startapp(appname='help'): if handle_help(appname, msg_help, 'help'): sys.exit() - destination = os.path.join(configuration.paths.django.apps, appname) + command = "startapp {appname}".format( + appname=appname) - cmd_mkdir = "mkdir {destination}".format( - destination=destination) + manage(command, workingdir=configuration.paths.django.apps) - command = "startapp {appname} {destination}".format( - appname=appname, - destination=destination) - - fabric_ops.run(cmd_mkdir) - - manage(command) # with lcd(configuration.paths.django.apps): # manage(command) From 73bdebbf1aa34ea762224a165ca98ba75dc45f42 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 6 Feb 2019 21:44:16 +0200 Subject: [PATCH 37/65] added python3 support to pip virtualenv setup --- modules/pip.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/pip.py b/modules/pip.py index 88b5ba1..61a21de 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -13,7 +13,7 @@ ERROR_BAD_PARAM = -2 @task -def setup_virtualenv(): +def setup_virtualenv(python3=False): configuration = env.config if env.debug: @@ -21,11 +21,18 @@ def setup_virtualenv(): format='\n%(levelname)s: deploy.setup_virtualenv %(message)s', level=logging.DEBUG) - mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " \ - "{virtualenv_name}".format( - virtualenv_name=configuration.virtualenv.name) + mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " + + if python3: + mkvirtualenv_cmd += "--python=python3 " + + mkvirtualenv_cmd += "{virtualenv_name}".format( + virtualenv_name=configuration.virtualenv.name) if env.debug: + + logging.debug("python3 install: %s" % python3) + logging.debug("virtualenv.workon : %s" % configuration.virtualenv.workon) @@ -41,6 +48,8 @@ def setup_virtualenv(): logging.debug("virtualenv.paths.root : %s" % configuration.virtualenv.paths.root) + logging.debug("mkvirtualenv_cmd: %s" % mkvirtualenv_cmd) + logging.debug("with virtualenv_source(): run(\"\n\t%s\n\t\")".format( mkvirtualenv_cmd)) else: From 4a2cdf9f16d9e17577308d94d4bbe773193632a0 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 6 Feb 2019 23:22:36 +0200 Subject: [PATCH 38/65] update pip for choosing which python --- modules/pip.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/modules/pip.py b/modules/pip.py index 69a774e..61a21de 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -5,15 +5,15 @@ import sys import os import logging -from .utils import virtualenv_source, virtualenv -from .utils import print_console, printerr +from utils import virtualenv_source, virtualenv +from utils import print_console, printerr ERROR_BAD_BRANCH_PARAM = -3 ERROR_BAD_PARAM = -2 @task -def setup_virtualenv(): +def setup_virtualenv(python3=False): configuration = env.config if env.debug: @@ -21,11 +21,18 @@ def setup_virtualenv(): format='\n%(levelname)s: deploy.setup_virtualenv %(message)s', level=logging.DEBUG) - mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " \ - "--python=`which python3` {virtualenv_name}".format( - virtualenv_name=configuration.virtualenv.name) + mkvirtualenv_cmd = "mkvirtualenv --no-site-packages " + + if python3: + mkvirtualenv_cmd += "--python=python3 " + + mkvirtualenv_cmd += "{virtualenv_name}".format( + virtualenv_name=configuration.virtualenv.name) if env.debug: + + logging.debug("python3 install: %s" % python3) + logging.debug("virtualenv.workon : %s" % configuration.virtualenv.workon) @@ -41,6 +48,8 @@ def setup_virtualenv(): logging.debug("virtualenv.paths.root : %s" % configuration.virtualenv.paths.root) + logging.debug("mkvirtualenv_cmd: %s" % mkvirtualenv_cmd) + logging.debug("with virtualenv_source(): run(\"\n\t%s\n\t\")".format( mkvirtualenv_cmd)) else: @@ -147,7 +156,7 @@ def freeze(param='help'): \tFalse (default) - print the freeze output to the console """ - from .utils import booleanize, handle_help + from utils import booleanize, handle_help if handle_help(param, msg_help, 'help'): sys.exit() @@ -209,7 +218,7 @@ def copyto(branch): "This will simply copy over \n\tthe requirements file for " "this branch with itself", ERROR_BAD_BRANCH_PARAM) - from .initialize import get_config + from initialize import get_config branch_config = get_config(branch) @@ -221,17 +230,17 @@ def copyto(branch): configuration.virtualenv.requirements.local, branch_config.virtualenv.requirements.filename) - print("current_local_path: %s" % current_local_path) - print("branch_local_path: %s" % branch_local_path) + print "current_local_path: %s" % current_local_path + print "branch_local_path: %s" % branch_local_path message = "Copying file from current branch '{branch_src}' to " \ "destination branch '{branch_dst}'. Continue? Y/n ".format( branch_src=configuration.project.branch, branch_dst=branch) - from .utils import prompt_continue + from utils import prompt_continue prompt_continue(message=message) - from .utils import upload_template + from utils import upload_template upload_template( filename=configuration.virtualenv.requirements.filename, destination=branch_local_path, From 8cc2e9a1013992f592d3f688be75e3b0b8d31252 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sat, 9 Feb 2019 23:46:19 +0200 Subject: [PATCH 39/65] update pip so it automatically tries to setup virtualenv with python3 --- modules/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pip.py b/modules/pip.py index 61a21de..d47e405 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -13,7 +13,7 @@ ERROR_BAD_PARAM = -2 @task -def setup_virtualenv(python3=False): +def setup_virtualenv(python3=True): configuration = env.config if env.debug: From 16f43982a0727485447400b7e514e383da787ae1 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sat, 9 Feb 2019 23:48:04 +0200 Subject: [PATCH 40/65] update django generate local settings os that it copies over internally generated values from the project files rather than directly connecting. this means ALL projects need to be updated wihenever project configuraiton settings are changed --- modules/django.py | 45 ++++++++++++++++ .../templates/conf/django/files/local.jinja2 | 53 ++++++++++--------- .../conf/gunicorn/files/local.jinja2 | 53 ++++++++++--------- 3 files changed, 99 insertions(+), 52 deletions(-) diff --git a/modules/django.py b/modules/django.py index 820c035..0fcd71c 100644 --- a/modules/django.py +++ b/modules/django.py @@ -328,6 +328,51 @@ def generate_scripts(template_name, make_copy=False): project_name=project_name) build_name = "%s.py" % project_branch + # add extra local specfic context variables + # NOTE: + # if you change project settings, you MUST run generate + + context['paths_django_templates'] = \ + configuration.paths.django.templates + + context['server_database_backend'] = \ + configuration.server.database.backend + + context['server_database_name'] = configuration.server.database.name + context['server_database_user'] = configuration.server.database.user + + context['server_database_password'] = \ + configuration.server.database.password + + context['server_database_host'] = configuration.server.database.host + context['server_database_port'] = configuration.server.database.port + + context['paths_server_media_static'] = \ + configuration.paths.server.media.static + + context['paths_server_media_dynamic'] = \ + configuration.paths.server.media.dynamic + + context['paths_project_root'] = configuration.paths.project.root + + context['project_django_allowedhosts'] = \ + configuration.project.django.allowedhosts + + # convenience variable naming, otherwise it's too long to deal with + file_debug_handler = configuration.logging.django.handlers.file_debug + + context['file_debug_handler__name_project'] = \ + file_debug_handler.name.project + + context['file_debug_handler__path_project'] = \ + file_debug_handler.path.project + + context['file_debug_handler__name_server'] = \ + file_debug_handler.name.server + + context['file_debug_handler__path_server'] = \ + file_debug_handler.path.server + copy_full_path = "{copy_path}/{build_name}".format( copy_path=copy_path, build_name=build_name) diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index d59b8e1..b276153 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -1,19 +1,20 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + LOCAL_SETTINGS = True # avoid recursive imports BRANCH = '{{ project_branch }}' from {{ project_name }}.settings import * -from modules import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -configuration = initialize.environment(BRANCH) - SITE_ID = 1 -TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") # # NOTE: a lot of the code in these local settings files are automated and you @@ -26,16 +27,13 @@ TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) DATABASES = { 'default': { 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend=configuration.server.database.backend), + db_backend='{{server_database_backend}}'), - 'NAME': '{db_name}'.format(db_name=configuration.server.database.name), - 'USER': '{db_user}'.format(db_user=configuration.server.database.user), - - 'PASSWORD': '{db_pass}'.format( - db_pass=configuration.server.database.password), - - 'HOST': '{db_host}'.format(db_host=configuration.server.database.host), - 'PORT': '{db_port}'.format(db_port=configuration.server.database.port), + 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), + 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), + 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), + 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), + 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), } } @@ -51,15 +49,15 @@ DATABASES = { # MEDIA_ROOT is the dynamic media information that the web server, user or # admin # will be adding and taking out. It's why I call it "dynamic" -STATIC_ROOT = configuration.paths.server.media.static -MEDIA_ROOT = configuration.paths.server.media.dynamic +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' # directories from which we search for static files to place in STATIC_ROOT # these static files are located within the project root as opposed to the # server root location STATICFILES_DIRS = ( - os.path.join(configuration.paths.project.root, "share", "media"), + os.path.join("{{paths_project_root}}", "share", "media"), ) # debug and debug toolbar settings @@ -70,14 +68,11 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = configuration.project.django.allowedhosts +ALLOWED_HOSTS = {{ project_django_allowedhosts }} # ----------------------------------------- # Debug logging to the console -# convenience variable naming, otherwise it's too long to deal with -file_debug_handler = configuration.logging.django.handlers.file_debug - LOGGING = { 'version': 1, 'formatters': { @@ -94,17 +89,23 @@ LOGGING = { }, }, 'handlers': { - file_debug_handler.name.project: { + '{{ file_debug_handler__name_project }}': { 'level': 'DEBUG', 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.project, + + 'filename': + '{{file_debug_handler__path_project}}', + 'formatter': 'code' }, - file_debug_handler.name.server: { + '{{ file_debug_handler__name_server }}': { 'level': 'DEBUG', 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.server, + + 'filename': + '{{file_debug_handler__path_server}}', + 'formatter': 'code' }, @@ -124,8 +125,8 @@ LOGGING = { 'handlers': [ 'console', - file_debug_handler.name.project, - file_debug_handler.name.server + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', ], 'level': 'DEBUG', diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index 76255b9..b276153 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -1,19 +1,20 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + LOCAL_SETTINGS = True # avoid recursive imports BRANCH = '{{ project_branch }}' from {{ project_name }}.settings import * -from modules import initialize import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -configuration = initialize.environment(BRANCH) - SITE_ID = 1 -TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") # # NOTE: a lot of the code in these local settings files are automated and you @@ -26,16 +27,13 @@ TEMPLATES[0]['DIRS'].append(configuration.paths.django.templates) DATABASES = { 'default': { 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend=configuration.server.database.backend), + db_backend='{{server_database_backend}}'), - 'NAME': '{db_name}'.format(db_name=configuration.server.database.name), - 'USER': '{db_user}'.format(db_user=configuration.server.database.user), - - 'PASSWORD': '{db_pass}'.format( - db_pass=configuration.server.database.password), - - 'HOST': '{db_host}'.format(db_host=configuration.server.database.host), - 'PORT': '{db_port}'.format(db_port=configuration.server.database.port), + 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), + 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), + 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), + 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), + 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), } } @@ -51,15 +49,15 @@ DATABASES = { # MEDIA_ROOT is the dynamic media information that the web server, user or # admin # will be adding and taking out. It's why I call it "dynamic" -STATIC_ROOT = configuration.paths.server.media.static -MEDIA_ROOT = configuration.paths.server.media.dynamic +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' # directories from which we search for static files to place in STATIC_ROOT # these static files are located within the project root as opposed to the # server root location STATICFILES_DIRS = ( - os.path.join(configuration.paths.project.root, "share", "media"), + os.path.join("{{paths_project_root}}", "share", "media"), ) # debug and debug toolbar settings @@ -70,14 +68,11 @@ USE_DEBUG_TOOLBAR = DEBUG # allow template debug outputs on {{ project_branch }} environment INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] -ALLOWED_HOSTS = configuration.project.allowedhosts +ALLOWED_HOSTS = {{ project_django_allowedhosts }} # ----------------------------------------- # Debug logging to the console -# convenience variable naming, otherwise it's too long to deal with -file_debug_handler = configuration.logging.django.handlers.file_debug - LOGGING = { 'version': 1, 'formatters': { @@ -94,17 +89,23 @@ LOGGING = { }, }, 'handlers': { - file_debug_handler.name.project: { + '{{ file_debug_handler__name_project }}': { 'level': 'DEBUG', 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.project, + + 'filename': + '{{file_debug_handler__path_project}}', + 'formatter': 'code' }, - file_debug_handler.name.server: { + '{{ file_debug_handler__name_server }}': { 'level': 'DEBUG', 'class': 'logging.FileHandler', - 'filename': file_debug_handler.path.server, + + 'filename': + '{{file_debug_handler__path_server}}', + 'formatter': 'code' }, @@ -124,8 +125,8 @@ LOGGING = { 'handlers': [ 'console', - file_debug_handler.name.project, - file_debug_handler.name.server + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', ], 'level': 'DEBUG', From 54654dbf4cde867cd2c6752a865d6794adb5d573 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 11 Feb 2019 23:28:18 +0200 Subject: [PATCH 41/65] update jinja settings so the entire server backend name must be loaded up --- share/templates/conf/django/files/local.jinja2 | 3 +-- share/templates/conf/gunicorn/files/local.jinja2 | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index b276153..e28f538 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -26,8 +26,7 @@ TEMPLATES[0]['DIRS'].append( DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend='{{server_database_backend}}'), + 'ENGINE': '{{server_database_backend}}', 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index b276153..e28f538 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -26,8 +26,7 @@ TEMPLATES[0]['DIRS'].append( DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.{db_backend}'.format( - db_backend='{{server_database_backend}}'), + 'ENGINE': '{{server_database_backend}}', 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), From 0211723fe9c14a365bc926d7db00fd8c873e4797 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 11 Feb 2019 23:50:30 +0200 Subject: [PATCH 42/65] add import for custom local settings --- modules/pip.py | 8 ++++---- share/templates/conf/django/files/local.jinja2 | 2 ++ share/templates/conf/gunicorn/files/local.jinja2 | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/pip.py b/modules/pip.py index d47e405..44e707e 100644 --- a/modules/pip.py +++ b/modules/pip.py @@ -5,8 +5,8 @@ import sys import os import logging -from utils import virtualenv_source, virtualenv -from utils import print_console, printerr +from .utils import virtualenv_source, virtualenv +from .utils import print_console, printerr ERROR_BAD_BRANCH_PARAM = -3 ERROR_BAD_PARAM = -2 @@ -230,8 +230,8 @@ def copyto(branch): configuration.virtualenv.requirements.local, branch_config.virtualenv.requirements.filename) - print "current_local_path: %s" % current_local_path - print "branch_local_path: %s" % branch_local_path + print("current_local_path: %s" % current_local_path) + print("branch_local_path: %s" % branch_local_path) message = "Copying file from current branch '{branch_src}' to " \ "destination branch '{branch_dst}'. Continue? Y/n ".format( diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index e28f538..a05d2fe 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -134,3 +134,5 @@ LOGGING = { } } +# import this AFTER all the above +from {{ project_name }}._settings.{{ project_branch }}_custom import * diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index e28f538..a05d2fe 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -134,3 +134,5 @@ LOGGING = { } } +# import this AFTER all the above +from {{ project_name }}._settings.{{ project_branch }}_custom import * From b961dad9326e002bdd40988f88b251fdc7fcf027 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Tue, 12 Feb 2019 01:21:05 +0200 Subject: [PATCH 43/65] updated generate so it creates a generated local file that is imported by the normal local file --- modules/django.py | 13 +- modules/initialize.py | 1 + .../conf/django/files/local.generated.jinja2 | 136 ++++++++++++++++++ .../templates/conf/django/files/local.jinja2 | 131 +---------------- .../gunicorn/files/local.generated.jinja2 | 136 ++++++++++++++++++ .../conf/gunicorn/files/local.jinja2 | 131 +---------------- 6 files changed, 287 insertions(+), 261 deletions(-) create mode 100644 share/templates/conf/django/files/local.generated.jinja2 create mode 100644 share/templates/conf/gunicorn/files/local.generated.jinja2 diff --git a/modules/django.py b/modules/django.py index 0fcd71c..bfdd393 100644 --- a/modules/django.py +++ b/modules/django.py @@ -323,11 +323,22 @@ def generate_scripts(template_name, make_copy=False): project_name=project_name) if template_name == 'local': + copy_path = "{project_path}/{project_name}/_settings".format( project_path=project_path, project_name=project_name) build_name = "%s.py" % project_branch + # generate the local generate scripts + generate('local.generated', make_copy) + + elif template_name == 'local.generated': + + copy_path = "{project_path}/{project_name}/_settings".format( + project_path=project_path, + project_name=project_name) + build_name = "%s_generated.py" % project_branch + # add extra local specfic context variables # NOTE: # if you change project settings, you MUST run generate @@ -451,7 +462,7 @@ def generate(param="help", make_copy=False): make_copy must be set to True if you want it to actually do anthing """ - SCRIPT_LIST = ['settings', 'local', 'wsgi'] + SCRIPT_LIST = ['settings', 'local', 'local.generated', 'wsgi'] PARAM_LIST = list(SCRIPT_LIST) PARAM_LIST.append('gunicorn') diff --git a/modules/initialize.py b/modules/initialize.py index 7db1b83..16eb3af 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -396,6 +396,7 @@ def get_config(branchname): add_template(dataobject, layout, config, "django", "settings") add_template(dataobject, layout, config, "django", "local") + add_template(dataobject, layout, config, "django", "local.generated") add_template(dataobject, layout, config, "django", "wsgi") # diff --git a/share/templates/conf/django/files/local.generated.jinja2 b/share/templates/conf/django/files/local.generated.jinja2 new file mode 100644 index 0000000..e28f538 --- /dev/null +++ b/share/templates/conf/django/files/local.generated.jinja2 @@ -0,0 +1,136 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + +LOCAL_SETTINGS = True # avoid recursive imports +BRANCH = '{{ project_branch }}' + +from {{ project_name }}.settings import * + +import logging + +LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH +logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) + +SITE_ID = 1 + +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") + +# +# NOTE: a lot of the code in these local settings files are automated and you +# might be inclined to take them out and move them into the main settings.py +# file. That would be a mistake. These automatic files AT THE VERY LEAST have +# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. +# Once I set that file, THEN all the rest of the information can be automated. +# So all these automated info needs to be here. + +DATABASES = { + 'default': { + 'ENGINE': '{{server_database_backend}}', + + 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), + 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), + 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), + 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), + 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), + } +} + + +# +# directory from which we serve static files +# +# NOTE: both STATIC and MEDIA roots are getting their values from the +# initialization files that are set up above. Also, MEDIA_ROOT is being set +# to something called "paths.server.media.dynamic" - the names are different, +# but it means the same thing. +# +# MEDIA_ROOT is the dynamic media information that the web server, user or +# admin # will be adding and taking out. It's why I call it "dynamic" + +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' + +# directories from which we search for static files to place in STATIC_ROOT +# these static files are located within the project root as opposed to the +# server root location + +STATICFILES_DIRS = ( + os.path.join("{{paths_project_root}}", "share", "media"), +) + +# debug and debug toolbar settings +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +USE_DEBUG_TOOLBAR = DEBUG + +# allow template debug outputs on {{ project_branch }} environment +INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] + +ALLOWED_HOSTS = {{ project_django_allowedhosts }} + +# ----------------------------------------- +# Debug logging to the console + +LOGGING = { + 'version': 1, + 'formatters': { + 'verbose': { + 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" + " %(thread)d %(message)s" + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + + 'code': { + 'format': "%(module)s:%(funcName)s - %(message)s" + }, + }, + 'handlers': { + '{{ file_debug_handler__name_project }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_project}}', + + 'formatter': 'code' + }, + + '{{ file_debug_handler__name_server }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_server}}', + + 'formatter': 'code' + }, + + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'null': { + 'level': 'DEBUG', + 'class': 'logging.NullHandler', + } + }, + 'loggers': { + 'django.debug': { + # use the console for logging + 'handlers': + [ + 'console', + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', + ], + + 'level': 'DEBUG', + 'propagate': True, + }, + } +} + diff --git a/share/templates/conf/django/files/local.jinja2 b/share/templates/conf/django/files/local.jinja2 index a05d2fe..95267b3 100644 --- a/share/templates/conf/django/files/local.jinja2 +++ b/share/templates/conf/django/files/local.jinja2 @@ -1,138 +1,9 @@ -# this file contains settings that are automatically generated -# put custom commands/ imports in a separate file! - LOCAL_SETTINGS = True # avoid recursive imports -BRANCH = '{{ project_branch }}' -from {{ project_name }}.settings import * +from {{ project_name }}._settings.{{ project_branch }}_generated import * import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -SITE_ID = 1 - -TEMPLATES[0]['DIRS'].append( - "{{paths_django_templates}}") - -# -# NOTE: a lot of the code in these local settings files are automated and you -# might be inclined to take them out and move them into the main settings.py -# file. That would be a mistake. These automatic files AT THE VERY LEAST have -# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. -# Once I set that file, THEN all the rest of the information can be automated. -# So all these automated info needs to be here. - -DATABASES = { - 'default': { - 'ENGINE': '{{server_database_backend}}', - - 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), - 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), - 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), - 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), - 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), - } -} - - -# -# directory from which we serve static files -# -# NOTE: both STATIC and MEDIA roots are getting their values from the -# initialization files that are set up above. Also, MEDIA_ROOT is being set -# to something called "paths.server.media.dynamic" - the names are different, -# but it means the same thing. -# -# MEDIA_ROOT is the dynamic media information that the web server, user or -# admin # will be adding and taking out. It's why I call it "dynamic" - -STATIC_ROOT = '{{paths_server_media_static}}' -MEDIA_ROOT = '{{paths_server_media_dynamic}}' - -# directories from which we search for static files to place in STATIC_ROOT -# these static files are located within the project root as opposed to the -# server root location - -STATICFILES_DIRS = ( - os.path.join("{{paths_project_root}}", "share", "media"), -) - -# debug and debug toolbar settings -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -USE_DEBUG_TOOLBAR = DEBUG - -# allow template debug outputs on {{ project_branch }} environment -INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] - -ALLOWED_HOSTS = {{ project_django_allowedhosts }} - -# ----------------------------------------- -# Debug logging to the console - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" - " %(thread)d %(message)s" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - - 'code': { - 'format': "%(module)s:%(funcName)s - %(message)s" - }, - }, - 'handlers': { - '{{ file_debug_handler__name_project }}': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - - 'filename': - '{{file_debug_handler__path_project}}', - - 'formatter': 'code' - }, - - '{{ file_debug_handler__name_server }}': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - - 'filename': - '{{file_debug_handler__path_server}}', - - 'formatter': 'code' - }, - - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - }, - 'null': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', - } - }, - 'loggers': { - 'django.debug': { - # use the console for logging - 'handlers': - [ - 'console', - '{{ file_debug_handler__name_project }}', - '{{ file_debug_handler__name_server }}', - ], - - 'level': 'DEBUG', - 'propagate': True, - }, - } -} - -# import this AFTER all the above -from {{ project_name }}._settings.{{ project_branch }}_custom import * diff --git a/share/templates/conf/gunicorn/files/local.generated.jinja2 b/share/templates/conf/gunicorn/files/local.generated.jinja2 new file mode 100644 index 0000000..e28f538 --- /dev/null +++ b/share/templates/conf/gunicorn/files/local.generated.jinja2 @@ -0,0 +1,136 @@ +# this file contains settings that are automatically generated +# put custom commands/ imports in a separate file! + +LOCAL_SETTINGS = True # avoid recursive imports +BRANCH = '{{ project_branch }}' + +from {{ project_name }}.settings import * + +import logging + +LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH +logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) + +SITE_ID = 1 + +TEMPLATES[0]['DIRS'].append( + "{{paths_django_templates}}") + +# +# NOTE: a lot of the code in these local settings files are automated and you +# might be inclined to take them out and move them into the main settings.py +# file. That would be a mistake. These automatic files AT THE VERY LEAST have +# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. +# Once I set that file, THEN all the rest of the information can be automated. +# So all these automated info needs to be here. + +DATABASES = { + 'default': { + 'ENGINE': '{{server_database_backend}}', + + 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), + 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), + 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), + 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), + 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), + } +} + + +# +# directory from which we serve static files +# +# NOTE: both STATIC and MEDIA roots are getting their values from the +# initialization files that are set up above. Also, MEDIA_ROOT is being set +# to something called "paths.server.media.dynamic" - the names are different, +# but it means the same thing. +# +# MEDIA_ROOT is the dynamic media information that the web server, user or +# admin # will be adding and taking out. It's why I call it "dynamic" + +STATIC_ROOT = '{{paths_server_media_static}}' +MEDIA_ROOT = '{{paths_server_media_dynamic}}' + +# directories from which we search for static files to place in STATIC_ROOT +# these static files are located within the project root as opposed to the +# server root location + +STATICFILES_DIRS = ( + os.path.join("{{paths_project_root}}", "share", "media"), +) + +# debug and debug toolbar settings +DEBUG = True +TEMPLATES[0]['OPTIONS']['debug'] = DEBUG +USE_DEBUG_TOOLBAR = DEBUG + +# allow template debug outputs on {{ project_branch }} environment +INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] + +ALLOWED_HOSTS = {{ project_django_allowedhosts }} + +# ----------------------------------------- +# Debug logging to the console + +LOGGING = { + 'version': 1, + 'formatters': { + 'verbose': { + 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" + " %(thread)d %(message)s" + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + + 'code': { + 'format': "%(module)s:%(funcName)s - %(message)s" + }, + }, + 'handlers': { + '{{ file_debug_handler__name_project }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_project}}', + + 'formatter': 'code' + }, + + '{{ file_debug_handler__name_server }}': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + + 'filename': + '{{file_debug_handler__path_server}}', + + 'formatter': 'code' + }, + + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'null': { + 'level': 'DEBUG', + 'class': 'logging.NullHandler', + } + }, + 'loggers': { + 'django.debug': { + # use the console for logging + 'handlers': + [ + 'console', + '{{ file_debug_handler__name_project }}', + '{{ file_debug_handler__name_server }}', + ], + + 'level': 'DEBUG', + 'propagate': True, + }, + } +} + diff --git a/share/templates/conf/gunicorn/files/local.jinja2 b/share/templates/conf/gunicorn/files/local.jinja2 index a05d2fe..95267b3 100644 --- a/share/templates/conf/gunicorn/files/local.jinja2 +++ b/share/templates/conf/gunicorn/files/local.jinja2 @@ -1,138 +1,9 @@ -# this file contains settings that are automatically generated -# put custom commands/ imports in a separate file! - LOCAL_SETTINGS = True # avoid recursive imports -BRANCH = '{{ project_branch }}' -from {{ project_name }}.settings import * +from {{ project_name }}._settings.{{ project_branch }}_generated import * import logging LOCAL_LOGGING_PREFIX = "%s %%(message)s" % BRANCH logging.basicConfig(format=LOCAL_LOGGING_PREFIX, level=logging.DEBUG) -SITE_ID = 1 - -TEMPLATES[0]['DIRS'].append( - "{{paths_django_templates}}") - -# -# NOTE: a lot of the code in these local settings files are automated and you -# might be inclined to take them out and move them into the main settings.py -# file. That would be a mistake. These automatic files AT THE VERY LEAST have -# one variable, and that is WHICH BRANCH CONFIGURATION FILE ARE WE LOOKING AT. -# Once I set that file, THEN all the rest of the information can be automated. -# So all these automated info needs to be here. - -DATABASES = { - 'default': { - 'ENGINE': '{{server_database_backend}}', - - 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), - 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), - 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), - 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), - 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), - } -} - - -# -# directory from which we serve static files -# -# NOTE: both STATIC and MEDIA roots are getting their values from the -# initialization files that are set up above. Also, MEDIA_ROOT is being set -# to something called "paths.server.media.dynamic" - the names are different, -# but it means the same thing. -# -# MEDIA_ROOT is the dynamic media information that the web server, user or -# admin # will be adding and taking out. It's why I call it "dynamic" - -STATIC_ROOT = '{{paths_server_media_static}}' -MEDIA_ROOT = '{{paths_server_media_dynamic}}' - -# directories from which we search for static files to place in STATIC_ROOT -# these static files are located within the project root as opposed to the -# server root location - -STATICFILES_DIRS = ( - os.path.join("{{paths_project_root}}", "share", "media"), -) - -# debug and debug toolbar settings -DEBUG = True -TEMPLATES[0]['OPTIONS']['debug'] = DEBUG -USE_DEBUG_TOOLBAR = DEBUG - -# allow template debug outputs on {{ project_branch }} environment -INTERNAL_IPS = ['127.0.0.1', '127.0.0.2', '127.0.0.3', ] - -ALLOWED_HOSTS = {{ project_django_allowedhosts }} - -# ----------------------------------------- -# Debug logging to the console - -LOGGING = { - 'version': 1, - 'formatters': { - 'verbose': { - 'format': "%(levelname)s %(asctime)s %(module)s %(process)d" - " %(thread)d %(message)s" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - - 'code': { - 'format': "%(module)s:%(funcName)s - %(message)s" - }, - }, - 'handlers': { - '{{ file_debug_handler__name_project }}': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - - 'filename': - '{{file_debug_handler__path_project}}', - - 'formatter': 'code' - }, - - '{{ file_debug_handler__name_server }}': { - 'level': 'DEBUG', - 'class': 'logging.FileHandler', - - 'filename': - '{{file_debug_handler__path_server}}', - - 'formatter': 'code' - }, - - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - }, - 'null': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', - } - }, - 'loggers': { - 'django.debug': { - # use the console for logging - 'handlers': - [ - 'console', - '{{ file_debug_handler__name_project }}', - '{{ file_debug_handler__name_server }}', - ], - - 'level': 'DEBUG', - 'propagate': True, - }, - } -} - -# import this AFTER all the above -from {{ project_name }}._settings.{{ project_branch }}_custom import * From faa2bbb0c92b36345107819be7b9421de04cc09a Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 13 Feb 2019 21:18:42 +0200 Subject: [PATCH 44/65] update local generated so that it has a cleaer description of the database --- .../conf/django/files/local.generated.jinja2 | 11 +++++------ .../conf/gunicorn/files/local.generated.jinja2 | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/share/templates/conf/django/files/local.generated.jinja2 b/share/templates/conf/django/files/local.generated.jinja2 index e28f538..e23c128 100644 --- a/share/templates/conf/django/files/local.generated.jinja2 +++ b/share/templates/conf/django/files/local.generated.jinja2 @@ -27,12 +27,11 @@ TEMPLATES[0]['DIRS'].append( DATABASES = { 'default': { 'ENGINE': '{{server_database_backend}}', - - 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), - 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), - 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), - 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), - 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), + 'NAME': '{{server_database_name}}', + 'USER': '{{server_database_user}}', + 'PASSWORD': '{{server_database_password}}', + 'HOST': '{{server_database_host}}', + 'PORT': '{{server_database_port}}', } } diff --git a/share/templates/conf/gunicorn/files/local.generated.jinja2 b/share/templates/conf/gunicorn/files/local.generated.jinja2 index e28f538..e23c128 100644 --- a/share/templates/conf/gunicorn/files/local.generated.jinja2 +++ b/share/templates/conf/gunicorn/files/local.generated.jinja2 @@ -27,12 +27,11 @@ TEMPLATES[0]['DIRS'].append( DATABASES = { 'default': { 'ENGINE': '{{server_database_backend}}', - - 'NAME': '{db_name}'.format(db_name='{{server_database_name}}'), - 'USER': '{db_user}'.format(db_user='{{server_database_user}}'), - 'PASSWORD': '{db_pass}'.format(db_pass='{{server_database_password}}'), - 'HOST': '{db_host}'.format(db_host='{{server_database_host}}'), - 'PORT': '{db_port}'.format(db_port='{{server_database_port}}'), + 'NAME': '{{server_database_name}}', + 'USER': '{{server_database_user}}', + 'PASSWORD': '{{server_database_password}}', + 'HOST': '{{server_database_host}}', + 'PORT': '{{server_database_port}}', } } From a380f55dba87b9452a1ee774edcf6e445a271e30 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 28 Mar 2019 14:35:54 +0200 Subject: [PATCH 45/65] some changes to fixtures code --- modules/django.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/django.py b/modules/django.py index bfdd393..d1a4f04 100644 --- a/modules/django.py +++ b/modules/django.py @@ -829,10 +829,15 @@ def fixtures(appname=None, backup=False): if configuration.project.host == 'localhost': p = re.compile("\[localhost\] out:") + for match in p.finditer(output): pass - pos = match.end() - output = output[pos:] + + try: + pos = match.end() + output = output[pos:] + except: + pass # now find the first occurence of [ on a newline, e.g. # [ From beb2156b47d7cf3e1052e4350fda999e668cfff4 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 11 Apr 2019 01:51:25 +0300 Subject: [PATCH 46/65] updated readmes and added new ones --- share/templates/readmes/aws.md | 13 ++- share/templates/readmes/fontawesome.md | 35 ++++++++ share/templates/readmes/foundation.md | 109 +++++++++++++++++++++++++ share/templates/readmes/sass.md | 2 + 4 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 share/templates/readmes/fontawesome.md create mode 100644 share/templates/readmes/foundation.md diff --git a/share/templates/readmes/aws.md b/share/templates/readmes/aws.md index dde69de..eaffcfe 100644 --- a/share/templates/readmes/aws.md +++ b/share/templates/readmes/aws.md @@ -147,9 +147,8 @@ replace it with: ssh-keygen -y -f /path/to/myinstance.pem -2. copy the above results. on osx this would look like: +2. copy the above results - ssh-keygen -y -f /path/to/myinstance.pem | pbcopy 3. log in to the ubuntu instance using default ubuntu user 4. change users to the custom username @@ -162,9 +161,15 @@ replace it with: mkdir .ssh touch .ssh/authorized_keys vim ~/.ssh/authorized_keys + +6. change the permissions -6. now paste in the public_key you got in step 1 -7. log out and test with the username + chown -R username:username_gropus .ssh + chmod 700 .ssh + chmod 600 .ssh/authorized_keys + +7. now paste in the public_key you got in step 1 +8. log out and test with the username ssh username@instance.domain diff --git a/share/templates/readmes/fontawesome.md b/share/templates/readmes/fontawesome.md new file mode 100644 index 0000000..556e0c9 --- /dev/null +++ b/share/templates/readmes/fontawesome.md @@ -0,0 +1,35 @@ +###specify the font path to the "webfonts directory" + +$fa-font-path: "../node_modules/@fortawesome/fontawesome-free/webfonts" !default; + +###when importing from an app-local installation: + +in this case "coffesshop" is the name of the app + +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/fontawesome'` +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/solid'` +`@import 'coffeeshop/node_modules/\@fortawesome/fontawesome-free/scss/brands'` + +### when importing from a shared directory + +#### set the location in static and sass + +`STATICFILES_DIRS = [ + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/@fortawesome'), +]` + +`SASS_PROCESSOR_INCLUDE_DIRS = [ + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/@fortawesome'), +]` + +#### now import directly from where you set loc above + +`@import 'fontawesome-free/scss/fontawesome'` +`@import 'fontawesome-free/scss/solid'` +`@import 'fontawesome-free/scss/brands'` + + diff --git a/share/templates/readmes/foundation.md b/share/templates/readmes/foundation.md new file mode 100644 index 0000000..5a4a1cd --- /dev/null +++ b/share/templates/readmes/foundation.md @@ -0,0 +1,109 @@ +# How to install Foundation Zurb + +## Resources +* [node js](https://nodejs.org/en/download/) +* [foundation zurb](foundation.zurb.com) + + +## install nodeenv + +[how to install nodeenv](https://calvinx.com/2013/07/11/python-virtualenv-with-node-environment-via-nodeenv/) + +1. mkvirtualenv myproject1 +2. pip install nodeenv +3. nodeenv -p +4. deactivate; workon myproject1 +5. npm install -g foundation-cli +6. npm install -g autoprefixer +7. npm install -g generate +8. npm install -g postcss-cli + +## install django sass processor + +github url [django sass processor](https://github.com/jrief/django-sass-processor) + +`pip install django-sass-processor` + +### add to installed apps + +`sass_processor` + +### additional configuration settings + +#### set extensions to look for + +sass processor searches template files to find links to sass stylesheets. If using templates with a .django extension, you have to set the ext types you want the processor to search + +`SASS_TEMPLATE_EXTS = ['.django', '.html']` + +#### file search root directory + +set the root value for which sass processor will search for sass files (set to the same value as static root) + +`SASS_PROCESSOR_ROOT = STATIC_ROOT` + +#### add cssfinder to static file finders + +we need to tell the static finders thingie to use sass_processor + +`STATICFILES_FINDERS = [` +` 'django.contrib.staticfiles.finders.FileSystemFinder',` +` 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ` +` 'sass_processor.finders.CssFinder', ]` + +#### directories to look through + +you must tell django wher eto find static files that you want to look through if you want to use them in a template + +you must also tell the sass processor where those files are if you want the sass processor to be able to do anything with it + +`STATICFILES_DIRS = [` +` ('node_modules', os.path.join(PROJECT_ROOT, 'share/media/node_modules/')),` +` os.path.join(PROJECT_ROOT,'share/media/node_modules/foundation-sites/scss'), ]` + +`SASS_PROCESSOR_INCLUDE_DIRS = [` +` os.path.join(PROJECT_ROOT, 'share/media/node_modules'),` +` os.path.join(PROJECT_ROOT, 'share/media/node_modules/foundation-sites/scss'), ]` + +Notice how "node modules" is attached without a prefix. + +also, I add direct links to the scss of the underlying packages so I can import them with a direct command + +`@import foundation` + +_instead of_ + +`@import foundation-sites/scss/foundation` + +When you want to import sass modules underneath the "node_modules" directory you must refer to the directory structure _underneath_ the node modules structure: + +`@import foundation-sites/scss/foundation` + +however, to directly import the foundation sass library, you must do so through the "assets" directory referenced in static. + +so foundation-sites/assets refers to what will be used in static, and foundation-sites/scss/foundation refers to compiling + +### Usage + +`{% load sass_tags %}` + +`` + + +## installing Zurb Foundation Template + +[how to install foundation with django](https://www.physics.utoronto.ca/~icom/workshop/django-install/foundation.html) + +either open up the virtualenv that contains foundation-cli or create one, or just install foundation-cli + +create a new template +`foundation new` + +## installing via foundation-sites + +npm install foundation-sites + + + + diff --git a/share/templates/readmes/sass.md b/share/templates/readmes/sass.md index 8841319..ce517ae 100644 --- a/share/templates/readmes/sass.md +++ b/share/templates/readmes/sass.md @@ -15,6 +15,8 @@ let g:syntastic_sass_sass_args=" -I /Full/share/media/projectwide_sass_files" ```` +you can also add multiple directories to the -I directive. Separate them with a space + [fixing syntastic check args](http://stackoverflow.com/questions/29041876/fixing-syntasticcheck-bootstrap-error-vim-syntastic-plugin) [syntastic manual](https://github.com/vim-syntastic/syntastic/blob/master/doc/syntastic.txt) \ No newline at end of file From 880ce718e7c164fb8c583ce17134711db6cb6ed6 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 11 Apr 2019 17:24:50 +0300 Subject: [PATCH 47/65] there was a bug where any build/* dir was being ignored, even build/* in the static apps, so I set it to /usr/etc/*/build/* --- modules/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/deploy.py b/modules/deploy.py index f7f9df1..a4dbee9 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -377,7 +377,7 @@ def sync(full=True, extras=False): command_mkdir_remote = "mkdir -p {remote}".format(remote=remote_dir) excludeitems = (".git", "*.swp", "*.swo", ".DS_Store", "*.pyc", "*.bak", - "build/*", "/extras", "/opt") + "/usr/etc/*/build/*", "/extras", "/opt") if env.debug: print("\n%s debug: %s" % (debug_prefix, env.debug)) From 35cfbcd4fd0152532252b2e06dfed199e11b84cd Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Apr 2019 11:14:13 +0300 Subject: [PATCH 48/65] got rid of unnecessary server database name for admin --- modules/database.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/database.py b/modules/database.py index e56dbf1..a7432d0 100644 --- a/modules/database.py +++ b/modules/database.py @@ -204,7 +204,6 @@ def commandline(dbuser='default'): if dbuser == 'admin': user = configuration.server.database.admin.user - db_name = configuration.server.database.admin.db_name elif dbuser == 'default': user = configuration.server.database.user From e83adf4a15e187590ecc1208e86eb6fe0cc97791 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 17 Apr 2019 11:15:30 +0300 Subject: [PATCH 49/65] update the database creation files to make everything utf8 --- share/templates/conf/database/files/db.init.sql.jinja2 | 2 +- share/templates/conf/database/files/db.re_init.sql.jinja2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/templates/conf/database/files/db.init.sql.jinja2 b/share/templates/conf/database/files/db.init.sql.jinja2 index dd8ee81..c8bc93a 100644 --- a/share/templates/conf/database/files/db.init.sql.jinja2 +++ b/share/templates/conf/database/files/db.init.sql.jinja2 @@ -1,5 +1,5 @@ CREATE USER {{db_user}} WITH PASSWORD '{{db_password}}'; ALTER USER {{db_user}} CREATEDB; -CREATE DATABASE {{db_name}}; +CREATE DATABASE {{db_name}} WITH ENCODING='UTF8'; ALTER DATABASE {{db_name}} OWNER TO {{db_user}}; GRANT ALL PRIVILEGES ON DATABASE {{db_name}} TO {{db_user}}; diff --git a/share/templates/conf/database/files/db.re_init.sql.jinja2 b/share/templates/conf/database/files/db.re_init.sql.jinja2 index d9e25c6..e1b4261 100644 --- a/share/templates/conf/database/files/db.re_init.sql.jinja2 +++ b/share/templates/conf/database/files/db.re_init.sql.jinja2 @@ -1,4 +1,4 @@ DROP DATABASE {{db_name}}; -CREATE DATABASE {{db_name}}; +CREATE DATABASE {{db_name}} WITH ENCODING='UTF8'; ALTER DATABASE {{db_name}} OWNER TO {{db_user}}; GRANT ALL PRIVILEGES ON DATABASE {{db_name}} TO {{db_user}}; From 0e69a5d65a27968df659d6633e8a5f49103eeea3 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 2 Jun 2019 15:07:45 +0300 Subject: [PATCH 50/65] add machine name to docker --- modules/docker.py | 16 +++++++++++++--- modules/initialize.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/modules/docker.py b/modules/docker.py index e64f277..269fa59 100644 --- a/modules/docker.py +++ b/modules/docker.py @@ -13,17 +13,27 @@ def docker_ip(): configuration = env.config if configuration.docker.database.host == 'local': - docker_cmd = 'docker-machine ip default' + docker_cmd = 'docker-machine ip %s' % \ + configuration.docker.machine + return run(docker_cmd) + else: return configuration.docker.database.host def docker_run(cmd): from fabric.context_managers import prefix + configuration = env.config - docker_eval = "eval $(docker-machine env default)" - with prefix(docker_eval): + if configuration.docker.machine: + docker_eval = "eval $(docker-machine env %s)" % \ + configuration.docker.machine + + with prefix(docker_eval): + run(cmd) + + else: run(cmd) diff --git a/modules/initialize.py b/modules/initialize.py index 16eb3af..29753fd 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -863,7 +863,7 @@ def _init_docker(configuration, layout, config): project_branch=configuration.project.branch) if 'name' in config['docker'] and config['docker']['name']: - configuration.docker.name = config['docker']['name'] + configuration.docker.name = config['docker']['name'] # # host information @@ -871,6 +871,15 @@ def _init_docker(configuration, layout, config): if 'host' in config['docker']: configuration.docker.host = config['docker']['host'] + # + # get machine name for docker-machine if exists + + configuration.docker.machine = None + + if 'machine' in config['docker']: + configuration.docker.machine = config['docker']['machine'] + + # # # configuration info for docker database From 926a126fb8926118e23927b2113ab046dc4021ad Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 2 Jun 2019 22:31:41 +0300 Subject: [PATCH 51/65] add edit for meta configuration files --- modules/deploy.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/modules/deploy.py b/modules/deploy.py index a4dbee9..9504760 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -737,3 +737,51 @@ def ssh(): logger.debug("ssh command: %s" % cmd_ssh) else: local(cmd_ssh) + + +@task +def edit(param='help', branch=None): + """ + calls up mvim on the configuration files for the branch + """ + + from .maintenance import edit as maintenance_edit + from .maintenance import _get_configuration_path + + configuration = env.config + + if not branch: + project_branch = configuration.project.branch + else: + project_branch = branch + + config_path = _get_configuration_path('config', project_branch) + + # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', + # 'settings', 'local'] + locations = { + 'configuration': { + 'path': config_path, + 'desc': 'meta configuration file', + }, + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab deploy.edit" automates editing branch configuration files + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return From 381c8a0568c5132d1b070b6aa27bc16fc6c9457b Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 2 Jun 2019 22:33:41 +0300 Subject: [PATCH 52/65] add edit for layout --- modules/deploy.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/deploy.py b/modules/deploy.py index 9504760..98574c0 100644 --- a/modules/deploy.py +++ b/modules/deploy.py @@ -755,15 +755,21 @@ def edit(param='help', branch=None): else: project_branch = branch - config_path = _get_configuration_path('config', project_branch) + meta_path = _get_configuration_path('config', project_branch) + layout_path = _get_configuration_path('layout', project_branch) # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', # 'settings', 'local'] locations = { - 'configuration': { - 'path': config_path, + 'meta': { + 'path': meta_path, 'desc': 'meta configuration file', }, + + 'layout': { + 'path': layout_path, + 'desc': 'layout configuration file', + }, } if param in locations.keys(): From 8ae7324ea3921cd47bcb19e0a1a1900322899ad6 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 2 Jun 2019 22:52:07 +0300 Subject: [PATCH 53/65] add debug for run_database_command to only print out the command --- modules/database.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/database.py b/modules/database.py index a7432d0..1a5ee7d 100644 --- a/modules/database.py +++ b/modules/database.py @@ -455,6 +455,15 @@ def run_database_command(cmd_string, username, hide=False): cmd_string=cmd_string, ) + if env.debug: + + # print out the full command and return + + logger = loggify('database', 'run_database_command') + logger.debug("run_database_command = %s" % cmd_full) + + return + # NOTE # DO NOT USE "with hide" # one of the commands is to access to psql commandline From dd797c3a77184c4f32d8a8daf57c8013f9e9e89e Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 2 Jun 2019 22:52:21 +0300 Subject: [PATCH 54/65] add generated to list of editable files --- modules/django.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/django.py b/modules/django.py index d1a4f04..c144185 100644 --- a/modules/django.py +++ b/modules/django.py @@ -645,6 +645,11 @@ def edit(param='help'): django_path=django_path, project_branch=project_branch) + settings_generated_path = "{django_path}/_settings/" \ + "{project_branch}_generated.py".format( + django_path=django_path, + project_branch=project_branch) + file_debug_handler = configuration.logging.django.handlers.file_debug # locations = ['gunicorn', 'gunicorn_link', 'gunicorn_build', @@ -680,6 +685,11 @@ def edit(param='help'): 'desc': 'local settings file for django project', }, + 'generated': { + 'path': settings_generated_path, + 'desc': 'generated settings file for django project', + }, + 'server_log': { 'path': file_debug_handler.path.server, 'desc': 'django server log - handler name: %s' % From 05a8ca6b1c9f3892887bb1a26020c74c8acc2d29 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 3 Jun 2019 22:01:02 +0300 Subject: [PATCH 55/65] add edit to docker --- modules/docker.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/modules/docker.py b/modules/docker.py index 269fa59..10ff6c2 100644 --- a/modules/docker.py +++ b/modules/docker.py @@ -84,6 +84,7 @@ def generate(): context['docker_database_port_internal'] = \ configuration.docker.database.port + context['projectbranch'] = configuration.project.branch context['database_user'] = configuration.server.database.admin.user context['database_pass'] = configuration.server.database.admin.password context['database_name'] = configuration.server.database.name @@ -117,6 +118,7 @@ def generate(): else: config_src = configuration.templates.docker.database.src + upload_template( filename=config_src, destination=build_path, @@ -220,3 +222,74 @@ def stop(remove=False): if remove: docker_run(docker_rm) + + +@task +def edit(param='help'): + """ + calls up mvim on the docker conf and build files + """ + + from .maintenance import edit as maintenance_edit + + configuration = env.config + print('keys: %s' % configuration.templates.docker.keys()) + print('path: %s' % configuration.templates.docker.path.keys()) + print('path.remote: %s' % configuration.templates.docker.path.remote) + print('path.local: %s' % configuration.templates.docker.path.local) + print('database: %s' % configuration.templates.docker.database.keys()) + print('database.src: %s' % configuration.templates.docker.database.src) + print('database.dst: %s' % configuration.templates.docker.database.dst) + + import os + + database_build_path = os.path.join( + configuration.templates.docker.path.remote, + 'build', + configuration.templates.docker.database.dst + ) + + database_template_path = os.path.join( + configuration.templates.docker.path.remote, + 'files', + configuration.templates.docker.database.src + ) + + print('database build path: %s' % database_build_path) + print('database template path: %s' % database_template_path) + + project_branch = configuration.project.branch + + locations = { + 'database': { + 'path': database_build_path, + 'desc': 'docker database file in scripts/conf/docker/build', + }, + + 'template': { + 'path': database_template_path, + 'desc': 'docker template file for database is' + 'scripts/conf/docker/files', + } + } + + if param in locations.keys(): + remote_path = locations[param]['path'] + maintenance_edit(remote_path=remote_path) + else: + # if param == 'help': + + print(""" + "fab docker.edit" automates editing template configuration files + for docker database + + to use this you must pass one of the editable locations in as a + parameter + + currently editable locations are: + """) + + for k_loc in locations.keys(): + print("\t{0: <20} - {1}".format(k_loc, locations[k_loc]['desc'])) + + return From 01c122ffcb06649f7e0d5976dc2c269beb81bba9 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Fri, 21 Jun 2019 14:53:12 +0300 Subject: [PATCH 56/65] update django manage so it has better info on debug --- modules/django.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/django.py b/modules/django.py index c144185..f9e1208 100644 --- a/modules/django.py +++ b/modules/django.py @@ -88,19 +88,25 @@ def manage(args="", workingdir=None): shell='/bin/bash') with cd(configuration.paths.django.root): - output = fabric_ops.run( - "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " + cmd = "{djangoroot}/manage.py {args} --pythonpath='{djangoroot}' " \ "--settings={djangosettings}".format( djangoroot=configuration.paths.django.root, args=args, - djangosettings=configuration.imports.settings, - ), - # MAKE SURE THIS IS ALWAYS HERE! - shell='/bin/bash' - ) + djangosettings=configuration.imports.settings,) - # fabric.run has the ability to give me back the output - return output + if env.debug: + print("command: with cd(%s)" % configuration.paths.django.root) + print("command: fabric_ops.run(%s)" % cmd) + else: + + output = fabric_ops.run( + cmd, + # MAKE SURE THIS IS ALWAYS HERE! + shell='/bin/bash' + ) + + # fabric.run has the ability to give me back the output + return output # NOTE: # there was a major problem using fabric commands of "local" or "prefix" From 0c50929581c356c26866ab08932026033316e9b4 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Fri, 21 Jun 2019 14:56:08 +0300 Subject: [PATCH 57/65] update readmes, and add a few more --- share/templates/readmes/aws.md | 1 + share/templates/readmes/foundation.md | 55 +++++++++++-------- share/templates/readmes/geo.md | 25 +++++++++ .../readmes/set_the_key_file_on_local.md | 15 +++++ share/templates/readmes/setup_foundation.md | 2 + share/templates/readmes/social_auth.md | 45 +++++++++++++++ 6 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 share/templates/readmes/geo.md create mode 100644 share/templates/readmes/set_the_key_file_on_local.md create mode 100644 share/templates/readmes/setup_foundation.md create mode 100644 share/templates/readmes/social_auth.md diff --git a/share/templates/readmes/aws.md b/share/templates/readmes/aws.md index eaffcfe..f5cbb66 100644 --- a/share/templates/readmes/aws.md +++ b/share/templates/readmes/aws.md @@ -234,6 +234,7 @@ AWS has a sudo group that allows a user sudo priveleges ### install python packages sudo apt-get install python-dev + sudo apt-get install python3-dev sudo apt-get install libjpeg-dev diff --git a/share/templates/readmes/foundation.md b/share/templates/readmes/foundation.md index 5a4a1cd..a08e959 100644 --- a/share/templates/readmes/foundation.md +++ b/share/templates/readmes/foundation.md @@ -22,11 +22,11 @@ github url [django sass processor](https://github.com/jrief/django-sass-processor) -`pip install django-sass-processor` + pip install django-sass-processor ### add to installed apps -`sass_processor` + sass_processor ### additional configuration settings @@ -34,22 +34,23 @@ github url [django sass processor](https://github.com/jrief/django-sass-processo sass processor searches template files to find links to sass stylesheets. If using templates with a .django extension, you have to set the ext types you want the processor to search -`SASS_TEMPLATE_EXTS = ['.django', '.html']` + SASS_TEMPLATE_EXTS = ['.django', '.html'] #### file search root directory set the root value for which sass processor will search for sass files (set to the same value as static root) -`SASS_PROCESSOR_ROOT = STATIC_ROOT` + SASS_PROCESSOR_ROOT = STATIC_ROOT #### add cssfinder to static file finders we need to tell the static finders thingie to use sass_processor -`STATICFILES_FINDERS = [` -` 'django.contrib.staticfiles.finders.FileSystemFinder',` -` 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ` -` 'sass_processor.finders.CssFinder', ]` + STATICFILES_FINDERS = [ + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'sass_processor.finders.CssFinder', + ] #### directories to look through @@ -57,27 +58,36 @@ you must tell django wher eto find static files that you want to look through if you must also tell the sass processor where those files are if you want the sass processor to be able to do anything with it -`STATICFILES_DIRS = [` -` ('node_modules', os.path.join(PROJECT_ROOT, 'share/media/node_modules/')),` -` os.path.join(PROJECT_ROOT,'share/media/node_modules/foundation-sites/scss'), ]` + STATICFILES_DIRS = [ + ('node_modules', + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/')), + + os.path.join( + PROJECT_ROOT, + 'share/media/node_modules/foundation-sites/scss'), + ] -`SASS_PROCESSOR_INCLUDE_DIRS = [` -` os.path.join(PROJECT_ROOT, 'share/media/node_modules'),` -` os.path.join(PROJECT_ROOT, 'share/media/node_modules/foundation-sites/scss'), ]` + SASS_PROCESSOR_INCLUDE_DIRS = [ + os.path.join(PROJECT_ROOT, 'share/media/node_modules'), + os.path.join(PROJECT_ROOT, + 'share/media/node_modules/foundation-sites/scss'), + ] Notice how "node modules" is attached without a prefix. also, I add direct links to the scss of the underlying packages so I can import them with a direct command -`@import foundation` + @import foundation _instead of_ -`@import foundation-sites/scss/foundation` + @import foundation-sites/scss/foundation When you want to import sass modules underneath the "node_modules" directory you must refer to the directory structure _underneath_ the node modules structure: -`@import foundation-sites/scss/foundation` + @import foundation-sites/scss/foundation however, to directly import the foundation sass library, you must do so through the "assets" directory referenced in static. @@ -85,10 +95,10 @@ so foundation-sites/assets refers to what will be used in static, and foundation ### Usage -`{% load sass_tags %}` + {% load sass_tags %} -`` + ## installing Zurb Foundation Template @@ -98,11 +108,12 @@ so foundation-sites/assets refers to what will be used in static, and foundation either open up the virtualenv that contains foundation-cli or create one, or just install foundation-cli create a new template -`foundation new` + + foundation new ## installing via foundation-sites -npm install foundation-sites + npm install foundation-sites diff --git a/share/templates/readmes/geo.md b/share/templates/readmes/geo.md new file mode 100644 index 0000000..9984965 --- /dev/null +++ b/share/templates/readmes/geo.md @@ -0,0 +1,25 @@ +##if you get the following error: + +django.core.exceptions.ImproperlyConfigured: Could not find the GDAL library (tried "gdal", "GDAL", "gdal2.2.0", "gdal2.1.0", "gdal2.0.0", "gdal1.11.0", "gdal1.10.0", "gdal1.9.0"). Is GDAL installed? If it is, try setting GDAL_LIBRARY_PATH in your settings. + +you must install on the host machine the following packages: +`sudo apt-get install binutils libproj-dev gdal-bin` + +next you must update POstgres to use POSTGIS + +[https://docs.djangoproject.com/en/2.1/ref/contrib/gis/install/postgis/](https://docs.djangoproject.com/en/2.1/ref/contrib/gis/install/postgis/) + +## update postgres + +### the following packages must be installed: +(x.x matching the PostgreSQL version you want to install) + +postgresql-x.x +postgresql-x.x-postgis +postgresql-server-dev-x.x +python-psycopg2 + +find out the postgres server number: + +`apt list --installed | grep 'post'` + diff --git a/share/templates/readmes/set_the_key_file_on_local.md b/share/templates/readmes/set_the_key_file_on_local.md new file mode 100644 index 0000000..2af0f1a --- /dev/null +++ b/share/templates/readmes/set_the_key_file_on_local.md @@ -0,0 +1,15 @@ +`Host **my-remote-server**` *a symbolic name of your choice* + + +`Hostname **server-IP-or-DNS**` *the real name* + +`User **username**` *the username you use for login* + +`IdentityFile ~/.ssh/a_suitable_ssh_key_if_any` *for authentication purposes* + +`ControlMaster auto` + +`ControlPath ~/.ssh/%C` +*or* `~/.ssh/%h-%p-%r` *for older versions* + +`ControlPersist 5m` *or* `yes` *for never expiring persistent connection* \ No newline at end of file diff --git a/share/templates/readmes/setup_foundation.md b/share/templates/readmes/setup_foundation.md new file mode 100644 index 0000000..c0903dd --- /dev/null +++ b/share/templates/readmes/setup_foundation.md @@ -0,0 +1,2 @@ +node.js +https://lincolnloop.com/blog/installing-nodejs-and-npm-python-virtualenv/ diff --git a/share/templates/readmes/social_auth.md b/share/templates/readmes/social_auth.md new file mode 100644 index 0000000..c397c38 --- /dev/null +++ b/share/templates/readmes/social_auth.md @@ -0,0 +1,45 @@ +# Social Auth notes + +## basic python_social_auth + +### How to Add Social Login to Django +[How to Add Social Login to Django](https://simpleisbetterthancomplex.com/tutorial/2016/10/24/how-to-add-social-login-to-django.html) + +#### pre test notes +1. disable 'test_social.middleware.ForwardHQMiddleware' +2. reset LOGIN_REDIRECT_URL = '/tsoc' or to wherever you want it to land +3. set SOCIAL_AUTH_URL_NAMESPACE = 'social' +4. DON'T use localhost if you plan on using twitter, twitter doesn't like it + +#### error in login +[stackoverflow desc](https://stackoverflow.com/questions/46975657/attributeerror-module-django-contrib-auth-views-has-no-attribute-login) + +change: +`path('login/', views.login, name='login')` +`path('login/', views.LoginView.as_view(template_name="registration/login.django"), name='login')` + +#### when making callback urls +- you can use localhost +- you can also specify a port on localhost, e.g. http://localhost:8026 +- you can also use minyanfinder.stg or any .stg + +- make sure to specify the "u" letter in oauth +- make sure to add the final forward slash "/" at the end of the callback url + +`minyanfinder.stg/oauth/complete/twitter/` + + + +## global variables +global variables should be in settings.py + +local info like the specific client id/secret key, etc, should be in the local setting sfiles + +test_social should only be activated in the local setting files + + +### twitter notes +1. you want to either use http or https, be specific +2. twitter does not use localhost and forwardhq is buggy +3. twitter DOES register 127.0.0.1:8026 as a callback + From 99c5d258994f5077241132d8ef2391b11128b744 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 23 Jun 2019 18:23:08 +0300 Subject: [PATCH 58/65] update deploy_meta to work with PyQt5 --- bin/deploy_meta.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/bin/deploy_meta.py b/bin/deploy_meta.py index c4d2c3c..03d02f3 100644 --- a/bin/deploy_meta.py +++ b/bin/deploy_meta.py @@ -1,7 +1,10 @@ import sys import yaml import os -from PyQt4 import QtGui, QtCore +from PyQt5.QtWidgets import QMainWindow, QLineEdit, QLabel, QAction +from PyQt5.QtWidgets import QFileDialog, QGridLayout, QWidget, QApplication +from PyQt5.QtWidgets import QMessageBox +from PyQt5.QtCore import QVariant class DictQuery(dict): @@ -74,7 +77,7 @@ def nested_set(dic, keys, value): dic[keys[-1]] = value -class DeployMeta(QtGui.QMainWindow): +class DeployMeta(QMainWindow): PATH_META = "../../../meta/project" PATH_TEMPLATES = "../share/templates/meta" @@ -273,7 +276,7 @@ class DeployMeta(QtGui.QMainWindow): path_config_full = os.path.join( self.path_templates, self.CONFIG_TYPES[configname]) - configuration_file = yaml.load(file(path_config_full, 'r')) + configuration_file = yaml.load(open(path_config_full, 'r')) return configuration_file def store_config(self): @@ -322,8 +325,8 @@ class DeployMeta(QtGui.QMainWindow): title = self.widgets[key]['title'] - label = QtGui.QLabel(title) - field = QtGui.QLineEdit() + label = QLabel(title) + field = QLineEdit() grid.addWidget(label, row, 0) grid.addWidget(field, row, 1) @@ -337,19 +340,23 @@ class DeployMeta(QtGui.QMainWindow): desc = "Load Configuration %s" % title - loadAction = QtGui.QAction(menuname, self) + loadAction = QAction(menuname, self) loadAction.setShortcut(shortcut) loadAction.setStatusTip(desc) loadAction.triggered.connect(self.action_loadconfig) - variant = QtCore.QVariant(key) + variant = QVariant(key) loadAction.setData(variant) return loadAction def action_loadconfig(self): sender = self.sender() - self.currentbranch = sender.data().toString().__str__() + + if type(sender.data()) is str: + self.currentbranch = sender.data() + else: + self.currentbranch = sender.data().toString().__str__() self.config_data = self.load_config(self.currentbranch) @@ -392,23 +399,22 @@ class DeployMeta(QtGui.QMainWindow): # # get the path value from the dialog box - path_newconfig = QtGui.QFileDialog.getSaveFileName( + path_newconfig = QFileDialog.getSaveFileName( self, dialog_title, path_default_save, - selectedFilter='*.yml') + filter='*.yml') # # if the user hit 'cancel' path_newconfig will be empty if path_newconfig: - stream = file(path_newconfig, 'w') + stream = open(path_newconfig[0], 'w') yaml.dump(self.config_data, stream) else: # # display message warning no configuration has been loaded - from PyQt4.QtGui import QMessageBox msg = QMessageBox() msg.setIcon(QMessageBox.Warning) msg.setText('Save Error') @@ -423,12 +429,12 @@ class DeployMeta(QtGui.QMainWindow): menubar = self.menuBar() menubar.setNativeMenuBar(False) - exitAction = QtGui.QAction('&Exit', self) + exitAction = QAction('&Exit', self) exitAction.setShortcut('Ctrl+Q') exitAction.setStatusTip("Exit application") - exitAction.triggered.connect(QtGui.qApp.quit) + exitAction.triggered.connect(self.close) - saveAction = QtGui.QAction('&Gore', self) + saveAction = QAction('&Gore', self) saveAction.setShortcut('Ctrl+T') saveAction.setStatusTip("Save Config File") saveAction.triggered.connect(self.action_save_config) @@ -446,7 +452,7 @@ class DeployMeta(QtGui.QMainWindow): self.setupMenu() - grid = QtGui.QGridLayout() + grid = QGridLayout() grid.setSpacing(10) self.add_widgetrow('PROJECT_NAME', grid) @@ -466,7 +472,7 @@ class DeployMeta(QtGui.QMainWindow): self.add_widgetrow('DJANGO_PORT', grid) self.add_widgetrow('NGINX_PORT', grid) - central = QtGui.QWidget() + central = QWidget() central.setLayout(grid) self.setCentralWidget(central) @@ -477,7 +483,7 @@ class DeployMeta(QtGui.QMainWindow): def main(): - app = QtGui.QApplication(sys.argv) + app = QApplication(sys.argv) app.setStyle("cleanlooks") ex = DeployMeta() sys.exit(app.exec_()) From d25fde92ad23146a20256e25eeff2c5ff273ae48 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 23 Jun 2019 18:25:07 +0300 Subject: [PATCH 59/65] readdme --- readme.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 readme.md diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..e69de29 From 3a25d617aee0ab21db8a7ce4b2bd948e52f5555d Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 23 Jun 2019 19:21:58 +0300 Subject: [PATCH 60/65] update work on development.yml, add more info to readme --- readme.md | 21 +++++++++++++++++++++ share/templates/meta/development.yml | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index e69de29..b04fe95 100644 --- a/readme.md +++ b/readme.md @@ -0,0 +1,21 @@ +## adding fabric + +`git add submodule git@bitbucket.org:ronnyabraham/fabric.git fabric` + +the second "fabric" puts the submogule in a directory called 'fabric" + +workon a virtualenv that has PyQt5 + +load up bin/deploy_meta.py + +check django.numbers to make sure your port numbers aren't already being used +- database +- django + +after you create the dev file, go to usr/meta/projects and copy the layout.yml file from another project + +now go to usr/meta/virtualenv, copy bootstrap.txt to development.txt +replace pycrypto with pycryptodome +replace psycopg2 with psycopg2-binary + +add allowed_hosts = [127.0.0.1] to the django section in development.yml diff --git a/share/templates/meta/development.yml b/share/templates/meta/development.yml index 1ea9a1e..b6db090 100644 --- a/share/templates/meta/development.yml +++ b/share/templates/meta/development.yml @@ -6,7 +6,10 @@ database: users: admin: {name: DATABASE_ADMIN_NAME, pass: DATABASE_ADMIN_PASS} default: {name: DATABASE_USER_NAME, pass: DATABASE_USER_PASS} -django: {host: DJANGO_IP, port: DJANGO_PORT} +django: + host: DJANGO_IP + port: DJANGO_PORT + allowed_hosts: [127.0.0.1, 'PROJECT_NAME.BRANCH_EXT',] docker: database: env: {name: POSTGRES_DB, pass: POSTGRES_PASSWORD, user: POSTGRES_USER} From 4c413005c1844b58ed4062dd7e14df2d3f43c805 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 27 Jun 2019 18:09:15 +0300 Subject: [PATCH 61/65] updated social auth readme --- share/templates/readmes/social_auth.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/share/templates/readmes/social_auth.md b/share/templates/readmes/social_auth.md index c397c38..080254c 100644 --- a/share/templates/readmes/social_auth.md +++ b/share/templates/readmes/social_auth.md @@ -43,3 +43,28 @@ test_social should only be activated in the local setting files 2. twitter does not use localhost and forwardhq is buggy 3. twitter DOES register 127.0.0.1:8026 as a callback + +### facebook notes + +1. can use localhost +2. minaynfinder.stg is not working, need a certificate of some kind +3. forwardhq also not working + +#### using a domain + +##### create registered users + +Go to Roles->Test Users + +##### create the ssl +1. set a registered domain name to point at the ip address (ie ronnyabraham.com) +2. create a [https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs#generating-ssl-certificates](crt certificate) + +3. `Settings->Basic-> App Domains` add `ronnyabraham.com` +4. `Settings->Basic->Website/Site URL` add `https://ronnyabraham.com` +5. in Products->FacebookLogin->Settings add the following to "VAlid OAuth Redirect URIs" + +- https://ronnyabraham.com +- https://ronnyabraham.com/oauth/complete/facebook/ + +6. use letsencrypt to create ssl keys From 43fd9e4cd7890e6a6ee3ba48273f97de6b351f2f Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Thu, 27 Jun 2019 19:31:31 +0300 Subject: [PATCH 62/65] add backup to nginx.edit optiosn --- modules/nginx.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/nginx.py b/modules/nginx.py index 6f3aaff..21d0c09 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -69,12 +69,19 @@ def edit(param='help'): configuration.nginx.sites_available, configuration.nginx.conf.name) + backup_file_path = config_file_path + '.bak' + locations = { 'conf': { 'path': config_file_path, 'desc': "nginx configuration file", }, + 'backup': { + 'path': backup_file_path, + 'desc': "backup configuration file", + }, + 'log.error': { 'path': configuration.logging.nginx.error, 'desc': "nginx error log file", @@ -88,7 +95,19 @@ def edit(param='help'): if param in locations.keys(): remote_path = locations[param]['path'] - maintenance_edit(remote_path=remote_path) + + if env.debug: + logger = loggify('nginx', 'edit') + logger.debug("param: %s\n" % param) + + logger.debug("locations:") + for key, value in locations.items(): + logger.debug("%s [path]: %s" % (key, value['path'])) + logger.debug("%s [desc]: %s\n" % (key, value['desc'])) + + logger.debug("remote_path: %s" % remote_path) + else: + maintenance_edit(remote_path=remote_path) else: # if param == 'help': From a0ac1eefa9dfd74a4ddbbc32d4acfdd961952dff Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Sun, 30 Jun 2019 22:42:29 +0300 Subject: [PATCH 63/65] add info for ssl --- modules/initialize.py | 47 ++++++++++++++++++++++++++++++++++++++++--- modules/nginx.py | 1 + 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index 29753fd..416871e 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -120,8 +120,9 @@ def add_template(dataobject, layout, config, section, template_name="conf"): # dataobject.templates.section.tname.src # dataobject.templates.section.tname.dst - # if env.debug: - # logger = loggify('intialize', 'add_template') + if env.debug: + # logger = loggify('intialize', 'add_template') + debug_section = "DEBUG initialize.add_templates[%s]" % section if not hasattr(dataobject.templates, section): dataobject.templates.addbranch(section) @@ -142,12 +143,35 @@ def add_template(dataobject, layout, config, section, template_name="conf"): var_section_source = layout['templates'][section][template_name]['source'] var_section_output = layout['templates'][section][template_name]['output'] + if env.debug and section == 'nginx': + print("%s -- var_section_path: %s" % (debug_section, var_section_path)) + print("%s -- var_section_source: %s" % + (debug_section, var_section_source)) + + print("%s -- templates in config: %s" % + (debug_section, 'templates' in config)) + + print("%s -- nginx in config: %s" % + (debug_section, 'nginx' in config)) + + if 'nginx' in config: + print("%s -- ssl in config[nginx]: %s" % + (debug_section, 'ssl' in config['nginx'])) + print("%s -- port in config[nginx]: %s" % + (debug_section, 'port' in config['nginx'])) + + if 'templates' in config: + print("%s -- templates in config" % debug_section) + if 'templates' in config and section in config['templates']: _config = config['templates'][section] var_section_path = _config['path'] var_section_source = _config[template_name]['source'] var_section_output = _config[template_name]['output'] + if env.debug: + print("%s -- breakpoint 2" % debug_section) + # define the local, and dest paths _template.path.local = os.path.join( dataobject.paths.project.local, @@ -163,9 +187,15 @@ def add_template(dataobject, layout, config, section, template_name="conf"): conf_template = getattr(_template, template_name) conf_template.src = var_section_source - conf_template.dst = var_section_output + if env.debug: + print("%s -- template_name: %s" % (debug_section, template_name)) + print("%s -- conf_template.src: %s" % + (debug_section, conf_template.src)) + print("%s -- conf_template.dst: %s" % + (debug_section, conf_template.dst)) + def get_config(branchname): @@ -363,6 +393,17 @@ def get_config(branchname): dataobject.server.nginx.port = config['nginx']['port'] dataobject.server.nginx.socket = config['nginx']['socket'] + + if 'ssl' in config['nginx']: + + dataobject.server.nginx.addbranch('ssl') + + dataobject.server.nginx.ssl.servername = \ + config['nginx']['ssl_server_name'] + + dataobject.server.nginx.ssl.serverext = \ + config['nginx']['ssl_server_ext'] + print("DEBUG -- socket: %s" % dataobject.server.nginx.socket) if config['project']['host'] == "localhost": diff --git a/modules/nginx.py b/modules/nginx.py index 21d0c09..7a0cbbd 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -151,6 +151,7 @@ def upload(): context['django_host'] = configuration.server.django.host context['django_port'] = configuration.server.django.port + context['project_name'] = configuration.project.name context['extended_name'] = configuration.project.extendedname context['virtualenv_sitepackages'] = \ From 0f94909568c71c81c0b7a14dca358deff2e252a9 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Mon, 1 Jul 2019 20:26:34 +0300 Subject: [PATCH 64/65] add certbot readme and updated social auth readme --- share/templates/readmes/certbot.md | 1 + share/templates/readmes/social_auth.md | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 share/templates/readmes/certbot.md diff --git a/share/templates/readmes/certbot.md b/share/templates/readmes/certbot.md new file mode 100644 index 0000000..b2462cd --- /dev/null +++ b/share/templates/readmes/certbot.md @@ -0,0 +1 @@ +certbot looks through all configuration files and modifies them for ssl diff --git a/share/templates/readmes/social_auth.md b/share/templates/readmes/social_auth.md index 080254c..defab85 100644 --- a/share/templates/readmes/social_auth.md +++ b/share/templates/readmes/social_auth.md @@ -68,3 +68,29 @@ Go to Roles->Test Users - https://ronnyabraham.com/oauth/complete/facebook/ 6. use letsencrypt to create ssl keys + + +### setting up facebook + +1. if used for testing make sure you set the app FOR TESTING you have to do this in the create app stage + +2. [select create test app from the app you like](https://imgur.com/RzK3pS3) + +3. go to settings->basic get the app-id and app-secret + +4. go to Settings-Basic-App Domains and add your FQDN (ie. mycrustyass.com) + +5. go to Settings-Basic-Website (it's near the bottom, add website if it isn't there), add the site URL with it's prefix ie http or https. + +Step 5 is VERY IMPORTANT b/c facebook doesn't recognize http prefix. +You must have SSL set up. But this is NOT the caes with localhost! + +6. in Settings-Products add Facebook Login then set Web OAuth Login to true +7. Settings-Products-Facebook Login add Valid OAuth redirect calls. + +Important! You must find out what the specific redirect call looks like. Once again, facebook doesn't require this for localhost, but for a named host it does + +e.g. https://mycrustyass.com/oauth/complete/facebook/ + +8. test this URL in the Settings-Products-FAcebook Login-Redirect URL Validator + From 0d68ef122d8d92962f221e4acc9b8e39d3a87648 Mon Sep 17 00:00:00 2001 From: Ronny Abraham Date: Wed, 7 Aug 2019 04:18:59 +0300 Subject: [PATCH 65/65] blanketed out annoying debug statements, added code to allow for a custom nginx host name --- modules/initialize.py | 89 +++++++++++++++++++++++++------------------ modules/nginx.py | 4 ++ 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/modules/initialize.py b/modules/initialize.py index 416871e..219ccc0 100644 --- a/modules/initialize.py +++ b/modules/initialize.py @@ -120,9 +120,9 @@ def add_template(dataobject, layout, config, section, template_name="conf"): # dataobject.templates.section.tname.src # dataobject.templates.section.tname.dst - if env.debug: - # logger = loggify('intialize', 'add_template') - debug_section = "DEBUG initialize.add_templates[%s]" % section + # if env.debug: + # # logger = loggify('intialize', 'add_template') + # debug_section = "DEBUG initialize.add_templates[%s]" % section if not hasattr(dataobject.templates, section): dataobject.templates.addbranch(section) @@ -143,25 +143,28 @@ def add_template(dataobject, layout, config, section, template_name="conf"): var_section_source = layout['templates'][section][template_name]['source'] var_section_output = layout['templates'][section][template_name]['output'] - if env.debug and section == 'nginx': - print("%s -- var_section_path: %s" % (debug_section, var_section_path)) - print("%s -- var_section_source: %s" % - (debug_section, var_section_source)) + # debug statements + # + # if env.debug and section == 'nginx': + # print("%s -- var_section_path: %s" % ( + # debug_section, var_section_path)) + # print("%s -- var_section_source: %s" % + # (debug_section, var_section_source)) - print("%s -- templates in config: %s" % - (debug_section, 'templates' in config)) + # print("%s -- templates in config: %s" % + # (debug_section, 'templates' in config)) - print("%s -- nginx in config: %s" % - (debug_section, 'nginx' in config)) + # print("%s -- nginx in config: %s" % + # (debug_section, 'nginx' in config)) - if 'nginx' in config: - print("%s -- ssl in config[nginx]: %s" % - (debug_section, 'ssl' in config['nginx'])) - print("%s -- port in config[nginx]: %s" % - (debug_section, 'port' in config['nginx'])) + # if 'nginx' in config: + # print("%s -- ssl in config[nginx]: %s" % + # (debug_section, 'ssl' in config['nginx'])) + # print("%s -- port in config[nginx]: %s" % + # (debug_section, 'port' in config['nginx'])) - if 'templates' in config: - print("%s -- templates in config" % debug_section) + # if 'templates' in config: + # print("%s -- templates in config" % debug_section) if 'templates' in config and section in config['templates']: _config = config['templates'][section] @@ -169,8 +172,10 @@ def add_template(dataobject, layout, config, section, template_name="conf"): var_section_source = _config[template_name]['source'] var_section_output = _config[template_name]['output'] - if env.debug: - print("%s -- breakpoint 2" % debug_section) + # debug statements + # + # if env.debug: + # print("%s -- breakpoint 2" % debug_section) # define the local, and dest paths _template.path.local = os.path.join( @@ -189,12 +194,14 @@ def add_template(dataobject, layout, config, section, template_name="conf"): conf_template.src = var_section_source conf_template.dst = var_section_output - if env.debug: - print("%s -- template_name: %s" % (debug_section, template_name)) - print("%s -- conf_template.src: %s" % - (debug_section, conf_template.src)) - print("%s -- conf_template.dst: %s" % - (debug_section, conf_template.dst)) + # debug statements + # + # if env.debug: + # print("%s -- template_name: %s" % (debug_section, template_name)) + # print("%s -- conf_template.src: %s" % + # (debug_section, conf_template.src)) + # print("%s -- conf_template.dst: %s" % + # (debug_section, conf_template.dst)) def get_config(branchname): @@ -393,20 +400,26 @@ def get_config(branchname): dataobject.server.nginx.port = config['nginx']['port'] dataobject.server.nginx.socket = config['nginx']['socket'] - - if 'ssl' in config['nginx']: - - dataobject.server.nginx.addbranch('ssl') - - dataobject.server.nginx.ssl.servername = \ - config['nginx']['ssl_server_name'] - - dataobject.server.nginx.ssl.serverext = \ - config['nginx']['ssl_server_ext'] - print("DEBUG -- socket: %s" % dataobject.server.nginx.socket) - if config['project']['host'] == "localhost": + if 'host' in config['nginx']: + + # if we specificed a host name in the + # configuration file under 'nginx' + + # dataobject.server.nginx.addbranch('ssl') + + # dataobject.server.nginx.ssl.host = \ + # config['nginx']['ssl']['host'] + + dataobject.server.nginx.host = \ + config['nginx']['host'] + + elif config['project']['host'] == "localhost": + + # if localhost, then create a nginx appropriate name based on + # the project name and the extension + dataobject.server.nginx.host = "{projectname}.{ext}".format( ext=dataobject.project.extension, projectname=dataobject.project.name) diff --git a/modules/nginx.py b/modules/nginx.py index 7a0cbbd..6cd72cd 100644 --- a/modules/nginx.py +++ b/modules/nginx.py @@ -91,6 +91,10 @@ def edit(param='help'): 'path': configuration.logging.nginx.access, 'desc': "nginx access log file", }, + 'log.nginx.error': { + 'path': '/var/log/nginx/error.log', + 'desc': "nginx main error file" + }, } if param in locations.keys():