From 571115a038d03bde365c7ba4a7b38c2aead9d3a1 Mon Sep 17 00:00:00 2001 From: Pavel Studenik Date: Thu, 19 Nov 2015 15:46:17 +0100 Subject: [PATCH] #40 move and store data to one directory - use for custom storage --- apps/core/management/commands/beaker.py | 2 +- apps/core/management/commands/check_beaker.py | 2 +- apps/core/management/commands/checkrepo.py | 6 ++++- apps/core/management/commands/schedule.py | 2 +- apps/core/utils/beaker.py | 9 ++++---- apps/core/views/jobs.py | 10 +++------ apps/taskomatic/management/commands/pickup.py | 2 +- apps/taskomatic/models.py | 4 ++-- tttt/settings/basic.py | 22 ++++++++++++++++--- 9 files changed, 38 insertions(+), 21 deletions(-) diff --git a/apps/core/management/commands/beaker.py b/apps/core/management/commands/beaker.py index 5bd88f4..dff07c6 100644 --- a/apps/core/management/commands/beaker.py +++ b/apps/core/management/commands/beaker.py @@ -17,7 +17,7 @@ from apps.core.utils.beaker import Beaker from apps.taskomatic.models import TaskPeriod, TaskPeriodSchedule -logger = logging.getLogger(__name__) +logger = logging.getLogger("main") class Command(AdvancedCommand): diff --git a/apps/core/management/commands/check_beaker.py b/apps/core/management/commands/check_beaker.py index 9a4a5c7..ccf284d 100644 --- a/apps/core/management/commands/check_beaker.py +++ b/apps/core/management/commands/check_beaker.py @@ -16,7 +16,7 @@ from apps.core.utils.beaker import Beaker from apps.core.utils.date_helpers import currentDate -logger = logging.getLogger(__name__) +logger = logging.getLogger("main") class Command(BaseCommand): diff --git a/apps/core/management/commands/checkrepo.py b/apps/core/management/commands/checkrepo.py index 8d51194..c368899 100644 --- a/apps/core/management/commands/checkrepo.py +++ b/apps/core/management/commands/checkrepo.py @@ -10,13 +10,14 @@ # Date: 20.07.2014 import logging +import os from django.conf import settings from django.core.management.base import BaseCommand from apps.core.models import Git -logger = logging.getLogger(__name__) +logger = logging.getLogger("main") class Command(BaseCommand): @@ -38,6 +39,9 @@ def handle(self, *args, **kwargs): for repo in settings.REPOSITORIES_GIT[path]: if len(repos) > 0 and repo not in repos: continue + if not os.path.exists("%s%s" % (path, repo)): + logger.error("repo %s doesn't exists in %s" % (repo, path)) + continue git = Git.getGitFromFolder("%s%s" % (path, repo)) git.log = logger if git: diff --git a/apps/core/management/commands/schedule.py b/apps/core/management/commands/schedule.py index eb4a800..3b53cd1 100644 --- a/apps/core/management/commands/schedule.py +++ b/apps/core/management/commands/schedule.py @@ -19,7 +19,7 @@ from apps.core.utils.beaker import Beaker from apps.taskomatic.models import TaskPeriodSchedule -logger = logging.getLogger(__name__) +logger = logging.getLogger("main") class Command(BaseCommand): diff --git a/apps/core/utils/beaker.py b/apps/core/utils/beaker.py index 84ae50d..14aa1ba 100644 --- a/apps/core/utils/beaker.py +++ b/apps/core/utils/beaker.py @@ -131,7 +131,7 @@ def generateXmlFile(self, jobT, filename, reserve=False): if filename.startswith('/'): tmp_path = filename else: - tmp_path = "%s/data/xmls/%s" % (settings.ROOT_PATH, filename) + tmp_path = "%s/xmls/%s" % (settings.STORAGE_ROOT, filename) tmp_path = re.sub(r'//+', '/', tmp_path) tmp_dir = os.path.dirname(tmp_path) if not os.path.exists(tmp_dir): @@ -225,19 +225,20 @@ def parse_job(self, jobid, running=True, date_created=None): def listLogs(self, recipe): #client = self._getXMLRPCClient() - #return client.recipes.files(int(recipe)) + # return client.recipes.files(int(recipe)) raw, status = self.execute("job-logs", recipe) return raw.split() def downloadLog(self, logurl): logparse = urlparse(logurl) - logpath = os.path.join(settings.MEDIA_ROOT, *(urlparse(logurl).path.split("/"))) + logpath = os.path.join( + settings.STORAGE_ROOT, *(urlparse(logurl).path.split("/"))) logdir = os.path.dirname(logpath) if not os.path.exists(logdir): os.makedirs(logdir) rawfile = urllib2.urlopen(logurl) logger.debug("download logfile: %s" % logurl) - of = open(logpath,'wb') + of = open(logpath, 'wb') of.write(rawfile.read()) of.close() diff --git a/apps/core/views/jobs.py b/apps/core/views/jobs.py index e024824..0d4f094 100644 --- a/apps/core/views/jobs.py +++ b/apps/core/views/jobs.py @@ -3,7 +3,6 @@ import hashlib import logging -import sys from copy import copy from datetime import datetime, timedelta @@ -16,16 +15,13 @@ from apps.core.forms import FilterForm from apps.core.models import (RESULT_CHOICES, Author, CheckProgress, Event, - JobTemplate, Recipe, RecipeTemplate, render_label) + JobTemplate, Recipe, render_label) from apps.taskomatic.models import TaskPeriodSchedule from apps.waiver.forms import WaiverForm from apps.waiver.models import Comment from base import create_matrix -if sys.version_info < (2, 7): - from ordereddict import OrderedDict -else: - from collections import OrderedDict +from collections import OrderedDict logger = logging.getLogger(__name__) @@ -233,7 +229,7 @@ def render_to_response(self, context, **response_kwargs): context, **response_kwargs) def prepare_matrix(self, jobs, recipes, label=None): - data, reschedule = OrderedDict(), 0 + data = OrderedDict() if not label: label = create_matrix(settings.PREVIOUS_DAYS) for job in jobs: diff --git a/apps/taskomatic/management/commands/pickup.py b/apps/taskomatic/management/commands/pickup.py index 20e9833..9c1e629 100644 --- a/apps/taskomatic/management/commands/pickup.py +++ b/apps/taskomatic/management/commands/pickup.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = ('Automatization for running task') - requires_model_validation = True + requires_system_checks = True can_import_settings = True def handle(self, *args, **kwargs): diff --git a/apps/taskomatic/models.py b/apps/taskomatic/models.py index 36840be..95b5e76 100644 --- a/apps/taskomatic/models.py +++ b/apps/taskomatic/models.py @@ -39,9 +39,9 @@ def __unicode__(self): return "[%d] %s" % (self.counter, self.title) def recount(self): - it.counter = TaskPeriodSchedule.object.filter( + self.counter = TaskPeriodSchedule.object.filter( period=self.period).count() - it.save() + self.save() class TaskPeriod(models.Model): diff --git a/tttt/settings/basic.py b/tttt/settings/basic.py index 5b2f25d..7973250 100644 --- a/tttt/settings/basic.py +++ b/tttt/settings/basic.py @@ -1,7 +1,6 @@ # Django settings for tttt project. import os - from django.conf import global_settings ROOT_PATH = os.path.abspath("%s/%s/" % @@ -80,6 +79,8 @@ # Example: "/home/media/media.lawrence.com/static/" STATIC_ROOT = "%s/tttt/%s/" % (ROOT_PATH, 'static') +STORAGE_ROOT = "%s/%s/" % (ROOT_PATH, 'storage') + # URL prefix for static files. # Example: "http://media.lawrence.com/static/" STATIC_URL = '/static/' @@ -210,12 +211,22 @@ '()': 'django.utils.log.RequireDebugFalse' } }, + 'formatters': { + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, 'handlers': { 'mail_admins': { 'level': 'ERROR', 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' - } + }, + 'log_to_stdout': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + }, }, 'loggers': { 'django.request': { @@ -223,6 +234,11 @@ 'level': 'ERROR', 'propagate': True, }, + 'main': { + 'handlers': ['log_to_stdout'], + 'level': 'DEBUG', + 'propagate': True, + } } } @@ -269,7 +285,7 @@ # If you want to change directory with repositories, # don't forget changed it in /var/www/gitweb/gitweb_config.perl REPOSITORIES_GIT = { - "~": + "%s/git" % STORAGE_ROOT: ("/tests",), }