From 30a36aa0f6a7001b0c601b1a5e1baa7c462edcd9 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Thu, 26 Jun 2014 11:07:15 +0200 Subject: [PATCH 01/14] [IMP] various code improvements in runbot --- runbot/runbot.py | 79 ++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 8ec2c636..3f301f33 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -35,63 +35,62 @@ _logger = logging.getLogger(__name__) #---------------------------------------------------------- def log(*l, **kw): - out = [] - for i in l: - if not isinstance(i, basestring): - i = repr(i) - out.append(i) - out += ["%s=%r" % (k, v) for k, v in kw.items()] + out = [i if isinstance(i, basestring) else repr(i) for i in l] + \ + ["%s=%r" % (k, v) for k, v in kw.items()] _logger.debug(' '.join(out)) -def dashes(s): +def dashes(string): + """Sanitize the input string""" for i in '~":\'': - s = s.replace(i, "") + string = string.replace(i, "") for i in '/_. ': - s = s.replace(i, "-") - return s + string = string.replace(i, "-") + return string def mkdirs(dirs): - for i in dirs: - if not os.path.exists(i): - os.makedirs(i) + for d in dirs: + if not os.path.exists(d): + os.makedirs(d) -def grep(filename, s): +def grep(filename, string): if os.path.isfile(filename): - return open(filename).read().find(s) != -1 + return open(filename).read().find(string) != -1 return False _re_error = r'^(?:\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ (?:ERROR|CRITICAL) )|(?:Traceback \(most recent call last\):)$' _re_warning = r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ WARNING ' -def rfind(filename, patern): +def rfind(filename, pattern): + """Determine in something in filename matches the pattern""" if os.path.isfile(filename): - p = re.compile(patern, re.M) + regexp = re.compile(pattern, re.M) with open(filename, 'r') as f: - if p.findall(f.read()): + if regexp.findall(f.read()): return True return False -def lock(name): - fd = os.open(name, os.O_CREAT | os.O_RDWR, 0600) +def lock(filename): + fd = os.open(filename, os.O_CREAT | os.O_RDWR, 0600) fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) -def locked(name): - r = False +def locked(filename): + result = False try: - fd = os.open(name, os.O_CREAT | os.O_RDWR, 0600) + fd = os.open(filename, os.O_CREAT | os.O_RDWR, 0600) try: fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: - r = True + result = True os.close(fd) except OSError: - r = False - return r + result = False + return result def nowait(): signal.signal(signal.SIGCHLD, signal.SIG_IGN) def run(l, env=None): + """Run a command described by l in environment env""" log("run", l) env = dict(os.environ, **env) if env else None if isinstance(l, list): @@ -111,14 +110,16 @@ def run(l, env=None): def now(): return time.strftime(openerp.tools.DEFAULT_SERVER_DATETIME_FORMAT) -def dt2time(dt): - return time.mktime(time.strptime(dt, openerp.tools.DEFAULT_SERVER_DATETIME_FORMAT)) +def dt2time(datetime): + """Convert datetime to time""" + return time.mktime(time.strptime(datetime, openerp.tools.DEFAULT_SERVER_DATETIME_FORMAT)) -def s2human(t): - for m,u in [(86400,'d'),(3600,'h'),(60,'m')]: - if t>=m: - return str(int(t/m))+u - return str(int(t))+"s" +def s2human(time): + """Convert a time in second into an human readable string""" + for delay, desc in [(86400,'d'),(3600,'h'),(60,'m')]: + if time >= delay: + return str(int(time / delay)) + desc + return str(int(time)) + "s" #---------------------------------------------------------- # RunBot Models @@ -129,14 +130,14 @@ class runbot_repo(osv.osv): _order = 'name' def _get_path(self, cr, uid, ids, field_name, arg, context=None): - wd = self.root(cr, uid) - r = {} + root = self.root(cr, uid) + result = {} for repo in self.browse(cr, uid, ids, context=context): name = repo.name for i in '@:/': name = name.replace(i, '_') - r[repo.id] = os.path.join(wd, 'repo', name) - return r + result[repo.id] = os.path.join(root, 'repo', name) + return result def _get_base(self, cr, uid, ids, field_name, arg, context=None): r = {} @@ -170,9 +171,9 @@ class runbot_repo(osv.osv): return domain def root(self, cr, uid, context=None): + """Return root directory of repository""" default = os.path.join(os.path.dirname(__file__), 'static') - root = self.pool.get('ir.config_parameter').get_param(cr, uid, 'runbot.root', default) - return root + return self.pool.get('ir.config_parameter').get_param(cr, uid, 'runbot.root', default) def git(self, cr, uid, ids, cmd, context=None): for repo in self.browse(cr, uid, ids, context=context): From a70e64e976e46df3f6320ff7a0be2c0b5c47f5fa Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Thu, 26 Jun 2014 14:36:03 +0200 Subject: [PATCH 02/14] [IMP] various code cleanups in runbot.py --- runbot/runbot.py | 85 ++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 3f301f33..24d1c3af 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -140,12 +140,12 @@ class runbot_repo(osv.osv): return result def _get_base(self, cr, uid, ids, field_name, arg, context=None): - r = {} + result = {} for repo in self.browse(cr, uid, ids, context=context): name = re.sub('.+@', '', repo.name) name = name.replace(':','/') - r[repo.id] = name - return r + result[repo.id] = name + return result _columns = { 'name': fields.char('Repository', required=True), @@ -176,6 +176,7 @@ class runbot_repo(osv.osv): return self.pool.get('ir.config_parameter').get_param(cr, uid, 'runbot.root', default) def git(self, cr, uid, ids, cmd, context=None): + """Execute git command cmd""" for repo in self.browse(cr, uid, ids, context=context): cmd = ['git', '--git-dir=%s' % repo.path] + cmd _logger.info("git: %s", ' '.join(cmd)) @@ -190,22 +191,23 @@ class runbot_repo(osv.osv): p2.communicate()[0] def github(self, cr, uid, ids, url, payload=None, delete=False, context=None): + """Return a http request to be sent to github""" for repo in self.browse(cr, uid, ids, context=context): - mo = re.search('([^/]+)/([^/]+)/([^/]+)', repo.base) - if mo: - url = url.replace(':owner', mo.group(2)) - url = url.replace(':repo', mo.group(3)) - url = 'https://api.%s%s' % (mo.group(1),url) - s = requests.Session() - s.auth = (repo.token,'x-oauth-basic') - s.headers.update({'Accept': 'application/vnd.github.she-hulk-preview+json'}) + match_object = re.search('([^/]+)/([^/]+)/([^/]+)', repo.base) + if match_object: + url = url.replace(':owner', match_object.group(2)) + url = url.replace(':repo', match_object.group(3)) + url = 'https://api.%s%s' % (match_object.group(1),url) + session = requests.Session() + session.auth = (repo.token,'x-oauth-basic') + session.headers.update({'Accept': 'application/vnd.github.she-hulk-preview+json'}) if payload: - r = s.post(url, data=simplejson.dumps(payload)) + response = session.post(url, data=simplejson.dumps(payload)) elif delete: - r = s.delete(url) + response = session.delete(url) else: - r = s.get(url) - return r.json() + response = session.get(url) + return response.json() def update(self, cr, uid, ids, context=None): for repo in self.browse(cr, uid, ids, context=context): @@ -222,18 +224,17 @@ class runbot_repo(osv.osv): repo.git(['fetch', '-p', 'origin', '+refs/pull/*/head:refs/pull/*']) fields = ['refname','objectname','authordate:iso8601','authorname','subject'] - fmt = "%00".join(["%("+i+")" for i in fields]) - out = repo.git(['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/heads', 'refs/pull']) - out = out.strip() - refs = [] - for l in out.split('\n'): - ref = [] - for i in l.split('\x00'): - try: - ref.append(i.decode('utf-8')) - except UnicodeDecodeError: - ref.append('') - refs.append(ref) + fmt = "%00".join(["%("+field+")" for field in fields]) + git_refs = repo.git(['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/heads', 'refs/pull']) + git_refs = git_refs.strip() + + def decode_utf(string): + try: + return field.decode('utf-8') + except UnicodeDecodeError: + return '' + refs = [[decode_utf(field) for field in line.split('\x00')] for line in git_refs.split('\n')] + for name, sha, date, author, subject in refs: # create or get branch branch_ids = self.pool['runbot.branch'].search(cr, uid, [('repo_id', '=', repo.id), ('name', '=', name)]) @@ -250,39 +251,39 @@ class runbot_repo(osv.osv): build_ids = self.pool['runbot.build'].search(cr, uid, [('branch_id', '=', branch.id), ('name', '=', sha)]) if not build_ids: _logger.debug('repo %s branch %s new build found revno %s', branch.repo_id.name, branch.name, sha) - v = { + build_info = { 'branch_id': branch.id, 'name': sha, 'author': author, 'subject': subject, } - self.pool['runbot.build'].create(cr, uid, v) + self.pool['runbot.build'].create(cr, uid, build_info) def scheduler(self, cr, uid, ids=None, context=None): for repo in self.browse(cr, uid, ids, context=context): - bo = self.pool['runbot.build'] + Build = self.pool['runbot.build'] dom = [('repo_id', '=', repo.id)] # schedule jobs - build_ids = bo.search(cr, uid, dom + [('state', 'in', ['testing', 'running'])]) - bo.schedule(cr, uid, build_ids) + build_ids = Build.search(cr, uid, dom + [('state', 'in', ['testing', 'running'])]) + Build.schedule(cr, uid, build_ids) # launch new tests - testing = bo.search(cr, uid, dom + [('state', '=', 'testing')], count=True) + testing = Build.search(cr, uid, dom + [('state', '=', 'testing')], count=True) while testing < repo.testing: # select the next build to process - pending_ids = bo.search(cr, uid, dom + [('state', '=', 'pending')]) + pending_ids = Build.search(cr, uid, dom + [('state', '=', 'pending')]) if pending_ids: - pending = bo.browse(cr, uid, pending_ids[0]) + pending = Build.browse(cr, uid, pending_ids[0]) else: break # gather information about currently running builds - running_ids = bo.search(cr, uid, dom + [('state', '=', 'running')]) + running_ids = Build.search(cr, uid, dom + [('state', '=', 'running')]) running_len = len(running_ids) running_max = 0 if running_ids: - running_max = bo.browse(cr, uid, running_ids[0]).sequence + running_max = Build.browse(cr, uid, running_ids[0]).sequence # determine if pending one should be launched if running_len < repo.running or pending.sequence >= running_max: @@ -291,14 +292,14 @@ class runbot_repo(osv.osv): break # compute the number of testing job again - testing = bo.search(cr, uid, dom + [('state', '=', 'testing')], count=True) + testing = Build.search(cr, uid, dom + [('state', '=', 'testing')], count=True) # terminate and reap doomed build - build_ids = bo.search(cr, uid, dom + [('state', '=', 'running')]) + build_ids = Build.search(cr, uid, dom + [('state', '=', 'running')]) # sort builds: the last build of each sticky branch then the rest sticky = {} non_sticky = [] - for build in bo.browse(cr, uid, build_ids): + for build in Build.browse(cr, uid, build_ids): if build.branch_id.sticky and build.branch_id.id not in sticky: sticky[build.branch_id.id] = build.id else: @@ -306,8 +307,8 @@ class runbot_repo(osv.osv): build_ids = sticky.values() build_ids += non_sticky # terminate extra running builds - bo.terminate(cr, uid, build_ids[repo.running:]) - bo.reap(cr, uid, build_ids) + Build.terminate(cr, uid, build_ids[repo.running:]) + Build.reap(cr, uid, build_ids) def nginx(self, cr, uid, context=None): v = {} From 6109efb122a0c8b588b3ff2607e147c275ee474e Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Thu, 26 Jun 2014 15:06:59 +0200 Subject: [PATCH 03/14] [IMP] various code cleanups --- runbot/runbot.py | 61 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 24d1c3af..2e6e1c7a 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -262,24 +262,24 @@ class runbot_repo(osv.osv): def scheduler(self, cr, uid, ids=None, context=None): for repo in self.browse(cr, uid, ids, context=context): Build = self.pool['runbot.build'] - dom = [('repo_id', '=', repo.id)] + domain = [('repo_id', '=', repo.id)] - # schedule jobs - build_ids = Build.search(cr, uid, dom + [('state', 'in', ['testing', 'running'])]) + # schedule jobs (transitions testing -> running, kill jobs, ...) + build_ids = Build.search(cr, uid, domain + [('state', 'in', ['testing', 'running'])]) Build.schedule(cr, uid, build_ids) # launch new tests - testing = Build.search(cr, uid, dom + [('state', '=', 'testing')], count=True) + testing = Build.search_count(cr, uid, domain + [('state', '=', 'testing')]) while testing < repo.testing: # select the next build to process - pending_ids = Build.search(cr, uid, dom + [('state', '=', 'pending')]) + pending_ids = Build.search(cr, uid, domain + [('state', '=', 'pending')]) if pending_ids: pending = Build.browse(cr, uid, pending_ids[0]) else: break # gather information about currently running builds - running_ids = Build.search(cr, uid, dom + [('state', '=', 'running')]) + running_ids = Build.search(cr, uid, domain + [('state', '=', 'running')]) running_len = len(running_ids) running_max = 0 if running_ids: @@ -292,10 +292,10 @@ class runbot_repo(osv.osv): break # compute the number of testing job again - testing = Build.search(cr, uid, dom + [('state', '=', 'testing')], count=True) + testing = Build.search_count(cr, uid, domain + [('state', '=', 'testing')]) # terminate and reap doomed build - build_ids = Build.search(cr, uid, dom + [('state', '=', 'running')]) + build_ids = Build.search(cr, uid, domain + [('state', '=', 'running')]) # sort builds: the last build of each sticky branch then the rest sticky = {} non_sticky = [] @@ -311,16 +311,16 @@ class runbot_repo(osv.osv): Build.reap(cr, uid, build_ids) def nginx(self, cr, uid, context=None): - v = {} - v['port'] = openerp.tools.config['xmlrpc_port'] + settings = {} + settings['port'] = openerp.tools.config['xmlrpc_port'] nginx_dir = os.path.join(self.root(cr, uid), 'nginx') - v['nginx_dir'] = nginx_dir + settings['nginx_dir'] = nginx_dir ids = self.search(cr, uid, [('nginx','=',True)], order='id') if ids: build_ids = self.pool['runbot.build'].search(cr, uid, [('repo_id','in',ids), ('state','=','running')]) - v['builds'] = self.pool['runbot.build'].browse(cr, uid, build_ids) + settings['builds'] = self.pool['runbot.build'].browse(cr, uid, build_ids) - nginx_config = self.pool['ir.ui.view'].render(cr, uid, "runbot.nginx_config", v) + nginx_config = self.pool['ir.ui.view'].render(cr, uid, "runbot.nginx_config", settings) mkdirs([nginx_dir]) open(os.path.join(nginx_dir, 'nginx.conf'),'w').write(nginx_config) try: @@ -333,10 +333,10 @@ class runbot_repo(osv.osv): def killall(self, cr, uid, ids=None, context=None): # kill switch - bo = self.pool['runbot.build'] - build_ids = bo.search(cr, uid, [('state', 'not in', ['done', 'pending'])]) - bo.terminate(cr, uid, build_ids) - bo.reap(cr, uid, build_ids) + Build = self.pool['runbot.build'] + build_ids = Build.search(cr, uid, [('state', 'not in', ['done', 'pending'])]) + Build.terminate(cr, uid, build_ids) + Build.reap(cr, uid, build_ids) def cron(self, cr, uid, ids=None, context=None): ids = self.search(cr, uid, [('auto', '=', True)]) @@ -385,6 +385,7 @@ class runbot_build(osv.osv): return r def _get_time(self, cr, uid, ids, field_name, arg, context=None): + """Return the time taken by the tests""" r = {} for build in self.browse(cr, uid, ids, context=context): r[build.id] = 0 @@ -395,6 +396,7 @@ class runbot_build(osv.osv): return r def _get_age(self, cr, uid, ids, field_name, arg, context=None): + """Return the time between job start and now""" r = {} for build in self.browse(cr, uid, ids, context=context): r[build.id] = 0 @@ -439,8 +441,8 @@ class runbot_build(osv.osv): } def create(self, cr, uid, values, context=None): - bid = super(runbot_build, self).create(cr, uid, values, context=context) - self.write(cr, uid, [bid], {'sequence' : bid}, context=context) + build_id = super(runbot_build, self).create(cr, uid, values, context=context) + self.write(cr, uid, [build_id], {'sequence' : build_id}, context=context) def reset(self, cr, uid, ids, context=None): self.write(cr, uid, ids, { 'state' : 'pending' }, context=context) @@ -452,7 +454,7 @@ class runbot_build(osv.osv): _logger.debug(*l) def list_jobs(self): - jobs = [i for i in dir(self) if i.startswith('job')] + jobs = [job for job in dir(self) if job.startswith('job')] jobs.sort() return jobs @@ -522,6 +524,7 @@ class runbot_build(osv.osv): openerp.service.db._create_empty_database(dbname) def cmd(self, cr, uid, ids, context=None): + """Return a list describing the command to start the build""" for build in self.browse(cr, uid, ids, context=context): # Server server_path = build.path("openerp-server") @@ -534,12 +537,12 @@ class runbot_build(osv.osv): # modules if build.repo_id.modules: - mods = build.repo_id.modules + modules = build.repo_id.modules else: l = glob.glob(build.path('openerp/addons/*/__init__.py')) - mods = set([os.path.basename(os.path.dirname(i)) for i in l]) - mods = mods - set(['auth_ldap', 'document_ftp', 'hw_escpos', 'hw_proxy', 'hw_scanner', 'base_gengo', 'website_gengo']) - mods = ",".join(list(mods)) + modules = set([os.path.basename(os.path.dirname(i)) for i in l]) + modules = modules - set(['auth_ldap', 'document_ftp', 'hw_escpos', 'hw_proxy', 'hw_scanner', 'base_gengo', 'website_gengo']) + modules = ",".join(list(modules)) # commandline cmd = [ @@ -562,7 +565,7 @@ class runbot_build(osv.osv): #self.run_log(cmd, logfile=self.test_all_path) #run(["coverage","html","-d",self.coverage_base_path,"--ignore-errors","--include=*.py"],env={'COVERAGE_FILE': self.coverage_file_path}) - return cmd, mods + return cmd, modules def spawn(self, cmd, lock_path, log_path, cpu_limit=None, shell=False, showstderr=False): def preexec_fn(): @@ -586,6 +589,7 @@ class runbot_build(osv.osv): return p.pid def github_status(self, cr, uid, ids, context=None): + """Notify github of failed/successful builds""" for build in self.browse(cr, uid, ids, context=context): desc = "runbot build %s" % (build.dest,) if build.state == 'testing': @@ -689,6 +693,7 @@ class runbot_build(osv.osv): return self.spawn(cmd, lock_path, log_path, cpu_limit=None, showstderr=True) def force(self, cr, uid, ids, context=None): + """Force a rebuild""" for build in self.browse(cr, uid, ids, context=context): max_id = self.search(cr, uid, [('repo_id','=',build.repo_id.id)], order='id desc', limit=1)[0] # Force it now @@ -872,9 +877,9 @@ class RunbotController(http.Controller): v['branches'] = branches # stats - v['testing'] = build_obj.search(cr, uid, [('repo_id','=',repo.id), ('state','=','testing')], count=True) - v['running'] = build_obj.search(cr, uid, [('repo_id','=',repo.id), ('state','=','running')], count=True) - v['pending'] = build_obj.search(cr, uid, [('repo_id','=',repo.id), ('state','=','pending')], count=True) + v['testing'] = build_obj.search_count(cr, uid, [('repo_id','=',repo.id), ('state','=','testing')]) + v['running'] = build_obj.search_count(cr, uid, [('repo_id','=',repo.id), ('state','=','running')]) + v['pending'] = build_obj.search_count(cr, uid, [('repo_id','=',repo.id), ('state','=','pending')]) v.update({ 'search': search, From a5a50a569467ba9206663c5b928dd852ccf6d299 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Fri, 4 Jul 2014 21:16:15 +0200 Subject: [PATCH 04/14] [FIX] correct a typo in variable name --- runbot/runbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 729f72b0..cf8c8b6d 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -264,7 +264,7 @@ class runbot_repo(osv.osv): 'author': author, 'subject': subject, } - Build.create(cr, uid, v) + Build.create(cr, uid, build_info) # skip old builds (if their sequence number is too low, they will not ever be built) skippable_domain = [('repo_id', '=', repo.id), ('state', '=', 'pending')] From 8def4a52f84137732f55ec4de440f57a88a7b983 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Sun, 6 Jul 2014 16:31:20 +0200 Subject: [PATCH 05/14] [IMP] change the kill timeout back to 30 min --- runbot/runbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index cf8c8b6d..f544577b 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -735,7 +735,7 @@ class runbot_build(osv.osv): lock_path = build.path('logs', '%s.lock' % build.job) if locked(lock_path): # kill if overpassed - if build.job != jobs[-1] and build.job_time > 2400: + if build.job != jobs[-1] and build.job_time > 1800: build.logger('%s time exceded (%ss)', build.job, build.job_time) build.kill() continue From a823519cdf49130889bd4ef9825f6004e8042717 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Mon, 14 Jul 2014 12:09:00 +0200 Subject: [PATCH 06/14] [FIX] preparation for v8 (fix various issues) --- runbot/runbot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index eb31a14f..40f8666b 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -458,7 +458,7 @@ class runbot_build(osv.osv): _logger.debug(*l) def list_jobs(self): - jobs = [job for job in dir(self) if job.startswith('job')] + jobs = [job for job in dir(self) if re.match(r'^job_\d+', job)] jobs.sort() return jobs @@ -882,10 +882,11 @@ class RunbotController(http.Controller): branch_ids.append(br[0]) branches = branch_obj.browse(cr, uid, branch_ids, context=context) + v['branches'] = [] for branch in branches: build_ids = build_obj.search(cr, uid, [('branch_id','=',branch.id)], limit=4) branch.builds = build_obj.browse(cr, uid, build_ids, context=context) - v['branches'] = branches + v['branches'].append(branch) # stats v['testing'] = build_obj.search_count(cr, uid, [('repo_id','=',repo.id), ('state','=','testing')]) From 48bd01e6f276f68a60b0d7104be2bca453ce169d Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Mon, 14 Jul 2014 17:06:42 +0200 Subject: [PATCH 07/14] [IMP] add a res_config to the runbot it can save/get two keys: default_workers and default_running_max --- runbot/__init__.py | 1 + runbot/__openerp__.py | 1 + runbot/res_config.py | 53 ++++++++++++++++++++++++++++++++++++++ runbot/res_config_view.xml | 43 +++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 runbot/res_config.py create mode 100644 runbot/res_config_view.xml diff --git a/runbot/__init__.py b/runbot/__init__.py index ef0a414f..5b93fcf8 100644 --- a/runbot/__init__.py +++ b/runbot/__init__.py @@ -1 +1,2 @@ import runbot +import res_config diff --git a/runbot/__openerp__.py b/runbot/__openerp__.py index f4fbd0c0..c538112d 100644 --- a/runbot/__openerp__.py +++ b/runbot/__openerp__.py @@ -11,6 +11,7 @@ }, 'data': [ 'runbot.xml', + 'res_config_view.xml', 'security/ir.model.access.csv', ], 'installable': True, diff --git a/runbot/res_config.py b/runbot/res_config.py new file mode 100644 index 00000000..80e7b3ba --- /dev/null +++ b/runbot/res_config.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Business Applications +# Copyright (C) 2004-2012 OpenERP S.A. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import fields, osv + +class runbot_config_settings(osv.osv_memory): + _name = 'runbot.config.settings' + _inherit = 'res.config.settings' + _columns = { + 'default_workers': fields.integer('Total Number of Workers'), + 'default_running_max': fields.integer('Maximum Number of Running Builds'), + } + + def get_default_parameters(self, cr, uid, fields, context=None): + icp = self.pool['ir.config_parameter'] + workers = icp.get_param(cr, uid, 'runbot.workers', default=6) + running_max = icp.get_param(cr, uid, 'runbot.running_max', default=75) + return { + 'default_workers': int(workers), + 'default_running_max': int(running_max) + } + + def set_default_parameters(self, cr, uid, ids, context=None): + config = self.browse(cr, uid, ids[0], context) + icp = self.pool['ir.config_parameter'] + icp.set_param(cr, uid, 'runbot.workers', config.default_workers) + icp.set_param(cr, uid, 'runbot.running_max', config.default_running_max) + + _defaults = { + 'default_workers': 6, + 'default_running_max': 75, + } + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/runbot/res_config_view.xml b/runbot/res_config_view.xml new file mode 100644 index 00000000..ba4c0a01 --- /dev/null +++ b/runbot/res_config_view.xml @@ -0,0 +1,43 @@ + + + + + + Configure Runbot + runbot.config.settings + +
+
+
+ + + + +
+ + + Configure Runbot + ir.actions.act_window + runbot.config.settings + form + inline + + + + + From 46ce71ee4ababfb32d763cae3cea1e51a1e64882 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Tue, 15 Jul 2014 10:19:04 +0200 Subject: [PATCH 08/14] [IMP] xml improvement --- runbot/runbot.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runbot/runbot.xml b/runbot/runbot.xml index d294094c..2fe922b1 100644 --- a/runbot/runbot.xml +++ b/runbot/runbot.xml @@ -266,7 +266,7 @@ } -
+
@@ -336,6 +338,7 @@ Branch + : testing, running, pending. From 7261f5f4794f02f031448e1b158d8ed1b29e90af Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Tue, 15 Jul 2014 14:38:36 +0200 Subject: [PATCH 12/14] [IMP] throw exception when no toke is available --- runbot/runbot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runbot/runbot.py b/runbot/runbot.py index 10bc6188..b565d68e 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -193,6 +193,8 @@ class runbot_repo(osv.osv): def github(self, cr, uid, ids, url, payload=None, delete=False, context=None): """Return a http request to be sent to github""" for repo in self.browse(cr, uid, ids, context=context): + if not repo.token: + raise Exception('Repository does not have a token to authenticate') match_object = re.search('([^/]+)/([^/]+)/([^/]+)', repo.base) if match_object: url = url.replace(':owner', match_object.group(2)) From 8e01b6b2c663a375d90104f94c67d5c628088c7d Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Tue, 15 Jul 2014 16:28:20 +0200 Subject: [PATCH 13/14] [IMP] put the default timeout in runbot options instead of hardcoding it --- runbot/res_config.py | 6 +++++- runbot/res_config_view.xml | 4 ++++ runbot/runbot.py | 5 ++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/runbot/res_config.py b/runbot/res_config.py index 80e7b3ba..04acd5c7 100644 --- a/runbot/res_config.py +++ b/runbot/res_config.py @@ -26,13 +26,15 @@ class runbot_config_settings(osv.osv_memory): _inherit = 'res.config.settings' _columns = { 'default_workers': fields.integer('Total Number of Workers'), - 'default_running_max': fields.integer('Maximum Number of Running Builds'), + 'default_running_max': fields.integer('Maximum Number of Running Builds'), + 'default_timeout': fields.integer('Default timeout (in seconds)'), } def get_default_parameters(self, cr, uid, fields, context=None): icp = self.pool['ir.config_parameter'] workers = icp.get_param(cr, uid, 'runbot.workers', default=6) running_max = icp.get_param(cr, uid, 'runbot.running_max', default=75) + timeout = icp.get_param(cr, uid, 'runbot.timeout', default=1800) return { 'default_workers': int(workers), 'default_running_max': int(running_max) @@ -43,10 +45,12 @@ class runbot_config_settings(osv.osv_memory): icp = self.pool['ir.config_parameter'] icp.set_param(cr, uid, 'runbot.workers', config.default_workers) icp.set_param(cr, uid, 'runbot.running_max', config.default_running_max) + icp.set_param(cr, uid, 'runbot.timeout', config.default_timeout) _defaults = { 'default_workers': 6, 'default_running_max': 75, + 'default_timeout': 1800, } diff --git a/runbot/res_config_view.xml b/runbot/res_config_view.xml index ba4c0a01..99bef250 100644 --- a/runbot/res_config_view.xml +++ b/runbot/res_config_view.xml @@ -24,6 +24,10 @@
+
+ +
diff --git a/runbot/runbot.py b/runbot/runbot.py index b565d68e..913acbe3 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -731,6 +731,9 @@ class runbot_build(osv.osv): def schedule(self, cr, uid, ids, context=None): jobs = self.list_jobs() + icp = self.pool['ir.config_parameter'] + timeout = int(icp.get_param(cr, uid, 'runbot.timeout', default=1800)) + for build in self.browse(cr, uid, ids, context=context): if build.state == 'pending': # allocate port and schedule first job @@ -749,7 +752,7 @@ class runbot_build(osv.osv): lock_path = build.path('logs', '%s.lock' % build.job) if locked(lock_path): # kill if overpassed - if build.job != jobs[-1] and build.job_time > 1800: + if build.job != jobs[-1] and build.job_time > timeout: build.logger('%s time exceded (%ss)', build.job, build.job_time) build.kill() continue From 2a48508bdf6fdc36c367f43d7494ca54e3e0a667 Mon Sep 17 00:00:00 2001 From: Gery Debongnie Date: Tue, 15 Jul 2014 16:31:15 +0200 Subject: [PATCH 14/14] [IMP] better code for list_jobs inspired by https://github.com/odoo/odoo-extra/pull/19 --- runbot/runbot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 913acbe3..350e3bce 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -59,6 +59,7 @@ def grep(filename, string): _re_error = r'^(?:\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ (?:ERROR|CRITICAL) )|(?:Traceback \(most recent call last\):)$' _re_warning = r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ WARNING ' +_re_job = re.compile('job_\d') def rfind(filename, pattern): """Determine in something in filename matches the pattern""" @@ -465,9 +466,7 @@ class runbot_build(osv.osv): _logger.debug(*l) def list_jobs(self): - jobs = [job for job in dir(self) if re.match(r'^job_\d+', job)] - jobs.sort() - return jobs + return sorted(job for job in dir(self) if _re_job.match(job)) def find_port(self, cr, uid): # currently used port