From 83029e48f5884051939a4cb361a8bcdac36ba9b9 Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Sat, 14 Jun 2014 19:32:08 -0400 Subject: [PATCH 1/8] Add runbot work files to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6af30ee7..6d19369b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ *.py[co] # emacs backup files *~ +# runbot work files +runbot/static/build +runbot/static/repo From 0fb5ef92c465e2c984fc2b1abbbb5bbb8f378e47 Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Fri, 27 Jun 2014 20:31:26 -0400 Subject: [PATCH 2/8] Allow some builds to determine which modules to test OCA addon repos will only want to test modules at root Glob before retrieving dependencies and store in build object Signed-off-by: Sandy Carter --- runbot/runbot.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index f9b5c291..ef85235e 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -173,7 +173,7 @@ class runbot_repo(osv.osv): 'auto': fields.boolean('Auto'), 'duplicate_id': fields.many2one('runbot.repo', 'Repository for finding duplicate builds'), 'fallback_id': fields.many2one('runbot.repo', 'Fallback repo'), - 'modules': fields.char("Modules to Install"), + 'modules': fields.char("Modules to Install", help="Comma-separated list of modules to install and test."), 'token': fields.char("Github token"), } _defaults = { @@ -277,7 +277,8 @@ class runbot_repo(osv.osv): 'name': sha, 'author': author, 'subject': subject, - 'date': dateutil.parser.parse(date[:19]) + 'date': dateutil.parser.parse(date[:19]), + 'modules': branch.repo_id.modules, } Build.create(cr, uid, build_info) @@ -449,6 +450,7 @@ class runbot_build(osv.osv): 'author': fields.char('Author'), 'subject': fields.text('Subject'), 'sequence': fields.integer('Sequence', select=1), + 'modules': fields.char("Modules to Install"), 'result': fields.char('Result'), # ok, ko, warn, skipped, killed 'pid': fields.integer('Pid'), 'state': fields.char('Status'), # pending, testing, running, done, duplicate @@ -544,8 +546,10 @@ class runbot_build(osv.osv): # fallback for addons-only community/projet branches if not os.path.isfile(build.server('__init__.py')): - l = glob.glob(build.path('*/__openerp__.py')) - for i in l: + # Find modules to test and store in build + modules_to_test = glob.glob(build.path('*/__openerp__.py')) + build.write({'modules': ','.join(modules_to_test)}) + for i in modules_to_test: shutil.move(os.path.dirname(i), build.server('addons')) name = build.branch_id.branch_name.split('-',1)[0] if build.repo_id.fallback_id: @@ -582,8 +586,8 @@ class runbot_build(osv.osv): server_path = build.path("bin/openerp-server.py") # modules - if build.repo_id.modules: - modules = build.repo_id.modules + if build.modules: + modules = build.modules else: l = glob.glob(build.server('addons', '*', '__init__.py')) modules = set(os.path.basename(os.path.dirname(i)) for i in l) From 9dbc76d9ab075fccbc6c320ab0876af844f296b9 Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Fri, 27 Jun 2014 20:34:35 -0400 Subject: [PATCH 3/8] Change fallback_id to many2many A Community addon can depend on more than just an odoo server, but other addon repos Bump version Add migration script Signed-off-by: Sandy Carter --- runbot/__openerp__.py | 2 +- runbot/migrations/8.0.1.1/post-migration.py | 33 +++++++++++++++++++ runbot/migrations/8.0.1.1/pre-migration.py | 35 +++++++++++++++++++++ runbot/runbot.py | 17 ++++++---- runbot/runbot.xml | 4 ++- 5 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 runbot/migrations/8.0.1.1/post-migration.py create mode 100644 runbot/migrations/8.0.1.1/pre-migration.py diff --git a/runbot/__openerp__.py b/runbot/__openerp__.py index c538112d..9e6fa1bb 100644 --- a/runbot/__openerp__.py +++ b/runbot/__openerp__.py @@ -2,7 +2,7 @@ 'name': 'Runbot', 'category': 'Website', 'summary': 'Runbot', - 'version': '1.0', + 'version': '1.1', 'description': "Runbot", 'author': 'OpenERP SA', 'depends': ['website'], diff --git a/runbot/migrations/8.0.1.1/post-migration.py b/runbot/migrations/8.0.1.1/post-migration.py new file mode 100644 index 00000000..e294b5d9 --- /dev/null +++ b/runbot/migrations/8.0.1.1/post-migration.py @@ -0,0 +1,33 @@ +# -*- encoding: utf-8 -*- + + +from openerp import SUPERUSER_ID +from openerp.modules.registry import RegistryManager + + +def get_legacy_name(original_name, version): + return 'legacy_%s_%s' % (version.replace('.', '_'), original_name) + + +def m2o_to_x2m(cr, model, table, field, source_field): + cr.execute('SELECT id, %(field)s ' + 'FROM %(table)s ' + 'WHERE %(field)s is not null' % { + 'table': table, + 'field': source_field, + }) + for row in cr.fetchall(): + model.write(cr, SUPERUSER_ID, row[0], {field: [(4, row[1])]}) + + +def migrate(cr, version): + if not version: + return + registry = RegistryManager.get(cr.dbname) + m2o_to_x2m( + cr, + registry['runbot.repo'], + 'runbot_repo', + 'dependency_ids', + get_legacy_name('fallback_id', version), + ) diff --git a/runbot/migrations/8.0.1.1/pre-migration.py b/runbot/migrations/8.0.1.1/pre-migration.py new file mode 100644 index 00000000..10a844c3 --- /dev/null +++ b/runbot/migrations/8.0.1.1/pre-migration.py @@ -0,0 +1,35 @@ +# -*- encoding: utf-8 -*- + +from openerp import release +import logging + +logger = logging.getLogger('upgrade') + + +def get_legacy_name(original_name, version): + return 'legacy_%s_%s' % (version.replace('.', '_'), original_name) + + +def rename_columns(cr, column_spec, version): + for table, renames in column_spec.iteritems(): + for old, new in renames: + if new is None: + new = get_legacy_name(old, version) + logger.info("table %s, column %s: renaming to %s", + table, old, new) + cr.execute('ALTER TABLE "%s" RENAME "%s" TO "%s"' + % (table, old, new,)) + cr.execute('DROP INDEX IF EXISTS "%s_%s_index"' + % (table, old)) + +column_renames = { + 'runbot_repo': [ + ('fallback_id', None) + ] +} + + +def migrate(cr, version): + if not version: + return + rename_columns(cr, column_renames, version) diff --git a/runbot/runbot.py b/runbot/runbot.py index ef85235e..e8245816 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -172,8 +172,12 @@ class runbot_repo(osv.osv): 'nginx': fields.boolean('Nginx'), 'auto': fields.boolean('Auto'), 'duplicate_id': fields.many2one('runbot.repo', 'Repository for finding duplicate builds'), - 'fallback_id': fields.many2one('runbot.repo', 'Fallback repo'), 'modules': fields.char("Modules to Install", help="Comma-separated list of modules to install and test."), + 'dependency_ids': fields.many2many( + 'runbot.repo', 'runbot_repo_dep_rel', + id1='dependant_id', id2='dependency_id', + string='Extra dependencies', + help="Community addon repos which need to be present to run tests."), 'token': fields.char("Github token"), } _defaults = { @@ -544,16 +548,17 @@ class runbot_build(osv.osv): if os.path.isdir(build.path('bin/addons')): shutil.move(build.path('bin'), build.server()) - # fallback for addons-only community/projet branches + # fallback for addons-only community/project branches if not os.path.isfile(build.server('__init__.py')): # Find modules to test and store in build modules_to_test = glob.glob(build.path('*/__openerp__.py')) build.write({'modules': ','.join(modules_to_test)}) - for i in modules_to_test: - shutil.move(os.path.dirname(i), build.server('addons')) name = build.branch_id.branch_name.split('-',1)[0] - if build.repo_id.fallback_id: - build.repo_id.fallback_id.git_export(name, build.path()) + for extra_repo in build.repo_id.dependency_ids: + extra_repo.git_export(name, build.path()) + # Finally move all addons to openerp/addons + for module in glob.glob(build.path('*/__openerp__.py')): + shutil.move(os.path.dirname(module), build.path('openerp/addons')) # move all addons to server addons path for i in glob.glob(build.path('addons/*')): diff --git a/runbot/runbot.xml b/runbot/runbot.xml index 39793784..5d133824 100644 --- a/runbot/runbot.xml +++ b/runbot/runbot.xml @@ -23,8 +23,10 @@ - + + + From 91beb61727ad24f9aa0849385e87849a1c346f7a Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Fri, 27 Jun 2014 20:49:43 -0400 Subject: [PATCH 4/8] Add function to better find proper fallback branch --- runbot/runbot.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index e8245816..ea26823a 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -5,6 +5,7 @@ import fcntl import glob import hashlib import logging +import operator import os import re import resource @@ -519,6 +520,37 @@ class runbot_build(osv.osv): return port + def get_closest_branch_name(self, cr, uid, ids, target_repo_id, context=None): + """Return the name of the closest common branch between both repos + Find common branch names, get merge-base with the branch name and + return the most recent. + Fallback repos will not have always have the same names for branch + names. + Pull request branches should not have any association with PR of other + repos + """ + branch_pool = self.pool['runbot.branch'] + for build in self.browse(cr, uid, ids, context=context): + branch, repo = build.branch_id, build.repo_id + name = branch.branch_name + # Find common branch names between repo and target repo + branch_ids = branch_pool.search(cr, uid, [('repo_id.id', '=', repo.id)]) + target_ids = branch_pool.search(cr, uid, [('repo_id.id', '=', target_repo_id)]) + branch_names = branch_pool.read(cr, uid, branch_ids, ['branch_name', 'name'], context=context) + target_names = branch_pool.read(cr, uid, target_ids, ['branch_name', 'name'], context=context) + possible_repo_branches = set([i['branch_name'] for i in branch_names if i['name'].startswith('refs/heads')]) + possible_target_branches = set([i['branch_name'] for i in target_names if i['name'].startswith('refs/heads')]) + possible_branches = possible_repo_branches.intersection(possible_target_branches) + if name not in possible_branches: + common_refs = {} + for target_branch_name in possible_branches: + commit = repo.git(['merge-base', branch.name, target_branch_name]).strip() + cmd = ['log', '-1', '--format=%cd', '--date=iso', commit] + common_refs[target_branch_name] = repo.git(cmd).strip() + if common_refs: + name = sorted(common_refs.iteritems(), key=operator.itemgetter(1), reverse=True)[0][0] + return name + def path(self, cr, uid, ids, *l, **kw): for build in self.browse(cr, uid, ids, context=None): root = self.pool['runbot.repo'].root(cr, uid) @@ -553,9 +585,9 @@ class runbot_build(osv.osv): # Find modules to test and store in build modules_to_test = glob.glob(build.path('*/__openerp__.py')) build.write({'modules': ','.join(modules_to_test)}) - name = build.branch_id.branch_name.split('-',1)[0] for extra_repo in build.repo_id.dependency_ids: - extra_repo.git_export(name, build.path()) + closest_name = build.get_closest_branch_name(extra_repo.id) + extra_repo.git_export(closest_name, build.path()) # Finally move all addons to openerp/addons for module in glob.glob(build.path('*/__openerp__.py')): shutil.move(os.path.dirname(module), build.path('openerp/addons')) From 5e25a8aeb2a8e5a2b9959fa876cdc99d2e1e3f8f Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Wed, 9 Jul 2014 21:28:26 -0400 Subject: [PATCH 5/8] Use git API to find base branch name for PRs when looking for common branch names in extra deps --- runbot/runbot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runbot/runbot.py b/runbot/runbot.py index ea26823a..b4b58de0 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -533,6 +533,11 @@ class runbot_build(osv.osv): for build in self.browse(cr, uid, ids, context=context): branch, repo = build.branch_id, build.repo_id name = branch.branch_name + # Use github API to find name of branch on which the PR is made + if repo.token and name.startswith('refs/pull/'): + pull_number = name[len('refs/pull/'):] + pr = repo.github('/repos/:owner/:repo/pulls/%s' % pull_number) + name = 'refs/heads/' + pr['base']['ref'] # Find common branch names between repo and target repo branch_ids = branch_pool.search(cr, uid, [('repo_id.id', '=', repo.id)]) target_ids = branch_pool.search(cr, uid, [('repo_id.id', '=', target_repo_id)]) From 071720433c81d936bcd27715e1411ab907ba43ab Mon Sep 17 00:00:00 2001 From: Moises Lopez Date: Sun, 20 Jul 2014 23:30:41 -0500 Subject: [PATCH 6/8] [REF][runbot] Fix find modules to test and add support to use one previously configured in the repository --- runbot/runbot.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index b4b58de0..fb25b08b 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -587,9 +587,15 @@ class runbot_build(osv.osv): # fallback for addons-only community/project branches if not os.path.isfile(build.server('__init__.py')): - # Find modules to test and store in build - modules_to_test = glob.glob(build.path('*/__openerp__.py')) - build.write({'modules': ','.join(modules_to_test)}) + # Use modules to test previously configured in the repository + modules_to_test = build.repo_id.modules + if not modules_to_test: + # Find modules to test from the folder branch + modules_to_test = ','.join( + os.path.basename(os.path.dirname(a)) + for a in glob.glob(build.path('*/__openerp__.py')) + ) + build.write({'modules': modules_to_test}) for extra_repo in build.repo_id.dependency_ids: closest_name = build.get_closest_branch_name(extra_repo.id) extra_repo.git_export(closest_name, build.path()) From be765dd6ef4a6c47254081a545756bb081ed9267 Mon Sep 17 00:00:00 2001 From: Moises Lopez Date: Thu, 24 Jul 2014 13:39:15 -0500 Subject: [PATCH 7/8] [REF][runbot] Add validation of module duplicate --- runbot/runbot.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index fb25b08b..61d321a5 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -586,6 +586,7 @@ class runbot_build(osv.osv): shutil.move(build.path('bin'), build.server()) # fallback for addons-only community/project branches + additional_modules = [] if not os.path.isfile(build.server('__init__.py')): # Use modules to test previously configured in the repository modules_to_test = build.repo_id.modules @@ -599,13 +600,22 @@ class runbot_build(osv.osv): for extra_repo in build.repo_id.dependency_ids: closest_name = build.get_closest_branch_name(extra_repo.id) extra_repo.git_export(closest_name, build.path()) - # Finally move all addons to openerp/addons - for module in glob.glob(build.path('*/__openerp__.py')): - shutil.move(os.path.dirname(module), build.path('openerp/addons')) + # Finally mark all addons to move to openerp/addons + additional_modules += [ + os.path.dirname(module) + for module in glob.glob(build.path('*/__openerp__.py')) + ] # move all addons to server addons path - for i in glob.glob(build.path('addons/*')): - shutil.move(i, build.server('addons')) + for module in set(glob.glob(build.path('addons/*')) + additional_modules): + basename = os.path.basename(module) + if not os.path.exists(build.server('addons', basename)): + shutil.move(module, build.server('addons')) + else: + build._log( + 'Building environment', + 'You have duplicate modules in your branches "%s"' % basename + ) def pg_dropdb(self, cr, uid, dbname): pid_col = 'pid' if cr._cnx.server_version >= 90200 else 'procpid' From 93f7a7d6c397266b8f990c0de565f04a2a0fb7a3 Mon Sep 17 00:00:00 2001 From: Sandy Carter Date: Thu, 24 Jul 2014 21:39:07 -0400 Subject: [PATCH 8/8] Add hint_repos set to help find the common branch in cases whith many dependent repos --- runbot/runbot.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/runbot/runbot.py b/runbot/runbot.py index 61d321a5..2664ec27 100644 --- a/runbot/runbot.py +++ b/runbot/runbot.py @@ -520,7 +520,7 @@ class runbot_build(osv.osv): return port - def get_closest_branch_name(self, cr, uid, ids, target_repo_id, context=None): + def get_closest_branch_name(self, cr, uid, ids, target_repo_id, hint_branches, context=None): """Return the name of the closest common branch between both repos Find common branch names, get merge-base with the branch name and return the most recent. @@ -547,6 +547,9 @@ class runbot_build(osv.osv): possible_target_branches = set([i['branch_name'] for i in target_names if i['name'].startswith('refs/heads')]) possible_branches = possible_repo_branches.intersection(possible_target_branches) if name not in possible_branches: + hinted_branches = possible_branches.intersection(hint_branches) + if hinted_branches: + possible_branches = hinted_branches common_refs = {} for target_branch_name in possible_branches: commit = repo.git(['merge-base', branch.name, target_branch_name]).strip() @@ -597,8 +600,10 @@ class runbot_build(osv.osv): for a in glob.glob(build.path('*/__openerp__.py')) ) build.write({'modules': modules_to_test}) + hint_branches = set() for extra_repo in build.repo_id.dependency_ids: - closest_name = build.get_closest_branch_name(extra_repo.id) + closest_name = build.get_closest_branch_name(extra_repo.id, hint_branches) + hint_branches.add(closest_name) extra_repo.git_export(closest_name, build.path()) # Finally mark all addons to move to openerp/addons additional_modules += [