From 3308829e80bdcb2d890f98a921f4a52948475347 Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Mon, 23 Sep 2019 23:08:50 +0200 Subject: [PATCH] [IMP] runbot: uses fnmatch patterns to select modules Actually some Odoo modules are black_listed from a set hardcoded in the runbot code. In some cases, one needs to blacklist custom modules, preferably in a config_step. With this commit, the repo.modules, branch.modules, config_step.install_modules fields are concatained in a comma separated list of fnmatch patterns. The patterns can be prefixed with a dash to exclude the matching module(s). Co-authored by @Xavier-Do --- runbot/__manifest__.py | 2 +- runbot/migrations/4.5/post-migration.py | 20 +++++++++ runbot/models/build.py | 60 ++++++++++--------------- runbot/models/build_config.py | 10 +---- runbot/tests/test_build.py | 30 ++++++++++++- 5 files changed, 76 insertions(+), 46 deletions(-) create mode 100644 runbot/migrations/4.5/post-migration.py diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 8644407a..37f00984 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -6,7 +6,7 @@ 'author': "Odoo SA", 'website': "http://runbot.odoo.com", 'category': 'Website', - 'version': '4.4', + 'version': '4.5', 'depends': ['website', 'base'], 'data': [ 'security/runbot_security.xml', diff --git a/runbot/migrations/4.5/post-migration.py b/runbot/migrations/4.5/post-migration.py new file mode 100644 index 00000000..0240bcba --- /dev/null +++ b/runbot/migrations/4.5/post-migration.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + + +def migrate(cr, version): + + repo_modules = '-auth_ldap,-document_ftp,-base_gengo,-website_gengo,-website_instantclick,-pad,-pad_project,-note_pad,-pos_cache,-pos_blackbox_be,-hw_*,-theme_*,-l10n_*,' + cr.execute("UPDATE runbot_repo SET modules = CONCAT(%s, modules) WHERE modules_auto = 'all' or modules_auto = 'repo';", (repo_modules,)) + + # ceux qui n'ont pas d'étoile on prefix par '-*,' + cr.execute("SELECT id,install_modules FROM runbot_build_config_step") + for step_id, install_modules in cr.fetchall(): + install_modules_list = [mod.strip() for mod in (install_modules or '').split(',') if mod.strip()] + if '*' in install_modules_list: + install_modules_list.remove('*') + install_modules = ', '.join(install_modules_list) + elif install_modules_list: + install_modules = '-*,%s' % install_modules + else: + install_modules = '-*' + cr.execute("UPDATE runbot_build_config_step SET install_modules = %s WHERE id=%s", (install_modules, step_id)) diff --git a/runbot/models/build.py b/runbot/models/build.py index 32b83d99..8594cb2a 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import fnmatch import glob import logging import os @@ -690,26 +691,6 @@ class runbot_build(models.Model): return commit._source_path('odoo', *path) return commit._source_path('openerp', *path) - def _filter_modules(self, modules, available_modules, explicit_modules): - # TODO add blacklist_modules and blacklist prefixes as data on repo - blacklist_modules = set(['auth_ldap', 'document_ftp', 'base_gengo', - 'website_gengo', 'website_instantclick', - 'pad', 'pad_project', 'note_pad', - 'pos_cache', 'pos_blackbox_be']) - - def mod_filter(module): - if module not in available_modules: - return False - if module in explicit_modules: - return True - if module.startswith(('hw_', 'theme_', 'l10n_')): - return False - if module in blacklist_modules: - return False - return True - - return uniq_list([module for module in modules if mod_filter(module)]) - def _get_available_modules(self, commit): for manifest_file_name in commit.repo.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' for addons_path in commit.repo.addons_paths.split(','): # '' 'addons' 'odoo/addons' @@ -745,11 +726,9 @@ class runbot_build(models.Model): self._kill(result='ko') return exports - def _get_modules_to_test(self, commits=None): - self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build. - # checkout branch - repo_modules = [] + def _get_repo_available_modules(self, commits=None): available_modules = [] + repo_modules = [] for commit in commits or self._get_all_commit(): for (addons_path, module, manifest_file_name) in self._get_available_modules(commit): if commit.repo == self.repo_id: @@ -762,25 +741,34 @@ class runbot_build(models.Model): ) else: available_modules.append(module) - explicit_modules = uniq_list([module for module in (self.branch_id.modules or '').split(',') + (self.repo_id.modules or '').split(',') if module]) + return repo_modules, available_modules - if explicit_modules: - _logger.debug("explicit modules_to_test for build %s: %s", self.dest, explicit_modules) + def _get_modules_to_test(self, commits=None, modules_patterns=''): + self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build. - if set(explicit_modules) - set(available_modules): - self._log('checkout', 'Some explicit modules (branch or repo defined) are not in available module list.', level='WARNING') + # checkout branch + repo_modules, available_modules = self._get_repo_available_modules(commits=commits) + + patterns_list = [] + for pats in [self.repo_id.modules, self.branch_id.modules, modules_patterns]: + patterns_list += [p.strip() for p in (pats or '').split(',')] if self.repo_id.modules_auto == 'all': - modules_to_test = available_modules + default_repo_modules = available_modules elif self.repo_id.modules_auto == 'repo': - modules_to_test = explicit_modules + repo_modules - _logger.debug("local modules_to_test for build %s: %s", self.dest, modules_to_test) + default_repo_modules = repo_modules else: - modules_to_test = explicit_modules + default_repo_modules = [] - modules_to_test = self._filter_modules(modules_to_test, available_modules, explicit_modules) - _logger.debug("modules_to_test for build %s: %s", self.dest, modules_to_test) - return modules_to_test + modules_to_install = set(default_repo_modules) + for pat in patterns_list: + if pat.startswith('-'): + pat = pat.strip('- ') + modules_to_install -= {mod for mod in modules_to_install if fnmatch.fnmatch(mod, pat)} + else: + modules_to_install |= {mod for mod in available_modules if fnmatch.fnmatch(mod, pat)} + + return sorted(modules_to_install) def _local_pg_dropdb(self, dbname): with local_pgadmin_cursor() as local_cr: diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 988d46b6..9f3b1218 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -97,7 +97,7 @@ class ConfigStep(models.Model): # install_odoo create_db = fields.Boolean('Create Db', default=True, track_visibility='onchange') # future custom_db_name = fields.Char('Custom Db Name', track_visibility='onchange') # future - install_modules = fields.Char('Modules to install', help="List of module to install, use * for all modules", default='*') + install_modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='*') db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', track_visibility='onchange') cpu_limit = fields.Integer('Cpu limit', default=3600, track_visibility='onchange') coverage = fields.Boolean('Coverage', dafault=False, track_visibility='onchange') @@ -339,13 +339,7 @@ class ConfigStep(models.Model): return docker_run(cmd.build(), log_path, build._path(), build._get_docker_name(), cpu_limit=timeout, ro_volumes=exports) def _modules_to_install(self, build): - modules_to_install = set([mod.strip() for mod in self.install_modules.split(',')]) - if '*' in modules_to_install: - modules_to_install.remove('*') - default_mod = set(build._get_modules_to_test()) - modules_to_install = default_mod | modules_to_install - # todo add without support - return modules_to_install + return set(build._get_modules_to_test(modules_patterns=self.install_modules)) def _post_install_command(self, build, modules_to_install, py_version=None): if self.coverage: diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 80247ef5..c701ebdb 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from unittest.mock import patch -from odoo.tools.config import configmanager from odoo.tests import common @@ -71,6 +70,35 @@ class Test_Build(common.TransactionCase): with self.assertRaises(AssertionError): builds.write({'local_state': 'duplicate'}) + @patch('odoo.addons.runbot.models.build.runbot_build._get_repo_available_modules') + @patch('odoo.addons.runbot.models.build.runbot_build._get_params') + @patch('odoo.addons.runbot.models.build.fqdn') + def test_filter_modules(self, mock_fqdn, mock_get_params, mock_get_repo_mods): + """ test module filtering """ + build = self.Build.create({ + 'branch_id': self.branch.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + 'port': '1234', + }) + + repo_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit'] + available_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2'] + mock_get_repo_mods.return_value = (repo_mods, available_mods) + self.repo.modules_auto = 'repo' + self.repo.modules = '-bad_module,-hw_*,hw_explicit,-l10n_*' + modules_to_test = build._get_modules_to_test(self, modules_patterns='') + self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit'])) + + modules_to_test = build._get_modules_to_test(self, modules_patterns='-*, l10n_be') + self.assertEqual(modules_to_test, sorted(['l10n_be'])) + + modules_to_test = build._get_modules_to_test(self, modules_patterns='l10n_be') + self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit', 'l10n_be'])) + + # star to get all available mods + modules_to_test = build._get_modules_to_test(self, modules_patterns='*, -hw_*, hw_explicit') + self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2'])) + @patch('odoo.addons.runbot.models.build.os.path.isfile') @patch('odoo.addons.runbot.models.build.os.mkdir') @patch('odoo.addons.runbot.models.build.grep')