diff --git a/runbot/models/build.py b/runbot/models/build.py index 5f232332..f1f09ce4 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import datetime -import fnmatch import getpass import hashlib import logging @@ -926,29 +925,11 @@ class BuildResult(models.Model): def _get_modules_to_test(self, modules_patterns=''): self.ensure_one() - - def _filter_patterns(patterns, default, all): - default = set(default) - patterns_list = (patterns or '').split(',') - patterns_list = [p.strip() for p in patterns_list] - for pat in patterns_list: - if pat.startswith('-'): - pat = pat.strip('- ') - default -= {mod for mod in default if fnmatch.fnmatch(mod, pat)} - elif pat: - default |= {mod for mod in all if fnmatch.fnmatch(mod, pat)} - return default - - available_modules = [] - modules_to_install = set() - for repo, module_list in self._get_available_modules().items(): - available_modules += module_list - modules_to_install |= _filter_patterns(repo.modules, module_list, module_list) - - modules_to_install = _filter_patterns(self.params_id.modules, modules_to_install, available_modules) - modules_to_install = _filter_patterns(modules_patterns, modules_to_install, available_modules) - - return sorted(modules_to_install) + trigger = self.params_id.trigger_id + modules = self._get_available_modules() + params_patterns = (self.params_id.modules or '').split(',') + modules_patterns = (modules_patterns or '').split(',') + return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns) def _local_pg_dropdb(self, dbname): msg = '' diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 881f2a36..6281b86c 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -75,6 +75,31 @@ class Commit(models.Model): module = os.path.basename(os.path.dirname(manifest_path)) yield (addons_path, module, manifest_file_name) + def _list_files(self, patterns): + #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py' + # note that glob is needed to avoid the star matching ** + self.ensure_one() + return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n') + + def _list_available_modules(self): + # beta version, may replace _get_available_modules latter + addons_paths = (self.repo_id.addons_paths or '').split(',') + patterns = [] + for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' + for addon_path in addons_paths: + addon_path = addon_path or '.' + patterns.append(f':(glob){addon_path}/*/{manifest_file_name}') + for file_path in self._list_files(patterns): + if file_path: + elems = file_path.rsplit('/', 2) + if len(elems) == 3: + addons_path, module, manifest_file_name = elems + else: + addons_path = '' + module, manifest_file_name = elems + yield (addons_path, module, manifest_file_name) + + def _export(self, build): """Export a git repo into a sources""" # TODO add automated tests diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 5498f940..20ed8b0a 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -1,24 +1,32 @@ # -*- coding: utf-8 -*- import datetime +import fnmatch import json import logging import re import subprocess import time - import requests import markupsafe from pathlib import Path from odoo import models, fields, api -from odoo.tools import file_open +from odoo.tools import file_open, mail from ..common import os, RunbotException, make_github_session, sanitize from odoo.exceptions import UserError from odoo.tools.safe_eval import safe_eval _logger = logging.getLogger(__name__) +class ModuleFilter(models.Model): + _name = 'runbot.module.filter' + _description = 'Module filter' + + trigger_id = fields.Many2one('runbot.trigger', string="", required=True) + repo_id = fields.Many2one('runbot.repo', string="Repo", required=True) + modules = fields.Char(string="Module filter", required=True) + description = fields.Char(string="Description") class Trigger(models.Model): @@ -38,6 +46,7 @@ class Trigger(models.Model): project_id = fields.Many2one('runbot.project', string="Project id", required=True) repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]") dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies") + module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger') config_id = fields.Many2one('runbot.build.config', string="Config", required=True) batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle") @@ -91,6 +100,65 @@ class Trigger(models.Model): return safe_eval(self.version_domain) return [] + def _filter_modules_to_test(self, modules, module_patterns=None): + repo_module_patterns = {} + for module_filter in self.module_filters: + repo_module_patterns.setdefault(module_filter.repo_id, []) + repo_module_patterns[module_filter.repo_id] += module_filter.modules.split(',') + module_patterns = module_patterns or [] + + def _filter_patterns(patterns_list, default, all): + current = set(default) + for pat in patterns_list: + pat = pat.strip() + if not pat: + continue + if pat.startswith('-'): + pat = pat.strip('- ') + current -= {mod for mod in current if fnmatch.fnmatch(mod, pat)} + elif pat: + current |= {mod for mod in all if fnmatch.fnmatch(mod, pat)} + return current + + available_modules = [] + modules_to_install = set() + for repo, repo_available_modules in modules.items(): + available_modules += repo_available_modules + + # repo specific filters + for repo, repo_available_modules in modules.items(): + repo_modules = repo_available_modules + repo_modules = _filter_patterns(repo.modules.split(','), repo_modules, repo_available_modules) + module_pattern = repo_module_patterns.get(repo) + if module_pattern: + repo_modules = _filter_patterns(module_pattern, repo_modules, repo_available_modules) + modules_to_install |= repo_modules + + # generic filters + modules_to_install = _filter_patterns(module_patterns, modules_to_install, available_modules) + + return sorted(modules_to_install) + + def action_test_modules_filters(self): + output = markupsafe.Markup() + sticky_bundles = self.env['runbot.bundle'].search([('project_id', '=', self.project_id.id), ('sticky', '=', True)]) + sticky_bundles = sticky_bundles.sorted(lambda b: b.version_id.number, reverse=True) + for sticky_bundle in sticky_bundles: + commits = sticky_bundle.last_batch.commit_ids + #if not commits: + # continue + output += markupsafe.Markup(f'''

%s

''') % sticky_bundle.name + for commit in commits: + if commit.repo_id in (self.repo_ids + self.dependency_ids).sorted('id'): + try: + module_list = [module for _addons_path, module, _manifest in commit._list_available_modules()] + filtered_modules = self._filter_modules_to_test({commit.repo_id: module_list}) + output += markupsafe.Markup(f'''

%s (%s/%s)

''') % (commit.repo_id.name, len(filtered_modules), len(module_list)) + output += ','.join(filtered_modules) + except subprocess.CalledProcessError as e: + output += markupsafe.Markup(f'''

{commit.repo_id.name}

Failed to get modules for {commit.repo_id.name}:{commit.name} {e}''') + self.message_post(body=output) + class Remote(models.Model): """ diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index c79b6e97..34c07cc4 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -67,6 +67,9 @@ access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,runbot access_runbot_trigger_user,access_runbot_trigger_user,runbot.model_runbot_trigger,runbot.group_user,1,0,0,0 access_runbot_trigger_runbot_admin,access_runbot_trigger_runbot_admin,runbot.model_runbot_trigger,runbot.group_runbot_admin,1,1,1,1 +access_runbot_module_filter_user,access_runbot_module_filter_user,runbot.model_runbot_module_filter,runbot.group_user,1,0,0,0 +access_runbot_module_filter_runbot_admin,access_runbot_module_filter_runbot_admin,runbot.model_runbot_module_filter,runbot.group_runbot_admin,1,1,1,1 + access_runbot_repo_user,access_runbot_repo_user,runbot.model_runbot_repo,runbot.group_user,1,0,0,0 access_runbot_repo_runbot_admin,access_runbot_repo_runbot_admin,runbot.model_runbot_repo,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 767d2b69..02fbc361 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -189,6 +189,8 @@ class TestBuildResult(RunbotCase): def test_filter_modules(self, mock_get_available_modules): """ test module filtering """ + self.addons_params.trigger_id = self.trigger_addons + build = self.Build.create({ 'params_id': self.addons_params.id, }) @@ -212,6 +214,23 @@ class TestBuildResult(RunbotCase): modules_to_test = build._get_modules_to_test(modules_patterns='*, -hw_*, hw_explicit') self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2'])) + self.env['runbot.module.filter'].create([{ + 'trigger_id': self.trigger_addons.id, + 'repo_id': self.repo_server.id, + 'modules': '-*', + }, { + 'trigger_id': self.trigger_addons.id, + 'repo_id': self.repo_addons.id, + 'modules': '*', + }, { + 'trigger_id': self.trigger_addons.id, + 'repo_id': self.repo_addons.id, + 'modules': '-other_mod_1', + }]) + modules_to_test = build._get_modules_to_test(modules_patterns='') + + self.assertEqual(modules_to_test, sorted(['other_mod_2'])) + def test_build_cmd_log_db(self, ): """ test that the log_db parameter is set in the .odoorc file """ build = self.Build.create({ diff --git a/runbot/views/repo_views.xml b/runbot/views/repo_views.xml index 49b4af5b..9d930c7f 100644 --- a/runbot/views/repo_views.xml +++ b/runbot/views/repo_views.xml @@ -33,6 +33,16 @@ + + + + + + + + + +