diff --git a/runbot/models/build.py b/runbot/models/build.py index 5f232332..f1f09ce4 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import datetime -import fnmatch import getpass import hashlib import logging @@ -926,29 +925,11 @@ class BuildResult(models.Model): def _get_modules_to_test(self, modules_patterns=''): self.ensure_one() - - def _filter_patterns(patterns, default, all): - default = set(default) - patterns_list = (patterns or '').split(',') - patterns_list = [p.strip() for p in patterns_list] - for pat in patterns_list: - if pat.startswith('-'): - pat = pat.strip('- ') - default -= {mod for mod in default if fnmatch.fnmatch(mod, pat)} - elif pat: - default |= {mod for mod in all if fnmatch.fnmatch(mod, pat)} - return default - - available_modules = [] - modules_to_install = set() - for repo, module_list in self._get_available_modules().items(): - available_modules += module_list - modules_to_install |= _filter_patterns(repo.modules, module_list, module_list) - - modules_to_install = _filter_patterns(self.params_id.modules, modules_to_install, available_modules) - modules_to_install = _filter_patterns(modules_patterns, modules_to_install, available_modules) - - return sorted(modules_to_install) + trigger = self.params_id.trigger_id + modules = self._get_available_modules() + params_patterns = (self.params_id.modules or '').split(',') + modules_patterns = (modules_patterns or '').split(',') + return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns) def _local_pg_dropdb(self, dbname): msg = '' diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 881f2a36..6281b86c 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -75,6 +75,31 @@ class Commit(models.Model): module = os.path.basename(os.path.dirname(manifest_path)) yield (addons_path, module, manifest_file_name) + def _list_files(self, patterns): + #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py' + # note that glob is needed to avoid the star matching ** + self.ensure_one() + return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n') + + def _list_available_modules(self): + # beta version, may replace _get_available_modules latter + addons_paths = (self.repo_id.addons_paths or '').split(',') + patterns = [] + for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' + for addon_path in addons_paths: + addon_path = addon_path or '.' + patterns.append(f':(glob){addon_path}/*/{manifest_file_name}') + for file_path in self._list_files(patterns): + if file_path: + elems = file_path.rsplit('/', 2) + if len(elems) == 3: + addons_path, module, manifest_file_name = elems + else: + addons_path = '' + module, manifest_file_name = elems + yield (addons_path, module, manifest_file_name) + + def _export(self, build): """Export a git repo into a sources""" # TODO add automated tests diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 5498f940..20ed8b0a 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -1,24 +1,32 @@ # -*- coding: utf-8 -*- import datetime +import fnmatch import json import logging import re import subprocess import time - import requests import markupsafe from pathlib import Path from odoo import models, fields, api -from odoo.tools import file_open +from odoo.tools import file_open, mail from ..common import os, RunbotException, make_github_session, sanitize from odoo.exceptions import UserError from odoo.tools.safe_eval import safe_eval _logger = logging.getLogger(__name__) +class ModuleFilter(models.Model): + _name = 'runbot.module.filter' + _description = 'Module filter' + + trigger_id = fields.Many2one('runbot.trigger', string="", required=True) + repo_id = fields.Many2one('runbot.repo', string="Repo", required=True) + modules = fields.Char(string="Module filter", required=True) + description = fields.Char(string="Description") class Trigger(models.Model): @@ -38,6 +46,7 @@ class Trigger(models.Model): project_id = fields.Many2one('runbot.project', string="Project id", required=True) repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]") dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies") + module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger') config_id = fields.Many2one('runbot.build.config', string="Config", required=True) batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle") @@ -91,6 +100,65 @@ class Trigger(models.Model): return safe_eval(self.version_domain) return [] + def _filter_modules_to_test(self, modules, module_patterns=None): + repo_module_patterns = {} + for module_filter in self.module_filters: + repo_module_patterns.setdefault(module_filter.repo_id, []) + repo_module_patterns[module_filter.repo_id] += module_filter.modules.split(',') + module_patterns = module_patterns or [] + + def _filter_patterns(patterns_list, default, all): + current = set(default) + for pat in patterns_list: + pat = pat.strip() + if not pat: + continue + if pat.startswith('-'): + pat = pat.strip('- ') + current -= {mod for mod in current if fnmatch.fnmatch(mod, pat)} + elif pat: + current |= {mod for mod in all if fnmatch.fnmatch(mod, pat)} + return current + + available_modules = [] + modules_to_install = set() + for repo, repo_available_modules in modules.items(): + available_modules += repo_available_modules + + # repo specific filters + for repo, repo_available_modules in modules.items(): + repo_modules = repo_available_modules + repo_modules = _filter_patterns(repo.modules.split(','), repo_modules, repo_available_modules) + module_pattern = repo_module_patterns.get(repo) + if module_pattern: + repo_modules = _filter_patterns(module_pattern, repo_modules, repo_available_modules) + modules_to_install |= repo_modules + + # generic filters + modules_to_install = _filter_patterns(module_patterns, modules_to_install, available_modules) + + return sorted(modules_to_install) + + def action_test_modules_filters(self): + output = markupsafe.Markup() + sticky_bundles = self.env['runbot.bundle'].search([('project_id', '=', self.project_id.id), ('sticky', '=', True)]) + sticky_bundles = sticky_bundles.sorted(lambda b: b.version_id.number, reverse=True) + for sticky_bundle in sticky_bundles: + commits = sticky_bundle.last_batch.commit_ids + #if not commits: + # continue + output += markupsafe.Markup(f'''