diff --git a/runbot/models/build.py b/runbot/models/build.py
index 5f232332..f1f09ce4 100644
--- a/runbot/models/build.py
+++ b/runbot/models/build.py
@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
import datetime
-import fnmatch
import getpass
import hashlib
import logging
@@ -926,29 +925,11 @@ class BuildResult(models.Model):
def _get_modules_to_test(self, modules_patterns=''):
self.ensure_one()
-
- def _filter_patterns(patterns, default, all):
- default = set(default)
- patterns_list = (patterns or '').split(',')
- patterns_list = [p.strip() for p in patterns_list]
- for pat in patterns_list:
- if pat.startswith('-'):
- pat = pat.strip('- ')
- default -= {mod for mod in default if fnmatch.fnmatch(mod, pat)}
- elif pat:
- default |= {mod for mod in all if fnmatch.fnmatch(mod, pat)}
- return default
-
- available_modules = []
- modules_to_install = set()
- for repo, module_list in self._get_available_modules().items():
- available_modules += module_list
- modules_to_install |= _filter_patterns(repo.modules, module_list, module_list)
-
- modules_to_install = _filter_patterns(self.params_id.modules, modules_to_install, available_modules)
- modules_to_install = _filter_patterns(modules_patterns, modules_to_install, available_modules)
-
- return sorted(modules_to_install)
+ trigger = self.params_id.trigger_id
+ modules = self._get_available_modules()
+ params_patterns = (self.params_id.modules or '').split(',')
+ modules_patterns = (modules_patterns or '').split(',')
+ return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns)
def _local_pg_dropdb(self, dbname):
msg = ''
diff --git a/runbot/models/commit.py b/runbot/models/commit.py
index 881f2a36..6281b86c 100644
--- a/runbot/models/commit.py
+++ b/runbot/models/commit.py
@@ -75,6 +75,31 @@ class Commit(models.Model):
module = os.path.basename(os.path.dirname(manifest_path))
yield (addons_path, module, manifest_file_name)
+ def _list_files(self, patterns):
+ #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py'
+ # note that glob is needed to avoid the star matching **
+ self.ensure_one()
+ return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n')
+
+ def _list_available_modules(self):
+ # beta version, may replace _get_available_modules latter
+ addons_paths = (self.repo_id.addons_paths or '').split(',')
+ patterns = []
+ for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py'
+ for addon_path in addons_paths:
+ addon_path = addon_path or '.'
+ patterns.append(f':(glob){addon_path}/*/{manifest_file_name}')
+ for file_path in self._list_files(patterns):
+ if file_path:
+ elems = file_path.rsplit('/', 2)
+ if len(elems) == 3:
+ addons_path, module, manifest_file_name = elems
+ else:
+ addons_path = ''
+ module, manifest_file_name = elems
+ yield (addons_path, module, manifest_file_name)
+
+
def _export(self, build):
"""Export a git repo into a sources"""
# TODO add automated tests
diff --git a/runbot/models/repo.py b/runbot/models/repo.py
index 5498f940..20ed8b0a 100644
--- a/runbot/models/repo.py
+++ b/runbot/models/repo.py
@@ -1,24 +1,32 @@
# -*- coding: utf-8 -*-
import datetime
+import fnmatch
import json
import logging
import re
import subprocess
import time
-
import requests
import markupsafe
from pathlib import Path
from odoo import models, fields, api
-from odoo.tools import file_open
+from odoo.tools import file_open, mail
from ..common import os, RunbotException, make_github_session, sanitize
from odoo.exceptions import UserError
from odoo.tools.safe_eval import safe_eval
_logger = logging.getLogger(__name__)
+class ModuleFilter(models.Model):
+ _name = 'runbot.module.filter'
+ _description = 'Module filter'
+
+ trigger_id = fields.Many2one('runbot.trigger', string="", required=True)
+ repo_id = fields.Many2one('runbot.repo', string="Repo", required=True)
+ modules = fields.Char(string="Module filter", required=True)
+ description = fields.Char(string="Description")
class Trigger(models.Model):
@@ -38,6 +46,7 @@ class Trigger(models.Model):
project_id = fields.Many2one('runbot.project', string="Project id", required=True)
repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]")
dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies")
+ module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger')
config_id = fields.Many2one('runbot.build.config', string="Config", required=True)
batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle")
@@ -91,6 +100,65 @@ class Trigger(models.Model):
return safe_eval(self.version_domain)
return []
+ def _filter_modules_to_test(self, modules, module_patterns=None):
+ repo_module_patterns = {}
+ for module_filter in self.module_filters:
+ repo_module_patterns.setdefault(module_filter.repo_id, [])
+ repo_module_patterns[module_filter.repo_id] += module_filter.modules.split(',')
+ module_patterns = module_patterns or []
+
+ def _filter_patterns(patterns_list, default, all):
+ current = set(default)
+ for pat in patterns_list:
+ pat = pat.strip()
+ if not pat:
+ continue
+ if pat.startswith('-'):
+ pat = pat.strip('- ')
+ current -= {mod for mod in current if fnmatch.fnmatch(mod, pat)}
+ elif pat:
+ current |= {mod for mod in all if fnmatch.fnmatch(mod, pat)}
+ return current
+
+ available_modules = []
+ modules_to_install = set()
+ for repo, repo_available_modules in modules.items():
+ available_modules += repo_available_modules
+
+ # repo specific filters
+ for repo, repo_available_modules in modules.items():
+ repo_modules = repo_available_modules
+ repo_modules = _filter_patterns(repo.modules.split(','), repo_modules, repo_available_modules)
+ module_pattern = repo_module_patterns.get(repo)
+ if module_pattern:
+ repo_modules = _filter_patterns(module_pattern, repo_modules, repo_available_modules)
+ modules_to_install |= repo_modules
+
+ # generic filters
+ modules_to_install = _filter_patterns(module_patterns, modules_to_install, available_modules)
+
+ return sorted(modules_to_install)
+
+ def action_test_modules_filters(self):
+ output = markupsafe.Markup()
+ sticky_bundles = self.env['runbot.bundle'].search([('project_id', '=', self.project_id.id), ('sticky', '=', True)])
+ sticky_bundles = sticky_bundles.sorted(lambda b: b.version_id.number, reverse=True)
+ for sticky_bundle in sticky_bundles:
+ commits = sticky_bundle.last_batch.commit_ids
+ #if not commits:
+ # continue
+ output += markupsafe.Markup(f'''%s
''') % sticky_bundle.name
+ for commit in commits:
+ if commit.repo_id in (self.repo_ids + self.dependency_ids).sorted('id'):
+ try:
+ module_list = [module for _addons_path, module, _manifest in commit._list_available_modules()]
+ filtered_modules = self._filter_modules_to_test({commit.repo_id: module_list})
+ output += markupsafe.Markup(f'''%s (%s/%s)
''') % (commit.repo_id.name, len(filtered_modules), len(module_list))
+ output += ','.join(filtered_modules)
+ except subprocess.CalledProcessError as e:
+ output += markupsafe.Markup(f'''{commit.repo_id.name}
Failed to get modules for {commit.repo_id.name}:{commit.name} {e}''')
+ self.message_post(body=output)
+
class Remote(models.Model):
"""
diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv
index c79b6e97..34c07cc4 100644
--- a/runbot/security/ir.model.access.csv
+++ b/runbot/security/ir.model.access.csv
@@ -67,6 +67,9 @@ access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,runbot
access_runbot_trigger_user,access_runbot_trigger_user,runbot.model_runbot_trigger,runbot.group_user,1,0,0,0
access_runbot_trigger_runbot_admin,access_runbot_trigger_runbot_admin,runbot.model_runbot_trigger,runbot.group_runbot_admin,1,1,1,1
+access_runbot_module_filter_user,access_runbot_module_filter_user,runbot.model_runbot_module_filter,runbot.group_user,1,0,0,0
+access_runbot_module_filter_runbot_admin,access_runbot_module_filter_runbot_admin,runbot.model_runbot_module_filter,runbot.group_runbot_admin,1,1,1,1
+
access_runbot_repo_user,access_runbot_repo_user,runbot.model_runbot_repo,runbot.group_user,1,0,0,0
access_runbot_repo_runbot_admin,access_runbot_repo_runbot_admin,runbot.model_runbot_repo,runbot.group_runbot_admin,1,1,1,1
diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py
index 767d2b69..02fbc361 100644
--- a/runbot/tests/test_build.py
+++ b/runbot/tests/test_build.py
@@ -189,6 +189,8 @@ class TestBuildResult(RunbotCase):
def test_filter_modules(self, mock_get_available_modules):
""" test module filtering """
+ self.addons_params.trigger_id = self.trigger_addons
+
build = self.Build.create({
'params_id': self.addons_params.id,
})
@@ -212,6 +214,23 @@ class TestBuildResult(RunbotCase):
modules_to_test = build._get_modules_to_test(modules_patterns='*, -hw_*, hw_explicit')
self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']))
+ self.env['runbot.module.filter'].create([{
+ 'trigger_id': self.trigger_addons.id,
+ 'repo_id': self.repo_server.id,
+ 'modules': '-*',
+ }, {
+ 'trigger_id': self.trigger_addons.id,
+ 'repo_id': self.repo_addons.id,
+ 'modules': '*',
+ }, {
+ 'trigger_id': self.trigger_addons.id,
+ 'repo_id': self.repo_addons.id,
+ 'modules': '-other_mod_1',
+ }])
+ modules_to_test = build._get_modules_to_test(modules_patterns='')
+
+ self.assertEqual(modules_to_test, sorted(['other_mod_2']))
+
def test_build_cmd_log_db(self, ):
""" test that the log_db parameter is set in the .odoorc file """
build = self.Build.create({
diff --git a/runbot/views/repo_views.xml b/runbot/views/repo_views.xml
index 49b4af5b..9d930c7f 100644
--- a/runbot/views/repo_views.xml
+++ b/runbot/views/repo_views.xml
@@ -33,6 +33,16 @@
+
+
+
+
+
+
+
+
+
+