[IMP] runbot: allow to customize repo filters from

When using a repo as a dependency for another trigger, the default
module filter for a repo is not always ideal

As an example, when using odoo as a dependency for another repo,
we may only want to install the module from the new repo.

This iss done right now by creating a custom config but this lead to
duplicates config and steps only to customize the module to install.

This commit proposes a new model to store the filters.

Note that this may be used later as module blacklist on repo too.

runbot/models/build.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import datetime
-import fnmatch
 import getpass
 import hashlib
 import logging
@@ -926,29 +925,11 @@ class BuildResult(models.Model):
 
     def _get_modules_to_test(self, modules_patterns=''):
         self.ensure_one()
-
-        def _filter_patterns(patterns, default, all):
-            default = set(default)
-            patterns_list = (patterns or '').split(',')
-            patterns_list = [p.strip() for p in patterns_list]
-            for pat in patterns_list:
-                if pat.startswith('-'):
-                    pat = pat.strip('- ')
-                    default -= {mod for mod in default if fnmatch.fnmatch(mod, pat)}
-                elif pat:
-                    default |= {mod for mod in all if fnmatch.fnmatch(mod, pat)}
-            return default
-
-        available_modules = []
-        modules_to_install = set()
-        for repo, module_list in self._get_available_modules().items():
-            available_modules += module_list
-            modules_to_install |= _filter_patterns(repo.modules, module_list, module_list)
-
-        modules_to_install = _filter_patterns(self.params_id.modules, modules_to_install, available_modules)
-        modules_to_install = _filter_patterns(modules_patterns, modules_to_install, available_modules)
-
-        return sorted(modules_to_install)
+        trigger = self.params_id.trigger_id
+        modules = self._get_available_modules()
+        params_patterns = (self.params_id.modules or '').split(',')
+        modules_patterns = (modules_patterns or '').split(',')
+        return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns)
 
     def _local_pg_dropdb(self, dbname):
         msg = ''

runbot/models/commit.py

@@ -75,6 +75,31 @@ class Commit(models.Model):
                     module = os.path.basename(os.path.dirname(manifest_path))
                     yield (addons_path, module, manifest_file_name)
 
+    def _list_files(self, patterns):
+        #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py'
+        # note that glob is needed to avoid the star matching **
+        self.ensure_one()
+        return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n')
+
+    def _list_available_modules(self):
+        # beta version, may replace _get_available_modules latter
+        addons_paths = (self.repo_id.addons_paths or '').split(',')
+        patterns = []
+        for manifest_file_name in self.repo_id.manifest_files.split(','):  # '__manifest__.py' '__openerp__.py'
+            for addon_path in addons_paths:
+                addon_path = addon_path or '.'
+                patterns.append(f':(glob){addon_path}/*/{manifest_file_name}')
+        for file_path in self._list_files(patterns):
+            if file_path:
+                elems = file_path.rsplit('/', 2)
+                if len(elems) == 3:
+                    addons_path, module, manifest_file_name = elems
+                else:
+                    addons_path = ''
+                    module, manifest_file_name = elems
+                yield (addons_path, module, manifest_file_name)
+
+
     def _export(self, build):
         """Export a git repo into a sources"""
         #  TODO add automated tests

runbot/models/repo.py

@@ -1,24 +1,32 @@
 # -*- coding: utf-8 -*-
 import datetime
+import fnmatch
 import json
 import logging
 import re
 import subprocess
 import time
-
 import requests
 import markupsafe
 
 from pathlib import Path
 
 from odoo import models, fields, api
-from odoo.tools import file_open
+from odoo.tools import file_open, mail
 from ..common import os, RunbotException, make_github_session, sanitize
 from odoo.exceptions import UserError
 from odoo.tools.safe_eval import safe_eval
 
 _logger = logging.getLogger(__name__)
 
+class ModuleFilter(models.Model):
+    _name = 'runbot.module.filter'
+    _description = 'Module filter'
+
+    trigger_id = fields.Many2one('runbot.trigger', string="", required=True)
+    repo_id = fields.Many2one('runbot.repo', string="Repo", required=True)
+    modules = fields.Char(string="Module filter", required=True)
+    description = fields.Char(string="Description")
 
 
 class Trigger(models.Model):
@@ -38,6 +46,7 @@ class Trigger(models.Model):
     project_id = fields.Many2one('runbot.project', string="Project id", required=True)
     repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]")
     dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies")
+    module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger')
     config_id = fields.Many2one('runbot.build.config', string="Config", required=True)
     batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle")
 
@@ -91,6 +100,65 @@ class Trigger(models.Model):
             return safe_eval(self.version_domain)
         return []
 
+    def _filter_modules_to_test(self, modules, module_patterns=None):
+        repo_module_patterns = {}
+        for module_filter in self.module_filters:
+            repo_module_patterns.setdefault(module_filter.repo_id, [])
+            repo_module_patterns[module_filter.repo_id] += module_filter.modules.split(',')
+        module_patterns = module_patterns or []
+
+        def _filter_patterns(patterns_list, default, all):
+            current = set(default)
+            for pat in patterns_list:
+                pat = pat.strip()
+                if not pat:
+                    continue
+                if pat.startswith('-'):
+                    pat = pat.strip('- ')
+                    current -= {mod for mod in current if fnmatch.fnmatch(mod, pat)}
+                elif pat:
+                    current |= {mod for mod in all if fnmatch.fnmatch(mod, pat)}
+            return current
+
+        available_modules = []
+        modules_to_install = set()
+        for repo, repo_available_modules in modules.items():
+            available_modules += repo_available_modules
+
+        # repo specific filters
+        for repo, repo_available_modules in modules.items():
+            repo_modules = repo_available_modules
+            repo_modules = _filter_patterns(repo.modules.split(','), repo_modules, repo_available_modules)
+            module_pattern = repo_module_patterns.get(repo)
+            if module_pattern:
+                repo_modules = _filter_patterns(module_pattern, repo_modules, repo_available_modules)
+            modules_to_install |= repo_modules
+
+        # generic filters
+        modules_to_install = _filter_patterns(module_patterns, modules_to_install, available_modules)
+
+        return sorted(modules_to_install)
+
+    def action_test_modules_filters(self):
+        output = markupsafe.Markup()
+        sticky_bundles = self.env['runbot.bundle'].search([('project_id', '=', self.project_id.id), ('sticky', '=', True)])
+        sticky_bundles = sticky_bundles.sorted(lambda b: b.version_id.number, reverse=True)
+        for sticky_bundle in sticky_bundles:
+            commits = sticky_bundle.last_batch.commit_ids
+            #if not commits:
+            #    continue
+            output += markupsafe.Markup(f'''<h2>%s</h2>''') % sticky_bundle.name
+            for commit in commits:
+                if commit.repo_id in (self.repo_ids + self.dependency_ids).sorted('id'):
+                    try:
+                        module_list = [module for _addons_path, module, _manifest in commit._list_available_modules()]
+                        filtered_modules = self._filter_modules_to_test({commit.repo_id: module_list})
+                        output += markupsafe.Markup(f'''<h4>%s (%s/%s)</h4>''') % (commit.repo_id.name, len(filtered_modules), len(module_list))
+                        output += ','.join(filtered_modules)
+                    except subprocess.CalledProcessError as e:
+                        output += markupsafe.Markup(f'''<h4>{commit.repo_id.name}</h4> Failed to get modules for {commit.repo_id.name}:{commit.name} {e}''')
+        self.message_post(body=output)
+
 
 class Remote(models.Model):
     """

runbot/security/ir.model.access.csv

@@ -67,6 +67,9 @@ access_runbot_build_stat_regex_admin,access_runbot_build_stat_regex_admin,runbot
 access_runbot_trigger_user,access_runbot_trigger_user,runbot.model_runbot_trigger,runbot.group_user,1,0,0,0
 access_runbot_trigger_runbot_admin,access_runbot_trigger_runbot_admin,runbot.model_runbot_trigger,runbot.group_runbot_admin,1,1,1,1
 
+access_runbot_module_filter_user,access_runbot_module_filter_user,runbot.model_runbot_module_filter,runbot.group_user,1,0,0,0
+access_runbot_module_filter_runbot_admin,access_runbot_module_filter_runbot_admin,runbot.model_runbot_module_filter,runbot.group_runbot_admin,1,1,1,1
+
 access_runbot_repo_user,access_runbot_repo_user,runbot.model_runbot_repo,runbot.group_user,1,0,0,0
 access_runbot_repo_runbot_admin,access_runbot_repo_runbot_admin,runbot.model_runbot_repo,runbot.group_runbot_admin,1,1,1,1
 

runbot/tests/test_build.py

@@ -189,6 +189,8 @@ class TestBuildResult(RunbotCase):
     def test_filter_modules(self, mock_get_available_modules):
         """ test module filtering """
 
+        self.addons_params.trigger_id = self.trigger_addons
+
         build = self.Build.create({
             'params_id': self.addons_params.id,
         })
@@ -212,6 +214,23 @@ class TestBuildResult(RunbotCase):
         modules_to_test = build._get_modules_to_test(modules_patterns='*, -hw_*, hw_explicit')
         self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']))
 
+        self.env['runbot.module.filter'].create([{
+            'trigger_id': self.trigger_addons.id,
+            'repo_id': self.repo_server.id,
+            'modules': '-*',
+        }, {
+            'trigger_id': self.trigger_addons.id,
+            'repo_id': self.repo_addons.id,
+            'modules': '*',
+        }, {
+            'trigger_id': self.trigger_addons.id,
+            'repo_id': self.repo_addons.id,
+            'modules': '-other_mod_1',
+        }])
+        modules_to_test = build._get_modules_to_test(modules_patterns='')
+
+        self.assertEqual(modules_to_test, sorted(['other_mod_2']))
+
     def test_build_cmd_log_db(self, ):
         """ test that the log_db parameter is set in the .odoorc file """
         build = self.Build.create({

runbot/views/repo_views.xml

@@ -33,6 +33,16 @@
                 <group string="Dependencies">
                   <field name="dependency_ids" nolabel="1" colspan="2"/>
                 </group>
+                <group string="Module filters">
+                  <field name="module_filters" nolabel="1" colspan="4">
+                    <tree string="Module filters" editable="bottom">
+                      <field name="repo_id"  domain="['|', ('id', 'in', parent.repo_ids), ('id', 'in', parent.dependency_ids)]"/>
+                      <field name="modules"/>
+                      <field name="description"/>
+                    </tree>
+                  </field>
+                </group>
+                <button class="btn btn-sm btn-primary" type="object" name="action_test_modules_filters" title="Test filters">List modules</button>
               </group>
               <group>
                 <group>