NextERP/runbot
5
0
Fork 0
mirror of https://github.com/odoo/runbot.git synced 2025-03-16 07:55:45 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

[IMP] runbot: uses fnmatch patterns to select modules

Browse Source 
Actually some Odoo modules are black_listed from a set hardcoded in the
runbot code. In some cases, one needs to blacklist custom modules,
preferably in a config_step.

With this commit, the repo.modules, branch.modules,
config_step.install_modules fields are concatained in a comma separated
list of fnmatch patterns. The patterns can be prefixed with a dash to
exclude the matching module(s).

Co-authored by @Xavier-Do
This commit is contained in:
Christophe Monniez 2019-09-23 23:08:50 +02:00 committed by XavierDo
parent 7c46a2006f
commit 3308829e80
5 changed files with 76 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runbot/__manifest__.py
View File

@ -6,7 +6,7 @@
'author': "Odoo SA",
'website': "http://runbot.odoo.com",
'category': 'Website',
'version': '4.4',
'version': '4.5',
'depends': ['website', 'base'],
'data': [
'security/runbot_security.xml',

20
runbot/migrations/4.5/post-migration.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
def migrate(cr, version):
repo_modules = '-auth_ldap,-document_ftp,-base_gengo,-website_gengo,-website_instantclick,-pad,-pad_project,-note_pad,-pos_cache,-pos_blackbox_be,-hw_*,-theme_*,-l10n_*,'
cr.execute("UPDATE runbot_repo SET modules = CONCAT(%s, modules) WHERE modules_auto = 'all' or modules_auto = 'repo';", (repo_modules,))
# ceux qui n'ont pas d'étoile on prefix par '-*,'
cr.execute("SELECT id,install_modules FROM runbot_build_config_step")
for step_id, install_modules in cr.fetchall():
install_modules_list = [mod.strip() for mod in (install_modules or '').split(',') if mod.strip()]
if '*' in install_modules_list:
install_modules_list.remove('*')
install_modules = ', '.join(install_modules_list)
elif install_modules_list:
install_modules = '-*,%s' % install_modules
else:
install_modules = '-*'
cr.execute("UPDATE runbot_build_config_step SET install_modules = %s WHERE id=%s", (install_modules, step_id))

60
runbot/models/build.py
View File

@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
import fnmatch
import glob
import logging
import os
@ -690,26 +691,6 @@ class runbot_build(models.Model):
return commit._source_path('odoo', *path)
return commit._source_path('openerp', *path)
def _filter_modules(self, modules, available_modules, explicit_modules):
# TODO add blacklist_modules and blacklist prefixes as data on repo
blacklist_modules = set(['auth_ldap', 'document_ftp', 'base_gengo',
'website_gengo', 'website_instantclick',
'pad', 'pad_project', 'note_pad',
'pos_cache', 'pos_blackbox_be'])
def mod_filter(module):
if module not in available_modules:
return False
if module in explicit_modules:
return True
if module.startswith(('hw_', 'theme_', 'l10n_')):
return False
if module in blacklist_modules:
return False
return True
return uniq_list([module for module in modules if mod_filter(module)])
def _get_available_modules(self, commit):
for manifest_file_name in commit.repo.manifest_files.split(','): # '__manifest__.py' '__openerp__.py'
for addons_path in commit.repo.addons_paths.split(','): # '' 'addons' 'odoo/addons'
@ -745,11 +726,9 @@ class runbot_build(models.Model):
self._kill(result='ko')
return exports
def _get_modules_to_test(self, commits=None):
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
# checkout branch
repo_modules = []
def _get_repo_available_modules(self, commits=None):
available_modules = []
repo_modules = []
for commit in commits or self._get_all_commit():
for (addons_path, module, manifest_file_name) in self._get_available_modules(commit):
if commit.repo == self.repo_id:
@ -762,25 +741,34 @@ class runbot_build(models.Model):
)
else:
available_modules.append(module)
explicit_modules = uniq_list([module for module in (self.branch_id.modules or '').split(',') + (self.repo_id.modules or '').split(',') if module])
return repo_modules, available_modules
if explicit_modules:
_logger.debug("explicit modules_to_test for build %s: %s", self.dest, explicit_modules)
def _get_modules_to_test(self, commits=None, modules_patterns=''):
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
if set(explicit_modules) - set(available_modules):
self._log('checkout', 'Some explicit modules (branch or repo defined) are not in available module list.', level='WARNING')
# checkout branch
repo_modules, available_modules = self._get_repo_available_modules(commits=commits)
patterns_list = []
for pats in [self.repo_id.modules, self.branch_id.modules, modules_patterns]:
patterns_list += [p.strip() for p in (pats or '').split(',')]
if self.repo_id.modules_auto == 'all':
modules_to_test = available_modules
default_repo_modules = available_modules
elif self.repo_id.modules_auto == 'repo':
modules_to_test = explicit_modules + repo_modules
_logger.debug("local modules_to_test for build %s: %s", self.dest, modules_to_test)
default_repo_modules = repo_modules
else:
modules_to_test = explicit_modules
default_repo_modules = []
modules_to_test = self._filter_modules(modules_to_test, available_modules, explicit_modules)
_logger.debug("modules_to_test for build %s: %s", self.dest, modules_to_test)
return modules_to_test
modules_to_install = set(default_repo_modules)
for pat in patterns_list:
if pat.startswith('-'):
pat = pat.strip('- ')
modules_to_install -= {mod for mod in modules_to_install if fnmatch.fnmatch(mod, pat)}
else:
modules_to_install |= {mod for mod in available_modules if fnmatch.fnmatch(mod, pat)}
return sorted(modules_to_install)
def _local_pg_dropdb(self, dbname):
with local_pgadmin_cursor() as local_cr:

10
runbot/models/build_config.py
View File

@ -97,7 +97,7 @@ class ConfigStep(models.Model):
# install_odoo
create_db = fields.Boolean('Create Db', default=True, track_visibility='onchange') # future
custom_db_name = fields.Char('Custom Db Name', track_visibility='onchange') # future
install_modules = fields.Char('Modules to install', help="List of module to install, use * for all modules", default='*')
install_modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='*')
db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', track_visibility='onchange')
cpu_limit = fields.Integer('Cpu limit', default=3600, track_visibility='onchange')
coverage = fields.Boolean('Coverage', dafault=False, track_visibility='onchange')
@ -339,13 +339,7 @@ class ConfigStep(models.Model):
return docker_run(cmd.build(), log_path, build._path(), build._get_docker_name(), cpu_limit=timeout, ro_volumes=exports)
def _modules_to_install(self, build):
modules_to_install = set([mod.strip() for mod in self.install_modules.split(',')])
if '*' in modules_to_install:
modules_to_install.remove('*')
default_mod = set(build._get_modules_to_test())
modules_to_install = default_mod | modules_to_install
# todo add without support
return modules_to_install
return set(build._get_modules_to_test(modules_patterns=self.install_modules))
def _post_install_command(self, build, modules_to_install, py_version=None):
if self.coverage:

30
runbot/tests/test_build.py
View File

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
from unittest.mock import patch
from odoo.tools.config import configmanager
from odoo.tests import common
@ -71,6 +70,35 @@ class Test_Build(common.TransactionCase):
with self.assertRaises(AssertionError):
builds.write({'local_state': 'duplicate'})
@patch('odoo.addons.runbot.models.build.runbot_build._get_repo_available_modules')
@patch('odoo.addons.runbot.models.build.runbot_build._get_params')
@patch('odoo.addons.runbot.models.build.fqdn')
def test_filter_modules(self, mock_fqdn, mock_get_params, mock_get_repo_mods):
""" test module filtering """
build = self.Build.create({
'branch_id': self.branch.id,
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
'port': '1234',
})
repo_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit']
available_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']
mock_get_repo_mods.return_value = (repo_mods, available_mods)
self.repo.modules_auto = 'repo'
self.repo.modules = '-bad_module,-hw_*,hw_explicit,-l10n_*'
modules_to_test = build._get_modules_to_test(self, modules_patterns='')
self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit']))
modules_to_test = build._get_modules_to_test(self, modules_patterns='-*, l10n_be')
self.assertEqual(modules_to_test, sorted(['l10n_be']))
modules_to_test = build._get_modules_to_test(self, modules_patterns='l10n_be')
self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit', 'l10n_be']))
# star to get all available mods
modules_to_test = build._get_modules_to_test(self, modules_patterns='*, -hw_*, hw_explicit')
self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']))
@patch('odoo.addons.runbot.models.build.os.path.isfile')
@patch('odoo.addons.runbot.models.build.os.mkdir')
@patch('odoo.addons.runbot.models.build.grep')