mirror of
https://github.com/odoo/runbot.git
synced 2025-03-16 07:55:45 +07:00
[IMP] runbot: uses fnmatch patterns to select modules
Actually some Odoo modules are black_listed from a set hardcoded in the runbot code. In some cases, one needs to blacklist custom modules, preferably in a config_step. With this commit, the repo.modules, branch.modules, config_step.install_modules fields are concatained in a comma separated list of fnmatch patterns. The patterns can be prefixed with a dash to exclude the matching module(s). Co-authored by @Xavier-Do
This commit is contained in:
parent
7c46a2006f
commit
3308829e80
@ -6,7 +6,7 @@
|
||||
'author': "Odoo SA",
|
||||
'website': "http://runbot.odoo.com",
|
||||
'category': 'Website',
|
||||
'version': '4.4',
|
||||
'version': '4.5',
|
||||
'depends': ['website', 'base'],
|
||||
'data': [
|
||||
'security/runbot_security.xml',
|
||||
|
20
runbot/migrations/4.5/post-migration.py
Normal file
20
runbot/migrations/4.5/post-migration.py
Normal file
@ -0,0 +1,20 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
def migrate(cr, version):
|
||||
|
||||
repo_modules = '-auth_ldap,-document_ftp,-base_gengo,-website_gengo,-website_instantclick,-pad,-pad_project,-note_pad,-pos_cache,-pos_blackbox_be,-hw_*,-theme_*,-l10n_*,'
|
||||
cr.execute("UPDATE runbot_repo SET modules = CONCAT(%s, modules) WHERE modules_auto = 'all' or modules_auto = 'repo';", (repo_modules,))
|
||||
|
||||
# ceux qui n'ont pas d'étoile on prefix par '-*,'
|
||||
cr.execute("SELECT id,install_modules FROM runbot_build_config_step")
|
||||
for step_id, install_modules in cr.fetchall():
|
||||
install_modules_list = [mod.strip() for mod in (install_modules or '').split(',') if mod.strip()]
|
||||
if '*' in install_modules_list:
|
||||
install_modules_list.remove('*')
|
||||
install_modules = ', '.join(install_modules_list)
|
||||
elif install_modules_list:
|
||||
install_modules = '-*,%s' % install_modules
|
||||
else:
|
||||
install_modules = '-*'
|
||||
cr.execute("UPDATE runbot_build_config_step SET install_modules = %s WHERE id=%s", (install_modules, step_id))
|
@ -1,4 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import fnmatch
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
@ -690,26 +691,6 @@ class runbot_build(models.Model):
|
||||
return commit._source_path('odoo', *path)
|
||||
return commit._source_path('openerp', *path)
|
||||
|
||||
def _filter_modules(self, modules, available_modules, explicit_modules):
|
||||
# TODO add blacklist_modules and blacklist prefixes as data on repo
|
||||
blacklist_modules = set(['auth_ldap', 'document_ftp', 'base_gengo',
|
||||
'website_gengo', 'website_instantclick',
|
||||
'pad', 'pad_project', 'note_pad',
|
||||
'pos_cache', 'pos_blackbox_be'])
|
||||
|
||||
def mod_filter(module):
|
||||
if module not in available_modules:
|
||||
return False
|
||||
if module in explicit_modules:
|
||||
return True
|
||||
if module.startswith(('hw_', 'theme_', 'l10n_')):
|
||||
return False
|
||||
if module in blacklist_modules:
|
||||
return False
|
||||
return True
|
||||
|
||||
return uniq_list([module for module in modules if mod_filter(module)])
|
||||
|
||||
def _get_available_modules(self, commit):
|
||||
for manifest_file_name in commit.repo.manifest_files.split(','): # '__manifest__.py' '__openerp__.py'
|
||||
for addons_path in commit.repo.addons_paths.split(','): # '' 'addons' 'odoo/addons'
|
||||
@ -745,11 +726,9 @@ class runbot_build(models.Model):
|
||||
self._kill(result='ko')
|
||||
return exports
|
||||
|
||||
def _get_modules_to_test(self, commits=None):
|
||||
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
|
||||
# checkout branch
|
||||
repo_modules = []
|
||||
def _get_repo_available_modules(self, commits=None):
|
||||
available_modules = []
|
||||
repo_modules = []
|
||||
for commit in commits or self._get_all_commit():
|
||||
for (addons_path, module, manifest_file_name) in self._get_available_modules(commit):
|
||||
if commit.repo == self.repo_id:
|
||||
@ -762,25 +741,34 @@ class runbot_build(models.Model):
|
||||
)
|
||||
else:
|
||||
available_modules.append(module)
|
||||
explicit_modules = uniq_list([module for module in (self.branch_id.modules or '').split(',') + (self.repo_id.modules or '').split(',') if module])
|
||||
return repo_modules, available_modules
|
||||
|
||||
if explicit_modules:
|
||||
_logger.debug("explicit modules_to_test for build %s: %s", self.dest, explicit_modules)
|
||||
def _get_modules_to_test(self, commits=None, modules_patterns=''):
|
||||
self.ensure_one() # will raise exception if hash not found, we don't want to fail for all build.
|
||||
|
||||
if set(explicit_modules) - set(available_modules):
|
||||
self._log('checkout', 'Some explicit modules (branch or repo defined) are not in available module list.', level='WARNING')
|
||||
# checkout branch
|
||||
repo_modules, available_modules = self._get_repo_available_modules(commits=commits)
|
||||
|
||||
patterns_list = []
|
||||
for pats in [self.repo_id.modules, self.branch_id.modules, modules_patterns]:
|
||||
patterns_list += [p.strip() for p in (pats or '').split(',')]
|
||||
|
||||
if self.repo_id.modules_auto == 'all':
|
||||
modules_to_test = available_modules
|
||||
default_repo_modules = available_modules
|
||||
elif self.repo_id.modules_auto == 'repo':
|
||||
modules_to_test = explicit_modules + repo_modules
|
||||
_logger.debug("local modules_to_test for build %s: %s", self.dest, modules_to_test)
|
||||
default_repo_modules = repo_modules
|
||||
else:
|
||||
modules_to_test = explicit_modules
|
||||
default_repo_modules = []
|
||||
|
||||
modules_to_test = self._filter_modules(modules_to_test, available_modules, explicit_modules)
|
||||
_logger.debug("modules_to_test for build %s: %s", self.dest, modules_to_test)
|
||||
return modules_to_test
|
||||
modules_to_install = set(default_repo_modules)
|
||||
for pat in patterns_list:
|
||||
if pat.startswith('-'):
|
||||
pat = pat.strip('- ')
|
||||
modules_to_install -= {mod for mod in modules_to_install if fnmatch.fnmatch(mod, pat)}
|
||||
else:
|
||||
modules_to_install |= {mod for mod in available_modules if fnmatch.fnmatch(mod, pat)}
|
||||
|
||||
return sorted(modules_to_install)
|
||||
|
||||
def _local_pg_dropdb(self, dbname):
|
||||
with local_pgadmin_cursor() as local_cr:
|
||||
|
@ -97,7 +97,7 @@ class ConfigStep(models.Model):
|
||||
# install_odoo
|
||||
create_db = fields.Boolean('Create Db', default=True, track_visibility='onchange') # future
|
||||
custom_db_name = fields.Char('Custom Db Name', track_visibility='onchange') # future
|
||||
install_modules = fields.Char('Modules to install', help="List of module to install, use * for all modules", default='*')
|
||||
install_modules = fields.Char('Modules to install', help="List of module patterns to install, use * to install all available modules, prefix the pattern with dash to remove the module.", default='*')
|
||||
db_name = fields.Char('Db Name', compute='_compute_db_name', inverse='_inverse_db_name', track_visibility='onchange')
|
||||
cpu_limit = fields.Integer('Cpu limit', default=3600, track_visibility='onchange')
|
||||
coverage = fields.Boolean('Coverage', dafault=False, track_visibility='onchange')
|
||||
@ -339,13 +339,7 @@ class ConfigStep(models.Model):
|
||||
return docker_run(cmd.build(), log_path, build._path(), build._get_docker_name(), cpu_limit=timeout, ro_volumes=exports)
|
||||
|
||||
def _modules_to_install(self, build):
|
||||
modules_to_install = set([mod.strip() for mod in self.install_modules.split(',')])
|
||||
if '*' in modules_to_install:
|
||||
modules_to_install.remove('*')
|
||||
default_mod = set(build._get_modules_to_test())
|
||||
modules_to_install = default_mod | modules_to_install
|
||||
# todo add without support
|
||||
return modules_to_install
|
||||
return set(build._get_modules_to_test(modules_patterns=self.install_modules))
|
||||
|
||||
def _post_install_command(self, build, modules_to_install, py_version=None):
|
||||
if self.coverage:
|
||||
|
@ -1,6 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from unittest.mock import patch
|
||||
from odoo.tools.config import configmanager
|
||||
from odoo.tests import common
|
||||
|
||||
|
||||
@ -71,6 +70,35 @@ class Test_Build(common.TransactionCase):
|
||||
with self.assertRaises(AssertionError):
|
||||
builds.write({'local_state': 'duplicate'})
|
||||
|
||||
@patch('odoo.addons.runbot.models.build.runbot_build._get_repo_available_modules')
|
||||
@patch('odoo.addons.runbot.models.build.runbot_build._get_params')
|
||||
@patch('odoo.addons.runbot.models.build.fqdn')
|
||||
def test_filter_modules(self, mock_fqdn, mock_get_params, mock_get_repo_mods):
|
||||
""" test module filtering """
|
||||
build = self.Build.create({
|
||||
'branch_id': self.branch.id,
|
||||
'name': 'd0d0caca0000ffffffffffffffffffffffffffff',
|
||||
'port': '1234',
|
||||
})
|
||||
|
||||
repo_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit']
|
||||
available_mods = ['good_module', 'bad_module', 'other_good', 'l10n_be', 'hw_foo', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']
|
||||
mock_get_repo_mods.return_value = (repo_mods, available_mods)
|
||||
self.repo.modules_auto = 'repo'
|
||||
self.repo.modules = '-bad_module,-hw_*,hw_explicit,-l10n_*'
|
||||
modules_to_test = build._get_modules_to_test(self, modules_patterns='')
|
||||
self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit']))
|
||||
|
||||
modules_to_test = build._get_modules_to_test(self, modules_patterns='-*, l10n_be')
|
||||
self.assertEqual(modules_to_test, sorted(['l10n_be']))
|
||||
|
||||
modules_to_test = build._get_modules_to_test(self, modules_patterns='l10n_be')
|
||||
self.assertEqual(modules_to_test, sorted(['good_module', 'hwgood', 'other_good', 'hw_explicit', 'l10n_be']))
|
||||
|
||||
# star to get all available mods
|
||||
modules_to_test = build._get_modules_to_test(self, modules_patterns='*, -hw_*, hw_explicit')
|
||||
self.assertEqual(modules_to_test, sorted(['good_module', 'bad_module', 'other_good', 'l10n_be', 'hwgood', 'hw_explicit', 'other_mod_1', 'other_mod_2']))
|
||||
|
||||
@patch('odoo.addons.runbot.models.build.os.path.isfile')
|
||||
@patch('odoo.addons.runbot.models.build.os.mkdir')
|
||||
@patch('odoo.addons.runbot.models.build.grep')
|
||||
|
Loading…
Reference in New Issue
Block a user