From 78f050b13203c37f359cdf7238d9febf91b0c011 Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Wed, 21 Oct 2020 12:05:36 +0200 Subject: [PATCH] [IMP] runbot: adapt for 14.0 --- runbot/controllers/frontend.py | 19 ++--- runbot/models/build.py | 4 +- runbot/models/build_config.py | 42 ++++++---- runbot/models/build_error.py | 4 +- runbot/models/build_stat.py | 4 +- runbot/models/bundle.py | 13 +-- runbot/models/commit.py | 4 +- runbot/models/custom_trigger.py | 2 +- runbot/models/database.py | 2 +- runbot/models/dockerfile.py | 2 +- runbot/models/event.py | 6 +- runbot/models/host.py | 2 +- runbot/models/project.py | 2 +- runbot/models/repo.py | 8 +- runbot/models/runbot.py | 19 ++--- runbot/security/ir.model.access.csv | 4 + runbot/templates/frontend.xml | 2 +- runbot/tests/test_build_config_step.py | 2 +- runbot/tests/test_commit.py | 108 ++++++++++++------------- runbot_cla/build_config.py | 2 +- 20 files changed, 131 insertions(+), 120 deletions(-) diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 6843746b..2ce959ec 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -143,17 +143,14 @@ class Runbot(Controller): domain = expression.AND([domain, search_domain]) e = expression.expression(domain, request.env['runbot.bundle']) - where_clause, where_params = e.to_sql() - - env.cr.execute(""" - SELECT id FROM runbot_bundle - WHERE {where_clause} - ORDER BY - (case when sticky then 1 when sticky is null then 2 else 2 end), - case when sticky then version_number end collate "C" desc, - last_batch desc - LIMIT 40""".format(where_clause=where_clause), where_params) - bundles = env['runbot.bundle'].browse([r[0] for r in env.cr.fetchall()]) + query = e.query + query.order = """ + (case when "runbot_bundle".sticky then 1 when "runbot_bundle".sticky is null then 2 else 2 end), + case when "runbot_bundle".sticky then "runbot_bundle".version_number end collate "C" desc, + "runbot_bundle".last_batch desc + """ + query.limit=40 + bundles = env['runbot.bundle'].browse(query) category_id = int(request.httprequest.cookies.get('category') or 0) or request.env['ir.model.data'].xmlid_to_res_id('runbot.default_category') diff --git a/runbot/models/build.py b/runbot/models/build.py index f4b0b09b..5521b2cd 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -145,7 +145,7 @@ class BuildResult(models.Model): # could be a default value, but possible to change it to allow duplicate accros branches description = fields.Char('Description', help='Informative description') - md_description = fields.Char(compute='_compute_md_description', String='MD Parsed Description', help='Informative description markdown parsed') + md_description = fields.Char(compute='_compute_md_description', string='MD Parsed Description', help='Informative description markdown parsed') display_name = fields.Char(compute='_compute_display_name') # Related fields for convenience @@ -171,7 +171,7 @@ class BuildResult(models.Model): # logs and stats log_ids = fields.One2many('ir.logging', 'build_id', string='Logs') error_log_ids = fields.One2many('ir.logging', 'build_id', domain=[('level', 'in', ['WARNING', 'ERROR', 'CRITICAL'])], string='Error Logs') - stat_ids = fields.One2many('runbot.build.stat', 'build_id', strings='Statistics values') + stat_ids = fields.One2many('runbot.build.stat', 'build_id', string='Statistics values') log_list = fields.Char('Comma separted list of step_ids names with logs', compute="_compute_log_list", store=True) active_step = fields.Many2one('runbot.build.config.step', 'Active step') diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index cca943da..67729468 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -20,6 +20,18 @@ _re_warning = r'^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} \d+ WARNING ' PYTHON_DEFAULT = "# type python code here\n\n\n\n\n\n" +class ReProxy(): + @classmethod + def match(cls, *args, **kwrags): + return re.match(*args, **kwrags) + + @classmethod + def search(cls, *args, **kwrags): + return re.search(*args, **kwrags) + + @classmethod + def compile(cls, *args, **kwrags): + return re.compile(*args, **kwrags) class Config(models.Model): _name = 'runbot.build.config' @@ -95,16 +107,7 @@ class ConfigStepUpgradeDb(models.Model): db_pattern = fields.Char('Db suffix pattern') min_target_version_id = fields.Many2one('runbot.version', "Minimal target version_id") - -class ConfigStep(models.Model): - _name = 'runbot.build.config.step' - _description = "Config step" - _inherit = 'mail.thread' - - # general info - name = fields.Char('Step name', required=True, unique=True, tracking=True, help="Unique name for step please use trigram as postfix for custom step_ids") - domain_filter = fields.Char('Domain filter', tracking=True) - job_type = fields.Selection([ +TYPES = [ ('install_odoo', 'Test odoo'), ('run_odoo', 'Run odoo'), ('python', 'Python code'), @@ -113,7 +116,16 @@ class ConfigStep(models.Model): ('configure_upgrade_complement', 'Configure Upgrade Complement'), ('test_upgrade', 'Test Upgrade'), ('restore', 'Restore') - ], default='install_odoo', required=True, tracking=True) + ] +class ConfigStep(models.Model): + _name = 'runbot.build.config.step' + _description = "Config step" + _inherit = 'mail.thread' + + # general info + name = fields.Char('Step name', required=True, tracking=True, help="Unique name for step please use trigram as postfix for custom step_ids") + domain_filter = fields.Char('Domain filter', tracking=True) + job_type = fields.Selection(TYPES, default='install_odoo', required=True, tracking=True, ondelete={t[0]: 'cascade' for t in [TYPES]}) protected = fields.Boolean('Protected', default=False, tracking=True) default_sequence = fields.Integer('Sequence', default=100, tracking=True) # or run after? # or in many2many rel? step_order_ids = fields.One2many('runbot.build.config.step.order', 'step_id') @@ -265,18 +277,18 @@ class ConfigStep(models.Model): child = build._add_child(child_data, orphan=self.make_orphan) build._log('create_build', 'created with config %s' % create_config.name, log_type='subbuild', path=str(child.id)) + def make_python_ctx(self, build): return { 'self': self, - 'fields': fields, - 'models': models, + # 'fields': fields, + # 'models': models, 'build': build, '_logger': _logger, 'log_path': build._path('logs', '%s.txt' % self.name), 'glob': glob.glob, 'Command': Command, - 're': re, - 'time': time, + 're': ReProxy, 'grep': grep, 'rfind': rfind, 'json_loads': json.loads, diff --git a/runbot/models/build_error.py b/runbot/models/build_error.py index 7b752b13..ee325a16 100644 --- a/runbot/models/build_error.py +++ b/runbot/models/build_error.py @@ -245,7 +245,7 @@ class RunbotTeam(models.Model): name = fields.Char('Team', required=True) user_ids = fields.Many2many('res.users', string='Team Members', domain=[('share', '=', False)]) - dashboard_id = fields.Many2one('runbot.dashboard', String='Dashboard') + dashboard_id = fields.Many2one('runbot.dashboard',string='Dashboard') build_error_ids = fields.One2many('runbot.build.error', 'team_id', string='Team Errors') path_glob = fields.Char('Module Wildcards', help='Comma separated list of `fnmatch` wildcards used to assign errors automaticaly\n' @@ -300,7 +300,7 @@ class RunbotDashboardTile(models.Model): trigger_id = fields.Many2one('runbot.trigger', 'Trigger', help='Trigger to monitor in chosen category') config_id = fields.Many2one('runbot.build.config', 'Config', help='Select a sub_build with this config') domain_filter = fields.Char('Domain Filter', help='If present, will be applied on builds', default="[('global_result', '=', 'ko')]") - custom_template_id = fields.Many2one('ir.ui.view', help='Change for a custom Dasbord card template', + custom_template_id = fields.Many2one('ir.ui.view', help='Change for a custom Dashboard card template', domain=[('type', '=', 'qweb')], default=lambda self: self.env.ref('runbot.default_dashboard_tile_view')) sticky_bundle_ids = fields.Many2many('runbot.bundle', compute='_compute_sticky_bundle_ids', string='Sticky Bundles') build_ids = fields.Many2many('runbot.build', compute='_compute_build_ids', string='Builds') diff --git a/runbot/models/build_stat.py b/runbot/models/build_stat.py index 05ae5f5a..c74da1b7 100644 --- a/runbot/models/build_stat.py +++ b/runbot/models/build_stat.py @@ -20,7 +20,7 @@ class BuildStat(models.Model): build_id = fields.Many2one("runbot.build", "Build", index=True, ondelete="cascade") config_step_id = fields.Many2one( - "runbot.build.config.step", "Step", index=True, ondelete="cascade" + "runbot.build.config.step", "Step", ondelete="cascade" ) key = fields.Char("key", index=True) value = fields.Float("Value") @@ -61,7 +61,7 @@ class RunbotBuildStatSql(models.Model): config_step_id = fields.Many2one( "runbot.build.config.step", string="Config Step", readonly=True ) - config_step_name = fields.Char(String="Config Step name", readonly=True) + config_step_name = fields.Char(string="Config Step name", readonly=True) build_id = fields.Many2one("runbot.build", string="Build", readonly=True) build_config_id = fields.Many2one("runbot.build.config", string="Config", readonly=True) diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index f25cccc8..9ec47a47 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -54,8 +54,9 @@ class Bundle(models.Model): def _compute_host_id(self): assigned_only = None runbots = {} - for bundle in self.filtered('name'): - elems = bundle.name.split('-') + for bundle in self: + bundle.host_id = False + elems = (bundle.name or '').split('-') for elem in elems: if elem.startswith('runbot'): if elem.replace('runbot', '') == '_x': @@ -126,8 +127,8 @@ class Bundle(models.Model): @api.depends_context('category_id') def _compute_last_batchs(self): - if self: - batch_ids = defaultdict(list) + batch_ids = defaultdict(list) + if self.ids: category_id = self.env.context.get('category_id', self.env['ir.model.data'].xmlid_to_res_id('runbot.default_category')) self.env.cr.execute(""" SELECT @@ -151,8 +152,8 @@ class Bundle(models.Model): for batch in batchs: batch_ids[batch.bundle_id.id].append(batch.id) - for bundle in self: - bundle.last_batchs = [(6, 0, batch_ids[bundle.id])] + for bundle in self: + bundle.last_batchs = [(6, 0, batch_ids[bundle.id])] if bundle.id in batch_ids else False @api.depends_context('category_id') def _compute_last_done_batch(self): diff --git a/runbot/models/commit.py b/runbot/models/commit.py index e2e3d0d4..8dd56f8b 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -179,7 +179,7 @@ class CommitStatus(models.Model): commit_id = fields.Many2one('runbot.commit', string='Commit', required=True, index=True) context = fields.Char('Context', required=True) - state = fields.Char('State', required=True) + state = fields.Char('State', required=True, copy=True) build_id = fields.Many2one('runbot.build', string='Build', index=True) target_url = fields.Char('Url') description = fields.Char('Description') @@ -226,7 +226,7 @@ class CommitStatus(models.Model): _logger.exception('Something went wrong sending notification for %s', commit_name) if post_commit: - self._cr.after('commit', send_github_status_async) + self._cr.postcommit.add(send_github_status_async) else: send_github_status(self.env) diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py index 7c8b8eae..bb05fc91 100644 --- a/runbot/models/custom_trigger.py +++ b/runbot/models/custom_trigger.py @@ -7,7 +7,7 @@ class BundleTriggerCustomization(models.Model): _name = 'runbot.bundle.trigger.custom' _description = 'Custom trigger' - trigger_id = fields.Many2one('runbot.trigger', domain="[('project_id', '=', bundle_id.project_id)]") + trigger_id = fields.Many2one('runbot.trigger') bundle_id = fields.Many2one('runbot.bundle') config_id = fields.Many2one('runbot.build.config') extra_params = fields.Char("Custom parameters") diff --git a/runbot/models/database.py b/runbot/models/database.py index 87ff69a2..98948f1a 100644 --- a/runbot/models/database.py +++ b/runbot/models/database.py @@ -7,7 +7,7 @@ class Database(models.Model): _name = 'runbot.database' _description = "Database" - name = fields.Char('Host name', required=True, unique=True) + name = fields.Char('Host name', required=True) build_id = fields.Many2one('runbot.build', index=True, required=True) db_suffix = fields.Char(compute='_compute_db_suffix') diff --git a/runbot/models/dockerfile.py b/runbot/models/dockerfile.py index dc9fe0e5..a3797f1b 100644 --- a/runbot/models/dockerfile.py +++ b/runbot/models/dockerfile.py @@ -37,7 +37,7 @@ class Dockerfile(models.Model): def _compute_dockerfile(self): for rec in self: try: - res = rec.template_id.render().decode() if rec.template_id else '' + res = rec.template_id._render().decode() if rec.template_id else '' rec.dockerfile = re.sub(r'^\s*$', '', res, flags=re.M).strip() except QWebException: rec.dockerfile = '' diff --git a/runbot/models/event.py b/runbot/models/event.py index 38b3facd..01269e32 100644 --- a/runbot/models/event.py +++ b/runbot/models/event.py @@ -20,8 +20,10 @@ class runbot_event(models.Model): build_id = fields.Many2one('runbot.build', 'Build', index=True, ondelete='cascade') active_step_id = fields.Many2one('runbot.build.config.step', 'Active step', index=True) - type = fields.Selection(selection_add=TYPES, string='Type', required=True, index=True) + type = fields.Selection(selection_add=TYPES, string='Type', required=True, index=True, ondelete={t[0]: 'cascade' for t in TYPES}) error_id = fields.Many2one('runbot.build.error', compute='_compute_known_error') # remember to never store this field + dbname = fields.Char(string='Database Name', index=False) + def init(self): parent_class = super(runbot_event, self) @@ -110,7 +112,7 @@ class RunbotErrorLog(models.Model): path = fields.Char(string='Path', readonly=True) line = fields.Char(string='Line', readonly=True) build_id = fields.Many2one('runbot.build', string='Build', readonly=True) - dest = fields.Char(String='Build dest', readonly=True) + dest = fields.Char(string='Build dest', readonly=True) local_state = fields.Char(string='Local state', readonly=True) local_result = fields.Char(string='Local result', readonly=True) global_state = fields.Char(string='Global state', readonly=True) diff --git a/runbot/models/host.py b/runbot/models/host.py index c159fa6d..833fafda 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -13,7 +13,7 @@ class Host(models.Model): _order = 'id' _inherit = 'mail.thread' - name = fields.Char('Host name', required=True, unique=True) + name = fields.Char('Host name', required=True) disp_name = fields.Char('Display name') active = fields.Boolean('Active', default=True, tracking=True) last_start_loop = fields.Datetime('Last start') diff --git a/runbot/models/project.py b/runbot/models/project.py index ef06bd86..5ac3d7ac 100644 --- a/runbot/models/project.py +++ b/runbot/models/project.py @@ -5,7 +5,7 @@ class Project(models.Model): _name = 'runbot.project' _description = 'Project' - name = fields.Char('Project name', required=True, unique=True) + name = fields.Char('Project name', required=True) group_ids = fields.Many2many('res.groups', string='Required groups') keep_sticky_running = fields.Boolean('Keep last sticky builds running') trigger_ids = fields.One2many('runbot.trigger', 'project_id', string='Triggers') diff --git a/runbot/models/repo.py b/runbot/models/repo.py index acc746de..54e71cdf 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -147,12 +147,12 @@ class Remote(models.Model): remote = super().create(values_list) if not remote.repo_id.main_remote_id: remote.repo_id.main_remote_id = remote - remote._cr.after('commit', remote.repo_id._update_git_config) + remote._cr.postcommit.add(remote.repo_id._update_git_config) return remote def write(self, values): res = super().write(values) - self._cr.after('commit', self.repo_id._update_git_config) + self._cr.postcommit.add(self.repo_id._update_git_config) return res def _github(self, url, payload=None, ignore_errors=False, nb_tries=2, recursive=False): @@ -217,7 +217,7 @@ class Repo(models.Model): _order = 'sequence, id' _inherit = 'mail.thread' - name = fields.Char("Name", unique=True, tracking=True) # odoo/enterprise/upgrade/security/runbot/design_theme + name = fields.Char("Name", tracking=True) # odoo/enterprise/upgrade/security/runbot/design_theme identity_file = fields.Char("Identity File", help="Identity file to use with git/ssh", groups="runbot.group_runbot_admin") main_remote_id = fields.Many2one('runbot.remote', "Main remote", tracking=True) remote_ids = fields.One2many('runbot.remote', 'repo_id', "Remotes") @@ -478,7 +478,7 @@ class Repo(models.Model): if os.path.isdir(os.path.join(repo.path, 'refs')): git_config_path = os.path.join(repo.path, 'config') template_params = {'repo': repo} - git_config = self.env['ir.ui.view'].render_template("runbot.git_config", template_params) + git_config = self.env['ir.ui.view']._render_template("runbot.git_config", template_params) with open(git_config_path, 'wb') as config_file: config_file.write(git_config) _logger.info('Config updated for repo %s' % repo.name) diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index c6430fad..bb71a08e 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -122,9 +122,9 @@ class Runbot(models.AbstractModel): if domain: non_allocated_domain = expression.AND([non_allocated_domain, domain]) e = expression.expression(non_allocated_domain, self.env['runbot.build']) - assert e.get_tables() == ['"runbot_build"'] - where_clause, where_params = e.to_sql() - + query = e.query + query.order = '"runbot_build".parent_path' + select_query, select_params = query.select() # self-assign to be sure that another runbot batch cannot self assign the same builds query = """UPDATE runbot_build @@ -132,17 +132,12 @@ class Runbot(models.AbstractModel): host = %%s WHERE runbot_build.id IN ( - SELECT runbot_build.id - FROM runbot_build - WHERE - %s - ORDER BY - parent_path + %s FOR UPDATE OF runbot_build SKIP LOCKED LIMIT %%s ) - RETURNING id""" % where_clause - self.env.cr.execute(query, [host.name] + where_params + [nb_slots]) + RETURNING id""" % select_query + self.env.cr.execute(query, [host.name] + select_params + [nb_slots]) return self.env.cr.fetchall() def _domain(self): @@ -165,7 +160,7 @@ class Runbot(models.AbstractModel): if nginx: settings['builds'] = env['runbot.build'].search([('local_state', '=', 'running'), ('host', '=', fqdn())]) - nginx_config = env['ir.ui.view'].render_template("runbot.nginx_config", settings) + nginx_config = env['ir.ui.view']._render_template("runbot.nginx_config", settings) os.makedirs(nginx_dir, exist_ok=True) content = None nginx_conf_path = os.path.join(nginx_dir, 'nginx.conf') diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index e906e7b5..cbfc0f25 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -116,3 +116,7 @@ access_runbot_codeowner_admin,runbot_codeowner_admin,runbot.model_runbot_codeown access_runbot_codeowner_user,runbot_codeowner_user,runbot.model_runbot_codeowner,group_user,1,0,0,0 access_runbot_commit_export_admin,runbot_commit_export_admin,runbot.model_runbot_commit_export,runbot.group_runbot_admin,1,1,1,1 + +access_runbot_trigger_custom_wizard,access_runbot_trigger_custom_wizard,model_runbot_trigger_custom_wizard,runbot.group_runbot_admin,1,1,1,1 + +access_runbot_build_stat_regex_wizard,access_runbot_build_stat_regex_wizard,model_runbot_build_stat_regex_wizard,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/templates/frontend.xml b/runbot/templates/frontend.xml index 4d6bd6cd..9e5b7556 100644 --- a/runbot/templates/frontend.xml +++ b/runbot/templates/frontend.xml @@ -42,7 +42,7 @@ - +