From a81591156d9f7474e7261824023959beee25c664 Mon Sep 17 00:00:00 2001 From: William Braeckman Date: Fri, 14 Mar 2025 17:26:26 +0100 Subject: [PATCH] [WIP] public api implementation --- runbot/controllers/__init__.py | 1 + runbot/controllers/public_api.py | 79 +++++++++++++ runbot/models/batch.py | 30 +++-- runbot/models/branch.py | 13 +- runbot/models/build.py | 54 +++++---- runbot/models/bundle.py | 56 +++++++-- runbot/models/commit.py | 20 +++- runbot/models/database.py | 7 +- runbot/models/host.py | 8 +- runbot/models/project.py | 9 +- runbot/models/public_model_mixin.py | 176 ++++++++++++++++++++++++++-- runbot/models/repo.py | 24 ++-- runbot/models/upgrade.py | 2 +- runbot/security/ir.rule.csv | 2 + runbot/tests/__init__.py | 1 + runbot/tests/test_public_api.py | 89 ++++++++++++++ runbot_test/models.py | 8 ++ runbot_test/tests/test_schema.py | 1 - 18 files changed, 501 insertions(+), 79 deletions(-) create mode 100644 runbot/controllers/public_api.py create mode 100644 runbot/tests/test_public_api.py diff --git a/runbot/controllers/__init__.py b/runbot/controllers/__init__.py index 96d149ab..a3adbc61 100644 --- a/runbot/controllers/__init__.py +++ b/runbot/controllers/__init__.py @@ -3,3 +3,4 @@ from . import frontend from . import hook from . import badge +from . import public_api diff --git a/runbot/controllers/public_api.py b/runbot/controllers/public_api.py new file mode 100644 index 00000000..03575702 --- /dev/null +++ b/runbot/controllers/public_api.py @@ -0,0 +1,79 @@ +import json + +from werkzeug.exceptions import BadRequest, Forbidden + +from odoo.exceptions import AccessError +from odoo.http import Controller, request, route +from odoo.tools import mute_logger + +from odoo.addons.runbot.models.public_model_mixin import PublicModelMixin + + +class PublicApi(Controller): + + @mute_logger('odoo.addons.base.models.ir_model') # We don't care about logging acl errors + def _get_model(self, model: str) -> PublicModelMixin: + """ + Returns the model from a model string. + + Raises the appropriate exception if: + - The model does not exist + - The model is not a public model + - The current user can not read the model + """ + pool = request.env.registry + try: + Model = pool[model] + except KeyError: + raise BadRequest('Unknown model') + if not issubclass(Model, pool['runbot.public.model.mixin']): + raise BadRequest('Unknown model') + Model = request.env[model] + Model.check_access('read') + if not Model._allow_direct_access(): + raise Forbidden('This model does not allow direct access') + return Model + + @route('/runbot/api/models', auth='public', methods=['GET'], readonly=True) + def models(self): + models = [] + for model in request.env.keys(): + try: + models.append(self._get_model(model)) + except (BadRequest, AccessError, Forbidden): + pass + return request.make_json_response( + [Model._name for Model in models] + ) + + @route('/runbot/api//read', auth='public', methods=['POST'], readonly=True, csrf=False) + def read(self, *, model: str): + Model = self._get_model(model) + required_keys = Model._get_request_required_keys() + allowed_keys = Model._get_request_allowed_keys() + try: + data = request.get_json_data() + except json.JSONDecodeError: + raise BadRequest('Invalid payload, missing or malformed json') + if not isinstance(data, dict): + raise BadRequest('Invalid payload, should be a dict.') + if (missing_keys := required_keys - set(data.keys())): + raise BadRequest(f'Invalid payload, missing keys: {", ".join(missing_keys)}') + if (unknown_keys := set(data.keys()) - allowed_keys): + raise BadRequest(f'Invalid payload, unknown keys: {", ".join(unknown_keys)}') + if Model._public_requires_project(): + if not isinstance(data['project_id'], int): + raise BadRequest('Invalid project_id, should be an int') + # This is an additional layer of protection for project_id + project = request.env['runbot.project'].browse(data['project_id']).exists() + if not project: + raise BadRequest('Unknown project_id') + project.check_access('read') + Model = Model.with_context(project_id=project.id) + return request.make_json_response(Model._process_read_request(data)) + + @route('/runbot/api//schema', auth='public', methods=['GET'], readonly=True) + def schema(self, *, model: str): + return request.make_json_response( + self._get_model(model)._get_public_schema() + ) diff --git a/runbot/models/batch.py b/runbot/models/batch.py index 1469d9e0..2ea32ac6 100644 --- a/runbot/models/batch.py +++ b/runbot/models/batch.py @@ -12,16 +12,17 @@ _logger = logging.getLogger(__name__) class Batch(models.Model): _name = 'runbot.batch' _description = "Bundle batch" + _inherit = ['runbot.public.model.mixin'] last_update = fields.Datetime('Last ref update') bundle_id = fields.Many2one('runbot.bundle', required=True, index=True, ondelete='cascade') - commit_link_ids = fields.Many2many('runbot.commit.link') + commit_link_ids = fields.Many2many('runbot.commit.link', public=True) commit_ids = fields.Many2many('runbot.commit', compute='_compute_commit_ids') - slot_ids = fields.One2many('runbot.batch.slot', 'batch_id') + slot_ids = fields.One2many('runbot.batch.slot', 'batch_id', public=True) all_build_ids = fields.Many2many('runbot.build', compute='_compute_all_build_ids', help="Recursive builds") - state = fields.Selection([('preparing', 'Preparing'), ('ready', 'Ready'), ('done', 'Done'), ('skipped', 'Skipped')]) + state = fields.Selection([('preparing', 'Preparing'), ('ready', 'Ready'), ('done', 'Done'), ('skipped', 'Skipped')], public=True) hidden = fields.Boolean('Hidden', default=False) - age = fields.Integer(compute='_compute_age', string='Build age') + age = fields.Integer(compute='_compute_age', string='Build age', public=True) category_id = fields.Many2one('runbot.category', index=True, default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False)) log_ids = fields.One2many('runbot.batch.log', 'batch_id') has_warning = fields.Boolean("Has warning") @@ -34,6 +35,10 @@ class Batch(models.Model): column2='referenced_batch_id', ) + @api.model + def _project_id_field_path(self): + return 'bundle_id.project_id' + @api.depends('slot_ids.build_id') def _compute_all_build_ids(self): all_builds = self.env['runbot.build'].search([('id', 'child_of', self.slot_ids.build_id.ids)]) @@ -522,20 +527,25 @@ class BatchSlot(models.Model): _name = 'runbot.batch.slot' _description = 'Link between a bundle batch and a build' _order = 'trigger_id,id' + _inherit = ['runbot.public.model.mixin'] - batch_id = fields.Many2one('runbot.batch', index=True) - trigger_id = fields.Many2one('runbot.trigger', index=True) - build_id = fields.Many2one('runbot.build', index=True) - all_build_ids = fields.Many2many('runbot.build', compute='_compute_all_build_ids') + batch_id = fields.Many2one('runbot.batch', index=True, public=True) + trigger_id = fields.Many2one('runbot.trigger', index=True, public=True) + build_id = fields.Many2one('runbot.build', index=True, public=True) + all_build_ids = fields.Many2many('runbot.build', compute='_compute_all_build_ids', public=True) params_id = fields.Many2one('runbot.build.params', index=True, required=True) - link_type = fields.Selection([('created', 'Build created'), ('matched', 'Existing build matched'), ('rebuild', 'Rebuild')], required=True) # rebuild type? - active = fields.Boolean('Attached', default=True) + link_type = fields.Selection([('created', 'Build created'), ('matched', 'Existing build matched'), ('rebuild', 'Rebuild')], required=True, public=True) # rebuild type? + active = fields.Boolean('Attached', default=True, public=True) skipped = fields.Boolean('Skipped', default=False) # rebuild, what to do: since build can be in multiple batch: # - replace for all batch? # - only available on batch and replace for batch only? # - create a new bundle batch will new linked build? + @api.model + def _allow_direct_access(self): + return False + @api.depends('build_id') def _compute_all_build_ids(self): all_builds = self.env['runbot.build'].search([('id', 'child_of', self.build_id.ids)]) diff --git a/runbot/models/branch.py b/runbot/models/branch.py index b5aeeedd..02db237d 100644 --- a/runbot/models/branch.py +++ b/runbot/models/branch.py @@ -13,10 +13,11 @@ class Branch(models.Model): _description = "Branch" _order = 'name' _rec_name = 'dname' + _inherit = ['runbot.public.model.mixin'] _sql_constraints = [('branch_repo_uniq', 'unique (name,remote_id)', 'The branch must be unique per repository !')] - name = fields.Char('Name', required=True) + name = fields.Char('Name', required=True, public=True) remote_id = fields.Many2one('runbot.remote', 'Remote', required=True, ondelete='cascade', index=True) head = fields.Many2one('runbot.commit', 'Head Commit', index=True) @@ -25,7 +26,7 @@ class Branch(models.Model): reference_name = fields.Char(compute='_compute_reference_name', string='Bundle name', store=True) bundle_id = fields.Many2one('runbot.bundle', 'Bundle', ondelete='cascade', index=True) - is_pr = fields.Boolean('IS a pr', required=True) + is_pr = fields.Boolean('IS a pr', required=True, public=True) pr_title = fields.Char('Pr Title') pr_body = fields.Char('Pr Body') pr_author = fields.Char('Pr Author') @@ -37,12 +38,16 @@ class Branch(models.Model): reflog_ids = fields.One2many('runbot.ref.log', 'branch_id') - branch_url = fields.Char(compute='_compute_branch_url', string='Branch url', readonly=True) - dname = fields.Char('Display name', compute='_compute_dname', search='_search_dname') + branch_url = fields.Char(compute='_compute_branch_url', string='Branch url', readonly=True, public=True) + dname = fields.Char('Display name', compute='_compute_dname', search='_search_dname', public=True) alive = fields.Boolean('Alive', default=True) draft = fields.Boolean('Draft', store=True) + @api.model + def _project_id_field_path(self): + return 'bundle_id.project_id' + @api.depends('name', 'remote_id.short_name') def _compute_dname(self): for branch in self: diff --git a/runbot/models/build.py b/runbot/models/build.py index 6815e8ca..432c724f 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -49,6 +49,7 @@ def make_selection(array): class BuildParameters(models.Model): _name = 'runbot.build.params' _description = "All information used by a build to run, should be unique and set on create only" + _inherit = ['runbot.public.model.mixin'] # on param or on build? # execution parametter @@ -56,17 +57,17 @@ class BuildParameters(models.Model): commit_ids = fields.Many2many('runbot.commit', compute='_compute_commit_ids') version_id = fields.Many2one('runbot.version', required=True, index=True) project_id = fields.Many2one('runbot.project', required=True, index=True) # for access rights - trigger_id = fields.Many2one('runbot.trigger', index=True) # for access rights - create_batch_id = fields.Many2one('runbot.batch', index=True) - category = fields.Char('Category', index=True) # normal vs nightly vs weekly, ... + trigger_id = fields.Many2one('runbot.trigger', index=True, public=True) # for access rights + create_batch_id = fields.Many2one('runbot.batch', index=True, public=True) + category = fields.Char('Category', index=True, public=True) # normal vs nightly vs weekly, ... dockerfile_id = fields.Many2one('runbot.dockerfile', index=True, default=lambda self: self.env.ref('runbot.docker_default', raise_if_not_found=False)) skip_requirements = fields.Boolean('Skip requirements.txt auto install') # other informations extra_params = fields.Char('Extra cmd args') - config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True, + config_id = fields.Many2one('runbot.build.config', 'Run Config', required=True, public=True, default=lambda self: self.env.ref('runbot.runbot_build_config_default', raise_if_not_found=False), index=True) - config_data = JsonDictField('Config Data') - used_custom_trigger = fields.Boolean('Custom trigger was used to generate this build') + config_data = JsonDictField('Config Data', public=True) + used_custom_trigger = fields.Boolean('Custom trigger was used to generate this build', public=True) build_ids = fields.One2many('runbot.build', 'params_id') builds_reference_ids = fields.Many2many('runbot.build', relation='runbot_build_params_references', copy=True) @@ -84,6 +85,10 @@ class BuildParameters(models.Model): ('unique_fingerprint', 'unique (fingerprint)', 'avoid duplicate params'), ] + @api.model + def _allow_direct_access(self): + return False + # @api.depends('version_id', 'project_id', 'extra_params', 'config_id', 'config_data', 'modules', 'commit_link_ids', 'builds_reference_ids') def _compute_fingerprint(self): for param in self: @@ -141,6 +146,7 @@ class BuildResult(models.Model): _name = 'runbot.build' _description = "Build" + _inherit = ['runbot.public.model.mixin'] _parent_store = True _order = 'id desc' @@ -154,26 +160,26 @@ class BuildResult(models.Model): no_auto_run = fields.Boolean('No run') # could be a default value, but possible to change it to allow duplicate accros branches - description = fields.Char('Description', help='Informative description') - md_description = fields.Html(compute='_compute_md_description', string='MD Parsed Description', help='Informative description markdown parsed', sanitize=False) - display_name = fields.Char(compute='_compute_display_name') + description = fields.Char('Description', help='Informative description', public=True) + md_description = fields.Html(compute='_compute_md_description', string='MD Parsed Description', help='Informative description markdown parsed', sanitize=False, public=True) + display_name = fields.Char(compute='_compute_display_name', public=True) # Related fields for convenience - version_id = fields.Many2one('runbot.version', related='params_id.version_id', store=True, index=True) - config_id = fields.Many2one('runbot.build.config', related='params_id.config_id', store=True, index=True) - trigger_id = fields.Many2one('runbot.trigger', related='params_id.trigger_id', store=True, index=True) - create_batch_id = fields.Many2one('runbot.batch', related='params_id.create_batch_id', store=True, index=True) + version_id = fields.Many2one('runbot.version', related='params_id.version_id', store=True, index=True, public=True) + config_id = fields.Many2one('runbot.build.config', related='params_id.config_id', store=True, index=True, public=True) + trigger_id = fields.Many2one('runbot.trigger', related='params_id.trigger_id', store=True, index=True, public=True) + create_batch_id = fields.Many2one('runbot.batch', related='params_id.create_batch_id', store=True, index=True, public=True) # state machine - global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True, recursive=True) - local_state = fields.Selection(make_selection(state_order), string='Build Status', default='pending', required=True, index=True) - global_result = fields.Selection(make_selection(result_order), string='Result', compute='_compute_global_result', store=True, recursive=True) - local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok') + global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True, recursive=True, public=True) + local_state = fields.Selection(make_selection(state_order), string='Build Status', default='pending', required=True, index=True, public=True) + global_result = fields.Selection(make_selection(result_order), string='Result', compute='_compute_global_result', store=True, recursive=True, public=True) + local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok', public=True) - requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True) + requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True, public=True) # web infos - host = fields.Char('Host name') - host_id = fields.Many2one('runbot.host', string="Host", compute='_compute_host_id') + host = fields.Char('Host name', public=True) + host_id = fields.Many2one('runbot.host', string="Host", compute='_compute_host_id', public=True) keep_host = fields.Boolean('Keep host on rebuild and for children') port = fields.Integer('Port') @@ -183,7 +189,7 @@ class BuildResult(models.Model): log_ids = fields.One2many('ir.logging', 'build_id', string='Logs') error_log_ids = fields.One2many('ir.logging', 'build_id', domain=[('level', 'in', ['WARNING', 'ERROR', 'CRITICAL'])], string='Error Logs') stat_ids = fields.One2many('runbot.build.stat', 'build_id', string='Statistics values') - log_list = fields.Char('Comma separted list of step_ids names with logs') + log_list = fields.Char('Comma separted list of step_ids names with logs', public=True) active_step = fields.Many2one('runbot.build.config.step', 'Active step') job = fields.Char('Active step display name', compute='_compute_job') @@ -234,13 +240,17 @@ class BuildResult(models.Model): slot_ids = fields.One2many('runbot.batch.slot', 'build_id') killable = fields.Boolean('Killable') - database_ids = fields.One2many('runbot.database', 'build_id') + database_ids = fields.One2many('runbot.database', 'build_id', public=True) commit_export_ids = fields.One2many('runbot.commit.export', 'build_id') static_run = fields.Char('Static run URL') access_token = fields.Char('Token', default=lambda self: uuid.uuid4().hex) + @api.model + def _project_id_field_path(self): + return 'params_id.project_id' + @api.depends('description', 'params_id.config_id') def _compute_display_name(self): for build in self: diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index 9cab8493..ea8b7000 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -1,3 +1,5 @@ +from werkzeug.exceptions import BadRequest + import time import logging import datetime @@ -5,40 +7,41 @@ import subprocess from collections import defaultdict from odoo import models, fields, api, tools +from odoo.osv import expression from ..common import dt2time, s2human_long class Bundle(models.Model): _name = 'runbot.bundle' _description = "Bundle" - _inherit = 'mail.thread' + _inherit = ['mail.thread', 'runbot.public.model.mixin'] - name = fields.Char('Bundle name', required=True, help="Name of the base branch") - project_id = fields.Many2one('runbot.project', required=True, index=True) - branch_ids = fields.One2many('runbot.branch', 'bundle_id') + name = fields.Char('Bundle name', required=True, help="Name of the base branch", public=True) + project_id = fields.Many2one('runbot.project', required=True, index=True, public=True) + branch_ids = fields.One2many('runbot.branch', 'bundle_id', public=True) # custom behaviour - no_build = fields.Boolean('No build') + no_build = fields.Boolean('No build', public=True) no_auto_run = fields.Boolean('No run') build_all = fields.Boolean('Force all triggers') always_use_foreign = fields.Boolean('Use foreign bundle', help='By default, check for the same bundle name in another project to fill missing commits.', default=lambda self: self.project_id.always_use_foreign) modules = fields.Char("Modules to install", help="Comma-separated list of modules to install and test.") batch_ids = fields.One2many('runbot.batch', 'bundle_id') - last_batch = fields.Many2one('runbot.batch', index=True, domain=lambda self: [('category_id', '=', self.env.ref('runbot.default_category').id)]) - last_batchs = fields.Many2many('runbot.batch', 'Last batchs', compute='_compute_last_batchs') + last_batch = fields.Many2one('runbot.batch', index=True, domain=lambda self: [('category_id', '=', self.env.ref('runbot.default_category').id)], public=True) + last_batchs = fields.Many2many('runbot.batch', 'Last batchs', compute='_compute_last_batchs', public=True) last_done_batch = fields.Many2many('runbot.batch', 'Last batchs', compute='_compute_last_done_batch') - sticky = fields.Boolean('Sticky', compute='_compute_sticky', store=True, index=True) - is_base = fields.Boolean('Is base', index=True) + sticky = fields.Boolean('Sticky', compute='_compute_sticky', store=True, index=True, public=True) + is_base = fields.Boolean('Is base', index=True, public=True) defined_base_id = fields.Many2one('runbot.bundle', 'Forced base bundle', domain="[('project_id', '=', project_id), ('is_base', '=', True)]") base_id = fields.Many2one('runbot.bundle', 'Base bundle', compute='_compute_base_id', store=True) to_upgrade = fields.Boolean('To upgrade To', compute='_compute_to_upgrade', store=True, index=False) to_upgrade_from = fields.Boolean('To upgrade From', compute='_compute_to_upgrade_from', store=True, index=False) - has_pr = fields.Boolean('Has PR', compute='_compute_has_pr', store=True) + has_pr = fields.Boolean('Has PR', compute='_compute_has_pr', store=True, public=True) - version_id = fields.Many2one('runbot.version', 'Version', compute='_compute_version_id', store=True, recursive=True) + version_id = fields.Many2one('runbot.version', 'Version', compute='_compute_version_id', store=True, recursive=True, public=True) version_number = fields.Char(related='version_id.number', store=True, index=True) previous_major_version_base_id = fields.Many2one('runbot.bundle', 'Previous base bundle', compute='_compute_relations_base_id') @@ -56,7 +59,36 @@ class Bundle(models.Model): disable_codeowner = fields.Boolean("Disable codeowners", tracking=True) # extra_info - for_next_freeze = fields.Boolean('Should be in next freeze') + for_next_freeze = fields.Boolean('Should be in next freeze', public=True) + + @api.model + def _get_request_allowed_keys(self): + return super()._get_request_allowed_keys() | {'category_id'} + + @api.model + def _project_id_field_path(self): + return 'project_id' + + @api.model + def _process_read_request_get_records(self, request_data): + if 'category_id' in request_data: + if not isinstance(request_data['category_id'], int): + raise BadRequest('Invalid category_id') + category_id = request_data['category_id'] + else: + category_id = self.env['ir.model.data']._xmlid_to_res_id('runbot.default_category') + self = self.with_context(category_id=category_id) + limit, offset = self._process_read_request_get_offset_limit(request_data) + e = expression.expression(request_data['domain'], self) + query = e.query + query.order = """ + (case when "runbot_bundle".sticky then 1 when "runbot_bundle".sticky is null then 2 else 2 end), + case when "runbot_bundle".sticky then "runbot_bundle".version_number end collate "C" desc, + "runbot_bundle".last_batch desc + """ + query.limit = limit + query.offset = offset + return self.browse(query) @api.depends('name') def _compute_host_id(self): diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 29072028..e1c96e18 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -16,6 +16,7 @@ _logger = logging.getLogger(__name__) class Commit(models.Model): _name = 'runbot.commit' _description = "Commit" + _inherit = ['runbot.public.model.mixin'] _sql_constraints = [ ( @@ -24,7 +25,7 @@ class Commit(models.Model): "Commit must be unique to ensure correct duplicate matching", ) ] - name = fields.Char('SHA') + name = fields.Char('SHA', public=True) tree_hash = fields.Char('Tree hash', readonly=True) repo_id = fields.Many2one('runbot.repo', string='Repo group') date = fields.Datetime('Commit date') @@ -32,10 +33,14 @@ class Commit(models.Model): author_email = fields.Char('Author Email') committer = fields.Char('Committer') committer_email = fields.Char('Committer Email') - subject = fields.Text('Subject') - dname = fields.Char('Display name', compute='_compute_dname') + subject = fields.Text('Subject', public=True) + dname = fields.Char('Display name', compute='_compute_dname', public=True) rebase_on_id = fields.Many2one('runbot.commit', 'Rebase on commit') + @api.model + def _project_id_field_path(self): + return 'repo_id.project_id' + @api.model_create_multi def create(self, vals_list): for vals in vals_list: @@ -194,10 +199,11 @@ class Commit(models.Model): class CommitLink(models.Model): _name = 'runbot.commit.link' _description = "Build commit" + _inherit = ['runbot.public.model.mixin'] - commit_id = fields.Many2one('runbot.commit', 'Commit', required=True, index=True) + commit_id = fields.Many2one('runbot.commit', 'Commit', required=True, index=True, public=True) # Link info - match_type = fields.Selection([('new', 'New head of branch'), ('head', 'Head of branch'), ('base_head', 'Found on base branch'), ('base_match', 'Found on base branch')]) # HEAD, DEFAULT + match_type = fields.Selection([('new', 'New head of branch'), ('head', 'Head of branch'), ('base_head', 'Found on base branch'), ('base_match', 'Found on base branch')], public=True) # HEAD, DEFAULT branch_id = fields.Many2one('runbot.branch', string='Found in branch') # Shouldn't be use for anything else than display base_commit_id = fields.Many2one('runbot.commit', 'Base head commit', index=True) @@ -208,6 +214,10 @@ class CommitLink(models.Model): diff_add = fields.Integer('# line added') diff_remove = fields.Integer('# line removed') + @api.model + def _allow_direct_access(self): + return False + class CommitStatus(models.Model): _name = 'runbot.commit.status' diff --git a/runbot/models/database.py b/runbot/models/database.py index 4d7f1016..799f6161 100644 --- a/runbot/models/database.py +++ b/runbot/models/database.py @@ -6,11 +6,16 @@ _logger = logging.getLogger(__name__) class Database(models.Model): _name = 'runbot.database' _description = "Database" + _inherit = ['runbot.public.model.mixin'] - name = fields.Char('Host name', required=True) + name = fields.Char('Host name', required=True, public=True) build_id = fields.Many2one('runbot.build', index=True, required=True) db_suffix = fields.Char(compute='_compute_db_suffix') + @api.model + def _allow_direct_access(self): + return False + def _compute_db_suffix(self): for record in self: record.db_suffix = record.name.replace('%s-' % record.build_id.dest, '') diff --git a/runbot/models/host.py b/runbot/models/host.py index 8db11c3a..e8636ed9 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -15,9 +15,9 @@ class Host(models.Model): _name = 'runbot.host' _description = "Host" _order = 'id' - _inherit = 'mail.thread' + _inherit = ['mail.thread', 'runbot.public.model.mixin'] - name = fields.Char('Host name', required=True) + name = fields.Char('Host name', required=True, public=True) disp_name = fields.Char('Display name') active = fields.Boolean('Active', default=True, tracking=True) last_start_loop = fields.Datetime('Last start') @@ -49,6 +49,10 @@ class Host(models.Model): use_remote_docker_registry = fields.Boolean('Use remote Docker Registry', default=False, help="Use docker registry for pulling images") docker_registry_url = fields.Char('Registry Url', help="Override global registry URL for this host.") + @api.model + def _public_requires_project(self): + return False + def _compute_nb(self): # Array of tuple (host, state, count) groups = self.env['runbot.build']._read_group( diff --git a/runbot/models/project.py b/runbot/models/project.py index 0235dcf9..666a3287 100644 --- a/runbot/models/project.py +++ b/runbot/models/project.py @@ -6,11 +6,12 @@ class Project(models.Model): _name = 'runbot.project' _description = 'Project' _order = 'sequence, id' + _inherit = ['runbot.public.model.mixin'] - name = fields.Char('Project name', required=True) + name = fields.Char('Project name', required=True, public=True) group_ids = fields.Many2many('res.groups', string='Required groups') keep_sticky_running = fields.Boolean('Keep last sticky builds running') - trigger_ids = fields.One2many('runbot.trigger', 'project_id', string='Triggers') + trigger_ids = fields.One2many('runbot.trigger', 'project_id', string='Triggers', public=True) dockerfile_id = fields.Many2one('runbot.dockerfile', index=True, help="Project Default Dockerfile") repo_ids = fields.One2many('runbot.repo', 'project_id', string='Repos') sequence = fields.Integer('Sequence') @@ -25,6 +26,10 @@ class Project(models.Model): active = fields.Boolean("Active", default=True) process_delay = fields.Integer('Process delay', default=60, required=True, help="Delay between a push and a batch starting its process.") + @api.model + def _public_requires_project(self): + return False + @api.constrains('process_delay') def _constraint_process_delay(self): if any(project.process_delay < 0 for project in self): diff --git a/runbot/models/public_model_mixin.py b/runbot/models/public_model_mixin.py index 4e1d2c0f..c0b673ae 100644 --- a/runbot/models/public_model_mixin.py +++ b/runbot/models/public_model_mixin.py @@ -1,8 +1,11 @@ from __future__ import annotations -from typing import Dict, Union, List +from werkzeug.exceptions import BadRequest, Forbidden + +from typing import Dict, Union, List, Self from odoo import models, api, fields, tools +from odoo.osv import expression ResponseSchema = Dict[str, Dict[str, Union[str, 'ResponseSchema']]] @@ -15,8 +18,15 @@ SUPPORTED_FIELD_TYPES = { # Perhaps this should be a list of class instead 'date', 'datetime', 'selection', 'jsonb', 'many2one', 'one2many', 'many2many', } +RELATIONAL_FIELD_TYPES = {'many2one', 'one2many', 'many2many'} SCHEMA_MAX_DEPTH = 10 -SCHEMA_METADATA_FIELDS = {'__type', '__relational', '__help'} +SCHEMA_METADATA_FIELDS = { + '__type', '__help', '__requires_project', + '__default_page_size', '__max_page_size', + '__required_keys', '__allowed_keys' +} +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 60 def _cleaned_schema(schema: RequestSchema) -> RequestSchema | None: if schema: @@ -35,7 +45,9 @@ class PublicModelMixin(models.AbstractModel): def _valid_field_parameter(self, field: fields.Field, name: str): if field.type in SUPPORTED_FIELD_TYPES: return name in ( - 'public', # boolean, whether the field is readable through the public api + # boolean, whether the field is readable through the public api, + # public fields on record on which the user does not have access are not exposed. + 'public', ) or super()._valid_field_parameter(field, name) return super()._valid_field_parameter(field, name) @@ -47,6 +59,135 @@ class PublicModelMixin(models.AbstractModel): if getattr(field, 'public', None) or field.name == 'id' ] + ########## REQUESTS ########## + + @api.model + def _allow_direct_access(self) -> bool: + """ Returns whether this model is accessible directly through the api. """ + return True + + @api.model + def _get_request_allowed_keys(self) -> set[str]: + """ Returns a list of allowed keys for request_data. """ + return self._get_request_required_keys() | {'page', 'page_size', 'limit', 'offset'} + + @api.model + def _get_request_required_keys(self) -> set[str]: + """ Returns a list of required keys for request_data. """ + required_keys = {'schema', 'domain'} + if self._public_requires_project(): + required_keys.add('project_id') + return required_keys + + @api.model + def _public_requires_project(self) -> bool: #TODO: rename me + """ Public models are by default based on a project_id (filtered on project_id). """ + return self._allow_direct_access() + + @api.model + def _project_id_field_path(self) -> str: + """ Returns the path from the current object to project_id. """ + raise NotImplementedError('_project_id_field_path not implemented') + + @api.model + def _validate_domain(self, domain: list[str | tuple | list]): + """ + Validates a domain against the public schema. + + This only validates that all the fields in the domain are queryable fields, + the actual validity of the domain will be checked by the orm when + searching for records. + + Returns: + domain: a transformed domain if necessary + + Raises: + AssertionError: unknown domain leaf + Forbidden: invalid field used + """ + + try: + self._where_calc(domain) + except ValueError as e: + raise BadRequest('Invalid domain') from e + + if self.env.user.has_group('runbot.group_runbot_admin'): + return # Admins can query all + schema: ResponseSchema = self._get_public_schema() + # recompiles the schema into a list of fields that can be present in the domain + valid_fields: str[str] = set() + def _visit_schema(schema, prefix: str | None = None): + schema = _cleaned_schema(schema) + if not schema: + return + for field, sub_schema in schema.items(): + this_field = f'{prefix}.{field}' if prefix else field + valid_fields.add(this_field) + if sub_schema: + _visit_schema(sub_schema, prefix=this_field) + _visit_schema(schema) + + for leaf in domain: + if not isinstance(leaf, (tuple, list)): + continue + assert len(leaf) == 3 # Can this happen in a valid domain? + if leaf[0] not in valid_fields: + raise Forbidden('Trying to filter from private field') + + if self._public_requires_project(): + assert 'project_id' in self.env.context + domain = expression.AND([ + [(self._project_id_field_path(), '=', self.env.context['project_id'])], + domain + ]) + + return domain + + @api.model + def _process_read_request_get_offset_limit(self, request_data: dict) -> tuple[int, int]: + if 'page_size' in request_data: + if not isinstance(request_data['page_size'], int): + raise BadRequest('Invalid page size (should be int)') + limit = request_data['page_size'] + if limit > self._get_max_page_size(): + raise BadRequest('Page size exceeds max size') + else: + limit = self._get_default_page_size() + offset = 0 + if 'page' in request_data: + if not isinstance(request_data['page'], int): + raise BadRequest('Invalid page (should be int)') + offset = limit * request_data['page'] + return limit, offset + + @api.model + def _process_read_request_get_records(self, request_data: dict) -> Self: + limit, offset = self._process_read_request_get_offset_limit(request_data) + return self.search(request_data['domain'], limit=limit, offset=offset) + + @api.model + def _process_read_request(self, request_data: dict) -> dict|list: + """ + Processes a frontend request and returns the data to be returned by the controller. + + This method is allowed to raise Http specific exceptions. + """ + schema, domain = request_data['schema'], request_data['domain'] + + try: + if not self._verify_schema(schema) and\ + not self.env.user.has_group('runbot.group_runbot_admin'): + raise Forbidden('Invalid schema or trying to access private data.') + except (ValueError, AssertionError) as e: + raise BadRequest('Invalid schema') from e + + request_data['domain'] = self._validate_domain(domain) + records = self._process_read_request_get_records(request_data) + + return records._read_schema(request_data['schema']) + + ########## SCHEMA ########## + @api.model def _get_relation_cache_key(self, field: fields.Field): """ Returns a cache key for a field, this is used to prevent infinite loops.""" @@ -64,23 +205,36 @@ class PublicModelMixin(models.AbstractModel): return CoModel._get_relation_cache_key(inverse_field) raise NotImplementedError('Unsupported field') + @api.model + def _get_default_page_size(self) -> int: + return DEFAULT_PAGE_SIZE + + @api.model + def _get_max_page_size(self) -> int: + return MAX_PAGE_SIZE + @tools.ormcache() @api.model - def _get_public_schema(self) -> RequestSchema: + def _get_public_schema(self) -> ResponseSchema: """ Returns an auto-generated schema according to public fields. """ # We want to prevent infinite loops so we need to track which relations # have already been explored, this concerns many2one, many2many def _visit_model(model: PublicModelMixin, visited_relations: set[str], depth = 0): - schema: RequestSchema = {} + schema: ResponseSchema = {} + if depth == 0: + schema['__requires_project'] = model._public_requires_project() + schema['__default_page_size'] = model._get_default_page_size() + schema['__max_page_size'] = model._get_max_page_size() + schema['__required_keys'] = list(model._get_request_required_keys()) + schema['__allowed_keys'] = list(model._get_request_allowed_keys()) for field in model._get_public_fields(): field_metadata = { '__type': field.type, - '__relational': field.relational, } if field.help: field_metadata['__help'] = field.help if field.relational and \ - PublicModelMixin._name in model.env[field.comodel_name]._inherit: + issubclass(self.pool[field.comodel_name], PublicModelMixin): field_key = model._get_relation_cache_key(field) if field_key in visited_relations or depth == SCHEMA_MAX_DEPTH: continue @@ -120,14 +274,14 @@ class PublicModelMixin(models.AbstractModel): request_schema: RequestSchema, ) -> bool: for field, sub_schema in request_schema.items(): - sub_schema = _cleaned_schema(sub_schema) if field in SCHEMA_METADATA_FIELDS: continue + sub_schema = _cleaned_schema(sub_schema) if field not in model_schema: return False - if sub_schema is None or not sub_schema.get('__relational', False): + if sub_schema is None: continue - if not model_schema[field].get('__relational'): + if model_schema[field].get('__type') not in RELATIONAL_FIELD_TYPES: raise ValueError( 'Invalid sub schema provided for non relational field' ) @@ -166,9 +320,9 @@ class PublicModelMixin(models.AbstractModel): """ Reads the given record according to the schema inline""" data['id'] = record.id for field, sub_schema in request_schema.items(): - sub_schema = _cleaned_schema(sub_schema) if field in SCHEMA_METADATA_FIELDS: # it is authorized to pass the public schema directly continue + sub_schema = _cleaned_schema(sub_schema) _field = record._fields[field] if not _field.relational: data[field] = record._read_schema_read_field(_field) diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 2f68d7eb..89a6ab1f 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -43,14 +43,14 @@ class Trigger(models.Model): """ _name = 'runbot.trigger' - _inherit = 'mail.thread' + _inherit = ['mail.thread', 'runbot.public.model.mixin'] _description = 'Triggers' _order = 'sequence, id' sequence = fields.Integer('Sequence') - name = fields.Char("Name") - description = fields.Char("Description", help="Informative description") + name = fields.Char("Name", public=True) + description = fields.Char("Description", help="Informative description", public=True) project_id = fields.Many2one('runbot.project', string="Project id", required=True) repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]") dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies") @@ -77,8 +77,8 @@ class Trigger(models.Model): ci_context = fields.Char("CI context", tracking=True) category_id = fields.Many2one('runbot.category', default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False)) version_domain = fields.Char(string="Version domain") - hide = fields.Boolean('Hide trigger on main page') - manual = fields.Boolean('Only start trigger manually', default=False) + hide = fields.Boolean('Hide trigger on main page', public=True) + manual = fields.Boolean('Only start trigger manually', default=False, public=True) restore_trigger_id = fields.Many2one('runbot.trigger', string='Restore Trigger ID for custom triggers', help="Mainly usefull to automatically define where to find a reference database when creating a custom trigger", tracking=True) upgrade_dumps_trigger_id = fields.Many2one('runbot.trigger', string='Template/complement trigger', tracking=True) @@ -98,6 +98,10 @@ class Trigger(models.Model): context={'default_type': 'qweb', 'default_arch_base': ''}, ) + @api.model + def _project_id_field_path(self): + return 'project_id' + @api.depends('config_id.step_order_ids.step_id.make_stats') def _compute_has_stats(self): for trigger in self: @@ -200,9 +204,9 @@ class Remote(models.Model): _name = 'runbot.remote' _description = 'Remote' _order = 'sequence, id' - _inherit = 'mail.thread' + _inherit = ['mail.thread', 'runbot.public.model.mixin'] - name = fields.Char('Url', required=True, tracking=True) + name = fields.Char('Url', required=True, tracking=True, public=True) repo_id = fields.Many2one('runbot.repo', required=True, tracking=True) owner = fields.Char(compute='_compute_base_infos', string='Repo Owner', store=True, readonly=True, tracking=True) @@ -211,7 +215,7 @@ class Remote(models.Model): base_url = fields.Char(compute='_compute_base_url', string='Base URL', readonly=True, tracking=True) - short_name = fields.Char('Short name', compute='_compute_short_name', tracking=True) + short_name = fields.Char('Short name', compute='_compute_short_name', tracking=True, public=True) remote_name = fields.Char('Remote name', compute='_compute_remote_name', tracking=True) sequence = fields.Integer('Sequence', tracking=True) @@ -221,6 +225,10 @@ class Remote(models.Model): token = fields.Char("Github token", groups="runbot.group_runbot_admin") + @api.model + def _allow_direct_access(self): + return False + @api.depends('name') def _compute_base_infos(self): for remote in self: diff --git a/runbot/models/upgrade.py b/runbot/models/upgrade.py index 35b1aced..07e3a11b 100644 --- a/runbot/models/upgrade.py +++ b/runbot/models/upgrade.py @@ -57,7 +57,7 @@ class UpgradeRegex(models.Model): class BuildResult(models.Model): - _inherit = 'runbot.build' + _inherit = ['runbot.build'] def _parse_upgrade_errors(self): ir_logs = self.env['ir.logging'].search([('level', 'in', ('ERROR', 'WARNING', 'CRITICAL')), ('type', '=', 'server'), ('build_id', 'in', self.ids)]) diff --git a/runbot/security/ir.rule.csv b/runbot/security/ir.rule.csv index 0e6bd929..fb6c2992 100644 --- a/runbot/security/ir.rule.csv +++ b/runbot/security/ir.rule.csv @@ -12,3 +12,5 @@ rule_commit,"limited to groups",model_runbot_commit,group_user,"['|', ('repo_id. rule_commit_mgmt,"manager can see all",model_runbot_commit,group_runbot_admin,"[(1, '=', 1)]",1,1,1,1 rule_build,"limited to groups",model_runbot_build,group_user,"['|', ('params_id.project_id.group_ids', '=', False), ('params_id.project_id.group_ids', 'in', [g.id for g in user.groups_id])]",1,1,1,1 rule_build_mgmt,"manager can see all",model_runbot_build,group_runbot_admin,"[(1, '=', 1)]",1,1,1,1 +rule_batch,"limited to groups",model_runbot_batch,group_user,"['|', ('bundle_id.project_id.group_ids', '=', False), ('bundle_id.project_id.group_ids', 'in', [g.id for g in user.groups_id])]",1,1,1,1 +rule_batch_mgmt,"manager can see all",model_runbot_batch,group_runbot_admin,"[(1, '=', 1)]",1,1,1,1 diff --git a/runbot/tests/__init__.py b/runbot/tests/__init__.py index e4410f29..85874773 100644 --- a/runbot/tests/__init__.py +++ b/runbot/tests/__init__.py @@ -16,3 +16,4 @@ from . import test_commit from . import test_upgrade from . import test_dockerfile from . import test_host +from . import test_public_api diff --git a/runbot/tests/test_public_api.py b/runbot/tests/test_public_api.py new file mode 100644 index 00000000..3ee0bfd9 --- /dev/null +++ b/runbot/tests/test_public_api.py @@ -0,0 +1,89 @@ +import json + +from odoo.tests.common import HttpCase, tagged + +from odoo.addons.runbot.models.public_model_mixin import PublicModelMixin + + +@tagged('-at_install', 'post_install') +class TestPublicApi(HttpCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.project = cls.env['runbot.project'].create({'name': 'Tests', 'process_delay': 0}) + + def get_public_models(self): + for Model in self.registry.values(): + if not issubclass(Model, PublicModelMixin) or Model._name == 'runbot.public.model.mixin': + continue + yield self.env[Model._name] + + + def test_requires_project_defines_project_id_path(self): + for Model in self.get_public_models(): + if not Model._public_requires_project(): + continue + # Try calling _project_id_field_path, none should fail + with self.subTest(model=Model._name): + try: + Model._project_id_field_path() + except NotImplementedError: + self.fail('_project_id_field_path not implemented') + + def test_direct_access_disabled(self): + DisabledModel = self.env['runbot.commit.link'] + self.assertFalse(DisabledModel._allow_direct_access()) + + resp = self.url_open('/runbot/api/models') + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertNotIn(DisabledModel._name, data) + + resp = self.url_open(f'/runbot/api/{DisabledModel._name}/schema') + self.assertEqual(resp.status_code, 403) + + resp = self.url_open(f'/runbot/api/{DisabledModel._name}/read', data="{}", headers={'Content-Type': 'application/json'}) # model checking happens before data checking + self.assertEqual(resp.status_code, 403) + + def test_api_public_basics(self): + # This serves as a basic read test through the api + Model = self.env['runbot.bundle'] + self.assertTrue(Model._allow_direct_access()) + + resp = self.url_open('/runbot/api/models') + self.assertEqual(resp.status_code, 200) + data = resp.json() + self.assertIn(Model._name, data) + + resp = self.url_open(f'/runbot/api/{Model._name}/schema') + self.assertEqual(resp.status_code, 200) + + request_data = json.dumps({ + 'domain': [], + 'schema': resp.json(), + 'project_id': self.project.id, + }) + resp = self.url_open(f'/runbot/api/{Model._name}/read', data=request_data, headers={'Content-Type': 'application/json'}) + self.assertEqual(resp.status_code, 200) + + def test_api_read_from_schema_public_models(self): + # This is not ideal as we don't have any data but it is better than nothing + for Model in self.get_public_models(): + if not Model._allow_direct_access(): + continue + with self.subTest(model=Model._name): + resp = self.url_open(f'/runbot/api/{Model._name}/schema') + self.assertEqual(resp.status_code, 200) + schema = resp.json() + if set(schema['__required_keys']) > self.env['runbot.public.model.mixin']._get_request_required_keys(): + self.skipTest('Skipping, request requires unknown keys, create a specific test') + request_data = { + 'domain': [], + 'schema': resp.json(), + } + if Model._public_requires_project(): + request_data['project_id'] = self.project.id + request_data = json.dumps(request_data) + resp = self.url_open(f'/runbot/api/{Model._name}/read', data=request_data, headers={'Content-Type': 'application/json'}) + self.assertEqual(resp.status_code, 200) diff --git a/runbot_test/models.py b/runbot_test/models.py index 00e9baa7..af1c254f 100644 --- a/runbot_test/models.py +++ b/runbot_test/models.py @@ -30,6 +30,10 @@ class TestModelParent(models.Model): field_one2many_computed = fields.One2many('runbot.test.model.child', compute='_compute_one2many', public=True) field_many2many_computed = fields.Many2many('runbot.test.model.parent', compute='_compute_many2many', public=True) + @api.model + def _allow_direct_access(self): + return False + @api.depends() def _compute_one2many(self): self.field_one2many_computed = self.env['runbot.test.model.child'].search([]) @@ -50,3 +54,7 @@ class TestModelChild(models.Model): parent_id = fields.Many2one('runbot.test.model.parent', required=True, public=True) data = fields.Integer(public=True) + + @api.model + def _allow_direct_access(self): + return False diff --git a/runbot_test/tests/test_schema.py b/runbot_test/tests/test_schema.py index 804bd10d..dee0caed 100644 --- a/runbot_test/tests/test_schema.py +++ b/runbot_test/tests/test_schema.py @@ -23,7 +23,6 @@ class TestSchema(TransactionCase): schema['field_bool'], { '__type': 'boolean', - '__relational': False, } )