diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index dfcd6b45..4a7c32cc 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -16,6 +16,7 @@ 'views/repo_views.xml', 'views/branch_views.xml', 'views/build_views.xml', + 'views/host_views.xml', 'views/config_views.xml', 'views/res_config_settings_views.xml', 'templates/frontend.xml', diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 2169e61f..c70994b8 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -273,8 +273,7 @@ class Runbot(Controller): return request.render("runbot.sticky-dashboard", qctx) - @route('/runbot/glances', type='http', auth='public', website=True) - def glances(self, refresh=None): + def _glances_ctx(self): repos = request.env['runbot.repo'].search([]) # respect record rules default_config_id = request.env.ref('runbot.runbot_build_config_default').id query = """ @@ -302,16 +301,45 @@ class Runbot(Controller): ctx = OrderedDict() for row in cr.fetchall(): ctx.setdefault(row[0], []).append(row[1:]) + return ctx + @route('/runbot/glances', type='http', auth='public', website=True) + def glances(self, refresh=None): + glances_ctx = self._glances_ctx() pending = self._pending() qctx = { 'refresh': refresh, 'pending_total': pending[0], 'pending_level': pending[1], - 'data': ctx, + 'glances_data': glances_ctx, } return request.render("runbot.glances", qctx) + @route('/runbot/monitoring', type='http', auth='user', website=True) + def monitoring(self, refresh=None): + glances_ctx = self._glances_ctx() + pending = self._pending() + hosts_data = request.env['runbot.host'].search([]) + + monitored_config_id = int(request.env['ir.config_parameter'].sudo().get_param('runbot.monitored_config_id', 1)) + request.env.cr.execute("""SELECT DISTINCT ON (branch_id) branch_id, id FROM runbot_build + WHERE config_id = %s + AND global_state in ('running', 'done') + AND branch_id in (SELECT id FROM runbot_branch where sticky='t') + ORDER BY branch_id ASC, id DESC""", [int(monitored_config_id)]) + last_monitored = request.env['runbot.build'].browse([r[1] for r in request.env.cr.fetchall()]) + + qctx = { + 'refresh': refresh, + 'pending_total': pending[0], + 'pending_level': pending[1], + 'glances_data': glances_ctx, + 'hosts_data': hosts_data, + 'last_monitored': last_monitored # nightly + + } + return request.render("runbot.monitoring", qctx) + @route(['/runbot/branch/', '/runbot/branch//page/'], website=True, auth='public', type='http') def branch_builds(self, branch_id=None, search='', page=1, limit=50, refresh='', **kwargs): """ list builds of a runbot branch """ diff --git a/runbot/models/__init__.py b/runbot/models/__init__.py index be43c42e..d7cc9322 100644 --- a/runbot/models/__init__.py +++ b/runbot/models/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- -from . import repo, branch, build, event, build_dependency, build_config, ir_cron +from . import repo, branch, build, event, build_dependency, build_config, ir_cron, host from . import res_config_settings diff --git a/runbot/models/host.py b/runbot/models/host.py new file mode 100644 index 00000000..4d719d00 --- /dev/null +++ b/runbot/models/host.py @@ -0,0 +1,60 @@ +import logging +from odoo import models, fields, api +from ..common import fqdn, local_pgadmin_cursor +_logger = logging.getLogger(__name__) + + +class RunboHost(models.Model): + _name = "runbot.host" + _order = 'id' + _inherit = 'mail.thread' + + name = fields.Char('Host name', required=True, unique=True) + disp_name = fields.Char('Display name') + active = fields.Boolean('Active', default=True) + last_start_loop = fields.Datetime('Last start') + last_end_loop = fields.Datetime('Last end') + last_success = fields.Datetime('Last success') + assigned_only = fields.Boolean('Only accept assigned build', default=False) + nb_worker = fields.Integer('Number of max paralel build', help="0 to use icp value", default=0) + nb_testing = fields.Integer(compute='_compute_nb') + nb_running = fields.Integer(compute='_compute_nb') + last_exception = fields.Char('Last exception') + exception_count = fields.Integer('Exception count') + psql_conn_count = fields.Integer('SQL connections count', default=0) + + def _compute_nb(self): + groups = self.env['runbot.build'].read_group( + [('host', 'in', self.mapped('name')), ('local_state', 'in', ('testing', 'running'))], + ['host', 'local_state'], + ['host', 'local_state'], + lazy=False + ) + count_by_host_state = {host.name: {} for host in self} + for group in groups: + count_by_host_state[group['host']][group['local_state']] = group['__count'] + for host in self: + host.nb_testing = count_by_host_state[host.name].get('testing', 0) + host.nb_running = count_by_host_state[host.name].get('running', 0) + + @api.model + def create(self, values): + if not 'disp_name' in values: + values['disp_name'] = values['name'] + return super().create(values) + + @api.model + def _get_current(self): + name = fqdn() + return self.search([('name', '=', name)]) or self.create({'name': name}) + + def get_nb_worker(self): + icp = self.env['ir.config_parameter'] + return self.nb_worker or int(icp.sudo().get_param('runbot.runbot_workers', default=6)) + + def set_psql_conn_count(self): + self.ensure_one() + with local_pgadmin_cursor() as local_cr: + local_cr.execute("SELECT sum(numbackends) FROM pg_stat_database;") + res = local_cr.fetchone() + self.psql_conn_count = res and res[0] or 0 diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 05d95e70..10a33dd0 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -15,7 +15,7 @@ import shutil from odoo.exceptions import UserError, ValidationError from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT -from odoo import models, fields, api +from odoo import models, fields, api, registry from odoo.modules.module import get_module_resource from odoo.tools import config from ..common import fqdn, dt2time, Commit @@ -398,21 +398,20 @@ class runbot_repo(models.Model): _logger.exception('Fail to update repo %s', repo.name) @api.multi - def _scheduler(self): + def _scheduler(self, host=None): """Schedule builds for the repository""" ids = self.ids if not ids: return icp = self.env['ir.config_parameter'] - host = fqdn() - settings_workers = int(icp.get_param('runbot.runbot_workers', default=6)) - workers = int(icp.get_param('%s.workers' % host, default=settings_workers)) + host = host or self.env['runbot.host']._get_current() + workers = host.get_nb_worker() running_max = int(icp.get_param('runbot.runbot_running_max', default=75)) - assigned_only = int(icp.get_param('%s.assigned_only' % host, default=False)) + assigned_only = host.assigned_only Build = self.env['runbot.build'] domain = [('repo_id', 'in', ids)] - domain_host = domain + [('host', '=', host)] + domain_host = domain + [('host', '=', host.name)] # schedule jobs (transitions testing -> running, kill jobs, ...) build_ids = Build.search(domain_host + ['|', ('local_state', 'in', ['testing', 'running']), ('requested_action', 'in', ['wake_up', 'deathrow'])]) @@ -458,7 +457,7 @@ class runbot_repo(models.Model): ) RETURNING id""" % where_clause - self.env.cr.execute(query, {'repo_ids': tuple(ids), 'host': fqdn(), 'limit': limit}) + self.env.cr.execute(query, {'repo_ids': tuple(ids), 'host': host.name, 'limit': limit}) return self.env.cr.fetchall() allocated = allocate_builds("""AND runbot_build.build_type != 'scheduled'""", assignable_slots) @@ -561,6 +560,10 @@ class runbot_repo(models.Model): """ if hostname != fqdn(): return 'Not for me' + host = self.env['runbot.host']._get_current() + host.set_psql_conn_count() + host.last_start_loop = fields.Datetime.now() + self.env.cr.commit() start_time = time.time() # 1. source cleanup # -> Remove sources when no build is using them @@ -576,7 +579,8 @@ class runbot_repo(models.Model): while time.time() - start_time < timeout: repos = self.search([('mode', '!=', 'disabled')]) try: - repos._scheduler() + repos._scheduler(host) + host.last_success = fields.Datetime.now() self.env.cr.commit() self.env.reset() self = self.env()[self._name] @@ -587,6 +591,21 @@ class runbot_repo(models.Model): self.env.cr.rollback() self.env.reset() time.sleep(random.uniform(0, 1)) + except Exception as e: + with registry(self._cr.dbname).cursor() as cr: # user another cursor since transaction will be rollbacked + message = str(e) + chost = host.with_env(self.env(cr=cr)) + if chost.last_exception == message: + chost.exception_count += 1 + else: + chost.with_env(self.env(cr=cr)).last_exception = str(e) + chost.exception_count = 1 + raise + + if host.last_exception: + host.last_exception = "" + host.exception_count = 0 + host.last_end_loop = fields.Datetime.now() def _source_cleanup(self): try: diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index 5a0f8429..bc2119a0 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -17,3 +17,6 @@ access_runbot_build_config_manager,runbot_build_config_manager,runbot.model_runb access_runbot_build_config_step_order_user,runbot_build_config_step_order_user,runbot.model_runbot_build_config_step_order,group_user,1,0,0,0 access_runbot_build_config_step_order_manager,runbot_build_config_step_order_manager,runbot.model_runbot_build_config_step_order,runbot.group_build_config_user,1,1,1,1 + +access_runbot_host_user,runbot_host_user,runbot.model_runbot_host,group_user,1,0,0,0 +access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/templates/dashboard.xml b/runbot/templates/dashboard.xml index b7173461..da5630b3 100644 --- a/runbot/templates/dashboard.xml +++ b/runbot/templates/dashboard.xml @@ -76,7 +76,7 @@ + + + + diff --git a/runbot/tests/test_cron.py b/runbot/tests/test_cron.py index db7c6433..283b2a98 100644 --- a/runbot/tests/test_cron.py +++ b/runbot/tests/test_cron.py @@ -48,13 +48,15 @@ class Test_Cron(common.TransactionCase): mock_update.assert_called_with(force=False) mock_create.assert_called_with() + @patch('odoo.addons.runbot.models.host.fqdn') @patch('odoo.addons.runbot.models.repo.runbot_repo._get_cron_period') @patch('odoo.addons.runbot.models.repo.runbot_repo._reload_nginx') @patch('odoo.addons.runbot.models.repo.runbot_repo._scheduler') @patch('odoo.addons.runbot.models.repo.fqdn') - def test_cron_build(self, mock_fqdn, mock_scheduler, mock_reload, mock_cron_period): + def test_cron_build(self, mock_fqdn, mock_scheduler, mock_reload, mock_cron_period, mock_host_fqdn): """ test that cron_fetch_and_build do its work """ - mock_fqdn.return_value = 'runbotx.foo.com' + hostname = 'runbotx.foo.com' + mock_fqdn.return_value = mock_host_fqdn.return_value = hostname mock_cron_period.return_value = 2 self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_update_frequency', 1) self.Repo.create({'name': '/path/somewhere/disabled.git', 'mode': 'disabled'}) # create a disabled @@ -64,3 +66,6 @@ class Test_Cron(common.TransactionCase): self.assertEqual(None, ret) mock_scheduler.assert_called() self.assertTrue(mock_reload.called) + host = self.env['runbot.host'].search([('name', '=', 'runbotx.foo.com')]) + self.assertEqual(host.name, hostname, 'A new host should have been created') + self.assertGreater(host.psql_conn_count, 0, 'A least one connection should exist on the current psql instance') diff --git a/runbot/tests/test_repo.py b/runbot/tests/test_repo.py index 9a395492..54e831b9 100644 --- a/runbot/tests/test_repo.py +++ b/runbot/tests/test_repo.py @@ -201,9 +201,9 @@ class Test_Repo_Scheduler(common.TransactionCase): @patch('odoo.addons.runbot.models.build.runbot_build._reap') @patch('odoo.addons.runbot.models.build.runbot_build._kill') @patch('odoo.addons.runbot.models.build.runbot_build._schedule') - @patch('odoo.addons.runbot.models.repo.fqdn') - def test_repo_scheduler(self, mock_repo_fqdn, mock_schedule, mock_kill, mock_reap): - mock_repo_fqdn.return_value = 'test_host' + @patch('odoo.addons.runbot.models.host.fqdn') + def test_repo_scheduler(self, mock_fqdn, mock_schedule, mock_kill, mock_reap): + mock_fqdn.return_value = 'test_host' self.env['ir.config_parameter'].set_param('runbot.runbot_workers', 6) Build_model = self.env['runbot.build'] builds = [] @@ -237,6 +237,7 @@ class Test_Repo_Scheduler(common.TransactionCase): 'local_state': 'pending', }) builds.append(build) + self.foo_repo._scheduler() build.invalidate_cache() diff --git a/runbot/views/host_views.xml b/runbot/views/host_views.xml new file mode 100644 index 00000000..bfbd09b5 --- /dev/null +++ b/runbot/views/host_views.xml @@ -0,0 +1,60 @@ + + + + + runbot.host.form + runbot.host + +
+ + + + + + + + + + + + + + + + +
+ + +
+
+
+
+ + + runbot.host.tree + runbot.host + + + + + + + + + + + + Host + runbot.host + tree,form + + + +
+