From eb68de40f30f4744448a1baed68eb20ed8bea9eb Mon Sep 17 00:00:00 2001 From: Christophe Monniez Date: Thu, 3 Jan 2019 11:08:48 +0100 Subject: [PATCH] [FIX] runbot: speedup and limit search in frontend When searching the builds for the frontend the resulting query can last a very long time (up to 7sec). With this commit, the search result is strictly limited to 100 builds, the limit query parameter is removed and the search string length is limited to 60 chars. The guess_result method is now optimized to guess results for testing builds only. The others have the same value as the final result. A few tests were added for this method. Thanks @KangOl for the optimization code. --- runbot/controllers/frontend.py | 8 ++++---- runbot/models/build.py | 9 ++++----- runbot/templates/frontend.xml | 4 ---- runbot/tests/test_build.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index c24a4256..c480b50b 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -47,7 +47,8 @@ class Runbot(Controller): return count, level @route(['/runbot', '/runbot/repo/'], website=True, auth='public', type='http') - def repo(self, repo=None, search='', limit='100', refresh='', **kwargs): + def repo(self, repo=None, search='', refresh='', **kwargs): + search = search if len(search) < 60 else search[:60] branch_obj = request.env['runbot.branch'] build_obj = request.env['runbot.build'] repo_obj = request.env['runbot.repo'] @@ -64,7 +65,6 @@ class Runbot(Controller): 'host_stats': [], 'pending_total': pending[0], 'pending_level': pending[1], - 'limit': limit, 'search': search, 'refresh': refresh, } @@ -81,7 +81,7 @@ class Runbot(Controller): search_domain += [('dest', 'ilike', to_search), ('subject', 'ilike', to_search), ('branch_id.branch_name', 'ilike', to_search)] domain += search_domain[1:] - build_ids = build_obj.search(domain, limit=int(limit)) + build_ids = build_obj.search(domain, limit=100) branch_ids, build_by_branch_ids = [], {} if build_ids: @@ -137,7 +137,7 @@ class Runbot(Controller): 'testing': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'testing')]), 'running': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'running')]), 'pending': build_obj.search_count([('repo_id', '=', repo.id), ('state', '=', 'pending')]), - 'qu': QueryURL('/runbot/repo/' + slug(repo), search=search, limit=limit, refresh=refresh, **filters), + 'qu': QueryURL('/runbot/repo/' + slug(repo), search=search, refresh=refresh, **filters), 'filters': filters, }) diff --git a/runbot/models/build.py b/runbot/models/build.py index 62e27ccb..894c783b 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -262,19 +262,18 @@ class runbot_build(models.Model): cr = self.env.cr cr.execute(""" SELECT b.id, - CASE WHEN b.state != 'testing' THEN b.result - WHEN array_agg(l.level)::text[] && ARRAY['ERROR', 'CRITICAL'] THEN 'ko' + CASE WHEN array_agg(l.level)::text[] && ARRAY['ERROR', 'CRITICAL'] THEN 'ko' WHEN array_agg(l.level)::text[] && ARRAY['WARNING'] THEN 'warn' ELSE 'ok' END FROM runbot_build b LEFT JOIN ir_logging l ON (l.build_id = b.id AND l.level != 'INFO') - WHERE b.id IN %s + WHERE b.id = ANY(%s) GROUP BY b.id - """, [tuple(self.ids)]) + """, [list(self.filtered(lambda b: b.state == 'testing').ids)]) result = {row[0]: row[1] for row in cr.fetchall()} for build in self: - build.guess_result = result[build.id] + build.guess_result = result.get(build.id, build.result) def _get_time(self): """Return the time taken by the tests""" diff --git a/runbot/templates/frontend.xml b/runbot/templates/frontend.xml index bae484c7..516c9ef8 100644 --- a/runbot/templates/frontend.xml +++ b/runbot/templates/frontend.xml @@ -82,10 +82,6 @@
  • Running
  • Done
  • Done
    • -
  • -
  • Show last 100
    • -
  • Show last 1000
    • -
  • Show last 10000
    • diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 3168fd13..39b5a4d7 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -45,6 +45,36 @@ class Test_Build(common.TransactionCase): build._get_domain() self.assertEqual(build.domain, 'runbot99.example.org:1234') + @patch('odoo.addons.runbot.models.build.fqdn') + def test_guess_result(self, mock_fqdn): + build = self.Build.create({ + 'branch_id': self.branch.id, + 'name': 'd0d0caca0000ffffffffffffffffffffffffffff', + 'port': '1234', + }) + # Testing the guess_result computed field + self.assertEqual(build.guess_result, '', 'A pending build guess_result should be empty') + + build.write({'state': 'done', 'result': 'ko'}) + build.invalidate_cache() + self.assertEqual(build.guess_result, 'ko', 'A finished build should return the same as result') + + build.write({'state': 'testing'}) + build.invalidate_cache() + self.assertEqual(build.guess_result, 'ok', 'A testing build without logs should be ok') + + self.env.cr.execute(""" + INSERT INTO ir_logging(name, type, path, func, line, build_id, level, message) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", ('testing', 'server', 'somewhere', 'test', 0, build.id, 'WARNING', 'blabla')) + build.invalidate_cache() + self.assertEqual(build.guess_result, 'warn', 'A testing build with warnings should be warn') + + self.env.cr.execute(""" + INSERT INTO ir_logging(name, type, path, func, line, build_id, level, message) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", ('testing', 'server', 'somewhere', 'test', 0, build.id, 'ERROR', 'blabla')) + build.invalidate_cache() + self.assertEqual(build.guess_result, 'ko', 'A testing build with errors should be ko') + @patch('odoo.addons.runbot.models.build.os.mkdir') @patch('odoo.addons.runbot.models.build.grep') def test_build_cmd_log_db(self, mock_grep, mock_mkdir):