[IMP] runbot: general runbot log improvement

runbot/models/build.py

@@ -11,7 +11,7 @@ import time
 import datetime
 from ..common import dt2time, fqdn, now, grep, uniq_list, local_pgadmin_cursor, s2human, Commit, dest_reg
 from ..container import docker_build, docker_stop, docker_is_running, Command
-from odoo.addons.runbot.models.repo import HashMissingException, ArchiveFailException
+from odoo.addons.runbot.models.repo import RunbotException
 from odoo import models, fields, api
 from odoo.exceptions import UserError, ValidationError
 from odoo.http import request
@@ -442,9 +442,7 @@ class runbot_build(models.Model):
                 new_build = build.with_context(force_rebuild=True).create(values)
                 forced_builds |= new_build
                 user = request.env.user if request else self.env.user
-                new_build._log('rebuild', 'Rebuild initiated by %s (%s)' % (user.name, 'exact' if exact else 'default'))
+                new_build._log('rebuild', 'Rebuild initiated by %s (%s)%s' % (user.name, 'exact' if exact else 'default', (' :%s' % message) if message else ''))
-                if message:
-                    new_build._log('rebuild', new_build)
         return forced_builds
 
     def _skip(self, reason=None):
@@ -661,7 +659,10 @@ class runbot_build(models.Model):
                     pid = build.active_step._run(build)  # run should be on build?
                     build.write({'pid': pid})  # no really usefull anymore with dockers
                 except Exception as e:
-                    message = '%s failed running step %s:\n %s' % (build.dest, build.job, str(e).replace('\\n', '\n').replace("\\'", "'"))
+                    if isinstance(e, RunbotException):
+                        message = e.args[0]
+                    else:
+                        message = '%s failed running step %s:\n %s' % (build.dest, build.job, str(e).replace('\\n', '\n').replace("\\'", "'"))
                     _logger.exception(message)
                     build._log("run", message, level='ERROR')
                     build._kill(result='ko')
@@ -713,14 +714,7 @@ class runbot_build(models.Model):
             if build_export_path in exports:
                 self._log('_checkout', 'Multiple repo have same export path in build, some source may be missing for %s' % build_export_path, level='ERROR')
                 self._kill(result='ko')
-            try:
+            exports[build_export_path] = commit.export()
-                exports[build_export_path] = commit.export()
-            except HashMissingException:
-                self._log('_checkout', "Commit %s is unreachable. Did you force push the branch since build creation?" % commit, level='ERROR')
-                self._kill(result='ko')
-            except ArchiveFailException:
-                self._log('_checkout', "Archive %s failed. Did you force push the branch since build creation?" % commit, level='ERROR')
-                self._kill(result='ko')
         return exports
 
     def _get_repo_available_modules(self, commits=None):
@@ -883,8 +877,8 @@ class runbot_build(models.Model):
         for server_file in commit.repo.server_files.split(','):
             if os.path.isfile(commit._source_path(server_file)):
                 return (commit, server_file)
-        self._log('server_info', 'No server found in %s' % commit, level='ERROR')
+        _logger.error('None of %s found in commit, actual commit content:\n %s' % (commit.repo.server_files, os.listdir(commit._source_path())))
-        raise ValidationError('No server found in %s' % commit)
+        raise RunbotException('No server found in %s' % commit)
 
     def _cmd(self, python_params=None, py_version=None, local_only=True):
         """Return a list describing the command to start the build

runbot/models/build_config.py

@@ -376,9 +376,12 @@ class ConfigStep(models.Model):
             build._logger('Step %s finished in %s' % (self.name, s2human(build.job_time)))
             return
 
-        message = 'Step %s finished in %s $$fa-download$$' % (self.name, s2human(build.job_time))
+        kwargs = dict(message='Step %s finished in %s' % (self.name, s2human(build.job_time)))
-        link = '%s%s-%s.zip' % (build.http_log_url(), build.dest, self.db_name)
+        if self.job_type == 'install_odoo':
-        build._log('end_job', message, log_type='link', path=link)
+            kwargs['message'] += ' $$fa-download$$'
+            kwargs['path'] = '%s%s-%s.zip' % (build.http_log_url(), build.dest, self.db_name)
+            kwargs['log_type'] = 'link'
+        build._log('', **kwargs)
 
         if self.flamegraph:
             link = self._perf_data_url(build, 'log.gz')

runbot/models/repo.py

@@ -23,10 +23,7 @@ from ..container import docker_ps, docker_stop
 from psycopg2.extensions import TransactionRollbackError
 _logger = logging.getLogger(__name__)
 
-class HashMissingException(Exception):
+class RunbotException(Exception):
-    pass
-
-class ArchiveFailException(Exception):
     pass
 
 class runbot_repo(models.Model):
@@ -146,13 +143,13 @@ class runbot_repo(models.Model):
 
         if not self._hash_exists(sha):
             self._update(force=True)
-        if not self._hash_exists(sha):
+            if not self._hash_exists(sha):
-            try:
+                try:
-                result = self._git(['fetch', 'origin', sha])
+                    result = self._git(['fetch', 'origin', sha])
-            except:
+                except:
-                pass
+                    pass
-        if not self._hash_exists(sha):
+                if not self._hash_exists(sha):
-            raise HashMissingException()
+                    raise RunbotException("Commit %s is unreachable. Did you force push the branch since build creation?" % commit)
 
         _logger.info('git export: checkouting to %s (new)' % export_path)
         os.makedirs(export_path)
@@ -162,7 +159,7 @@ class runbot_repo(models.Model):
         p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
         (out, err) = p2.communicate()
         if err:
-            raise ArchiveFailException(err)
+            raise RunbotException("Archive %s failed. Did you force push the branch since build creation? (%s)" % (commit, err))
 
         # TODO get result and fallback on cleaing in case of problem
         return export_path
@@ -395,7 +392,7 @@ class runbot_repo(models.Model):
         """ Update the physical git reposotories on FS"""
         for repo in reversed(self):
             try:
-                repo._update_git(force)
+                repo._update_git(force) # TODO xdo, check gc log and log warning
             except Exception:
                 _logger.exception('Fail to update repo %s', repo.name)