[IMP] runbot: store docker build output in a dedicated model

When a docker fails to build, the output is logged in the chatter
leading to a lot of noise and a not so readable output. Moreover, the
output tries to format markdown and don't render line break correctly.

This commit proposes to introduce a model to store this output, as well
as some other info like the image identifier, build time, ...

This will help to compare images versions between hosts and should be
useful later to have multiple version of the same image with variant
once the docker registry is introduced.

README.md

@@ -309,7 +309,7 @@ Runbot is using a Dockerfile Odoo model to define the Dockerfile used for builds
 
 The model is using Odoo QWeb views as templates.
 
-A new Dockerfile can be created as needed either by duplicating the default one and adapt parameters in the view. e.g.: changing the key `'from': 'ubuntu:bionic'` to `'from': 'debian:buster'` will create a new Dockerfile based on Debian instead of ubuntu.
+A new Dockerfile can be created as needed either by duplicating the default one and adapt parameters in the view. e.g.: changing the key `'from': 'ubuntu:jammy'` to `'from': 'debian:buster'` will create a new Dockerfile based on Debian instead of ubuntu.
 Or by providing a plain Dockerfile in the template.
 
 Once the Dockerfile is created and the `to_build` field is checked, the Dockerfile will be built (pay attention that no other operations will occur during the build).

runbot/container.py

@@ -110,15 +110,17 @@ def _docker_build(build_dir, image_tag):
     """
     docker_client = docker.from_env()
     try:
-        docker_client.images.build(path=build_dir, tag=image_tag, rm=True)
+        docker_image, result_stream = docker_client.images.build(path=build_dir, tag=image_tag, rm=True)
+        result_stream = list(result_stream)
+        msg = ''.join([r.get('stream', '') for r in result_stream])
+        return docker_image, msg
     except docker.errors.APIError as e:
-        _logger.error('Build of image %s failed with this API error:', image_tag)
+        _logger.error('Build of image %s failed', image_tag)
         return (False, e.explanation)
     except docker.errors.BuildError as e:
-        _logger.error('Build of image %s failed with this BUILD error:', image_tag)
+        _logger.error('Build of image %s failed', image_tag)
         msg = f"{e.msg}\n{''.join(l.get('stream') or '' for l in e.build_log)}"
         return (False, msg)
-    return (True, None)
 
 
 def docker_run(*args, **kwargs):

runbot/models/__init__.py

@@ -11,7 +11,7 @@ from . import codeowner
 from . import commit
 from . import custom_trigger
 from . import database
-from . import dockerfile
+from . import docker
 from . import host
 from . import ir_cron
 from . import ir_http

runbot/models/docker.py

@@ -22,6 +22,8 @@ class Dockerfile(models.Model):
     view_ids = fields.Many2many('ir.ui.view', compute='_compute_view_ids', groups="runbot.group_runbot_admin")
     project_ids = fields.One2many('runbot.project', 'dockerfile_id', string='Default for Projects')
     bundle_ids = fields.One2many('runbot.bundle', 'dockerfile_id', string='Used in Bundles')
+    build_results = fields.One2many('runbot.docker_build_result', 'dockerfile_id', string='Build results')
+    last_successful_result = fields.Many2one('runbot.docker_build_result', compute='_compute_last_successful_result')
 
     _sql_constraints = [('runbot_dockerfile_name_unique', 'unique(name)', 'A Dockerfile with this name already exists')]
 
@@ -33,6 +35,10 @@ class Dockerfile(models.Model):
         copied_record.template_id.key = '%s (copy)' % copied_record.template_id.key
         return copied_record
 
+    def _compute_last_successful_result(self):
+        for record in self:
+            record.last_successful_result = next((result for result in record.build_results if result.result == 'success'), record.build_results.browse())
+
     @api.depends('template_id.arch_base')
     def _compute_dockerfile(self):
         for rec in self:
@@ -53,3 +59,28 @@ class Dockerfile(models.Model):
         for rec in self:
             keys = re.findall(r'<t.+t-call="(.+)".+', rec.arch_base or '')
             rec.view_ids = self.env['ir.ui.view'].search([('type', '=', 'qweb'), ('key', 'in', keys)]).ids
+
+
+class DockerBuildOutput(models.Model):
+    _name = 'runbot.docker_build_result'
+    _description = "Result of a docker file build"
+    _order = 'id desc'
+
+    result = fields.Selection(string="Result", selection=[('error', 'Error'), ('success', 'Success')])
+    host_id = fields.Many2one('runbot.host', string="Host")
+    duration = fields.Float("Exec time")
+    dockerfile_id = fields.Many2one('runbot.dockerfile', string="Docker file")
+    output = fields.Text('Output')
+    content = fields.Text('Content')
+    identifier = fields.Char('Identifier')
+    summary = fields.Char("Summary", compute='_compute_summary', store=True)
+
+    @api.depends('output')
+    def _compute_summary(self):
+        for record in self:
+            summary = ''
+            for line in reversed(self.output.split('\n')):
+                if len(line) > 5:
+                    summary = line
+                    break
+            record.summary = summary

runbot/models/host.py

@@ -143,19 +143,45 @@ class Host(models.Model):
             USER {user}
             ENV COVERAGE_FILE /data/build/.coverage
             """
+        content = dockerfile.dockerfile + docker_append
         with open(self.env['runbot.runbot']._path('docker', dockerfile.image_tag, 'Dockerfile'), 'w') as Dockerfile:
-            Dockerfile.write(dockerfile.dockerfile + docker_append)
+            Dockerfile.write(content)
 
-        docker_build_success, msg = docker_build(docker_build_path, dockerfile.image_tag)
-        if not docker_build_success:
-            dockerfile.to_build = False
-            dockerfile.message_post(body=f'Build failure:\n{msg}')
-            # self.env['runbot.runbot']._warning(f'Dockerfile build "{dockerfile.image_tag}" failed on host {self.name}')
-        else:
-            duration = time.time() - start
+        docker_build_identifier, msg = docker_build(docker_build_path, dockerfile.image_tag)
+        duration = time.time() - start
+        docker_build_result_values = {'dockerfile_id': dockerfile.id, 'output': msg, 'duration': duration, 'content': content, 'host_id': self.id}
+        duration = time.time() - start
+        if docker_build_identifier:
+            docker_build_result_values['result'] = 'success'
+            docker_build_result_values['identifier'] = docker_build_identifier.id
             if duration > 1:
                 _logger.info('Dockerfile %s finished build in %s', dockerfile.image_tag, duration)
-    
+        else:
+            docker_build_result_values['result'] = 'error'
+            dockerfile.to_build = False
+
+        should_save_result = not docker_build_identifier  # always save in case of failure
+        if not should_save_result:
+            # check previous result anyway
+            previous_result = self.env['runbot.docker_build_result'].search([
+                ('dockerfile_id', '=', dockerfile.id),
+                ('host_id', '=', self.id),
+            ], order='id desc', limit=1)
+            # identifier changed
+            if docker_build_identifier.id != previous_result.identifier:
+                should_save_result = True
+            if previous_result.output != docker_build_result_values['output']:  # to discuss
+                should_save_result = True
+            if previous_result.content != docker_build_result_values['content']:  # docker image changed
+                should_save_result = True
+
+        if should_save_result:
+            result = self.env['runbot.docker_build_result'].create(docker_build_result_values)
+            if not docker_build_identifier:
+                message = f'Build failure, check results for more info ({result.summary})'
+                dockerfile.message_post(body=message)
+                _logger.error(message)
+
     @ormcache()
     def _host_list(self):
         return {host.name: host.id for host in self.search([])}

runbot/models/runbot.py

@@ -34,7 +34,7 @@ class Runbot(models.AbstractModel):
     def _root(self):
         """Return root directory of repository"""
         return os.path.abspath(os.sep.join([os.path.dirname(__file__), '../static']))
-    
+
     def _path(self, *path_parts):
         """Return the repo build path"""
         root = self.env['runbot.runbot']._root()

runbot/security/ir.model.access.csv

@@ -128,6 +128,9 @@ access_runbot_upgrade_exception_admin,access_runbot_upgrade_exception_admin,runb
 access_runbot_dockerfile_user,access_runbot_dockerfile_user,runbot.model_runbot_dockerfile,runbot.group_user,1,0,0,0
 access_runbot_dockerfile_admin,access_runbot_dockerfile_admin,runbot.model_runbot_dockerfile,runbot.group_runbot_admin,1,1,1,1
 
+access_runbot_docker_build_result_user,access_runbot_docker_build_result_user,runbot.model_runbot_docker_build_result,runbot.group_user,1,0,0,0
+access_runbot_docker_build_result_admin,access_runbot_docker_build_result_admin,runbot.model_runbot_docker_build_result,runbot.group_runbot_admin,1,1,1,1
+
 access_runbot_codeowner_admin,runbot_codeowner_admin,runbot.model_runbot_codeowner,runbot.group_runbot_admin,1,1,1,1
 access_runbot_codeowner_user,runbot_codeowner_user,runbot.model_runbot_codeowner,group_user,1,0,0,0
 

runbot/templates/dockerfile.xml

@@ -56,7 +56,7 @@ RUN curl -sSL <t t-esc="values['wkhtml_url']"/> -o /tmp/wkhtml.deb \
     </template>
 
     <template id="runbot.docker_install_nodejs">
-      <t t-set="node_version" t-value="node_version or '15'"/>
+      <t t-set="node_version" t-value="node_version or '20'"/>
 # Install nodejs
 RUN curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
     &amp;&amp; echo "deb https://deb.nodesource.com/node_<t t-esc="values['node_version']"/>.x `lsb_release -c -s` main" > /etc/apt/sources.list.d/nodesource.list \
@@ -110,19 +110,19 @@ RUN <t t-esc="values['python_version']"/> -m pip install --no-cache-dir setuptoo
 
     <template id="runbot.docker_base">
       <t t-set="default" t-value="{
-'from': 'ubuntu:focal',
+'from': 'ubuntu:jammy',
 'odoo_branch': 'master',
 'chrome_source': 'google',
-'chrome_version': '90.0.4430.93-1',
+'chrome_version': '123.0.6312.58-1',
 'node_packages': 'rtlcss es-check eslint',
-'node_version': '15',
+'node_version': '20',
 'psql_version': '12',
 'wkhtml_url': 'https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb',
 'chrome': True,
 'phantom': False,
 'do_requirements': True,
 'python_version': 'python3',
-'deb_packages_python': 'python3 python3-dbfread python3-dev python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-vatnumber python3-websocket libpq-dev',
+'deb_packages_python': 'python3 python3-dbfread python3-dev python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket libpq-dev python3-gevent',
 'deb_package_default': 'apt-transport-https build-essential ca-certificates curl ffmpeg file fonts-freefont-ttf fonts-noto-cjk gawk gnupg gsfonts libldap2-dev libjpeg9-dev libsasl2-dev libxslt1-dev lsb-release node-less ocrmypdf sed sudo unzip xfonts-75dpi zip zlib1g-dev',
 'additional_pip': 'ebaysdk==2.1.5 pdf417gen==0.7.1',
 'runbot_pip': 'coverage==4.5.4 astroid==2.4.2 pylint==2.5.0 flamegraph'

runbot/tests/test_dockerfile.py

@@ -15,7 +15,7 @@ class TestDockerfile(RunbotCase, HttpCase):
     def test_dockerfile_base_fields(self):
         xml_content = """<t t-call="runbot.docker_base">
     <t t-set="custom_values" t-value="{
-      'from': 'ubuntu:focal',
+      'from': 'ubuntu:jammy',
       'phantom': True,
       'additional_pip': 'babel==2.8.0',
       'chrome_source': 'odoo',
@@ -38,7 +38,7 @@ class TestDockerfile(RunbotCase, HttpCase):
         })
 
         self.assertEqual(dockerfile.image_tag, 'odoo:TestsUbuntuFocal20.0Chrome86')
-        self.assertTrue(dockerfile.dockerfile.startswith('FROM ubuntu:focal'))
+        self.assertTrue(dockerfile.dockerfile.startswith('FROM ubuntu:jammy'))
         self.assertIn(' apt-get install -y -qq google-chrome-stable=86.0.4240.183-1', dockerfile.dockerfile)
         self.assertIn('# Install phantomjs', dockerfile.dockerfile)
         self.assertIn('pip install --no-cache-dir babel==2.8.0', dockerfile.dockerfile)

runbot/views/dockerfile_views.xml

@@ -41,6 +41,19 @@
                     </tree>
                   </field>
                 </page>
+                <page string="Build results">
+                  <field name="build_results">
+                    <tree>
+                      <field name="dockerfile_id"/>
+                      <field name="host_id"/>
+                      <field name="duration"/>
+                      <field name="result" decoration-bg-danger="result != 'success'" decoration-bg-success="result == 'success'"/>
+                      <field name="identifier"/>
+                      <field name="create_date"/>
+                      <field name="summary"/>
+                    </tree>
+                  </field>
+                </page>
               </notebook>
             </sheet>
             <div class="oe_chatter">
@@ -68,11 +81,59 @@
         </field>
     </record>
 
-    <record id="open_view_dockerfile_tree" model="ir.actions.act_window">
-        <field name="name">Dockerfiles</field>
-        <field name="res_model">runbot.dockerfile</field>
-        <field name="view_mode">tree,form</field>
+    <record id="docker_build_result_form" model="ir.ui.view">
+      <field name="name">runbot.docker_build_result.form</field>
+      <field name="model">runbot.docker_build_result</field>
+      <field name="arch" type="xml">
+        <form string="Docker build result">
+          <sheet>
+            <group>
+              <field name="dockerfile_id"/>
+              <field name="host_id"/>
+              <field name="duration"/>
+              <field name="result"/>
+              <field name="identifier"/>
+              <field name="summary"/>
+            </group>
+            <notebook>
+              <page string="Output">
+                <field name="output"/>
+              </page>
+              <page string="Content">
+                <field name="content"/>
+              </page>
+            </notebook>
+          </sheet>
+        </form>
+      </field>
     </record>
 
+    <record id="docker_build_result_tree" model="ir.ui.view">
+      <field name="name">runbot.docker_build_result.tree</field>
+      <field name="model">runbot.docker_build_result</field>
+      <field name="arch" type="xml">
+        <tree string="Docker build result">
+          <field name="dockerfile_id"/>
+          <field name="host_id"/>
+          <field name="duration"/>
+          <field name="result"/>
+          <field name="identifier"/>
+          <field name="summary"/>
+        </tree>
+      </field>
+    </record>
+
+  <record id="open_view_dockerfile_tree" model="ir.actions.act_window">
+    <field name="name">Dockerfiles</field>
+    <field name="res_model">runbot.dockerfile</field>
+    <field name="view_mode">tree,form</field>
+  </record>
+
+  <record id="open_view_docker_result_tree" model="ir.actions.act_window">
+    <field name="name">Docker build results</field>
+    <field name="res_model">runbot.docker_build_result</field>
+    <field name="view_mode">tree,form</field>
+  </record>
+
   </data>
 </odoo>

runbot/views/menus.xml

@@ -28,7 +28,9 @@
 
     <menuitem id="runbot_menu_upgrade_exceptions_tree" parent="runbot_menu_root" sequence="700" action="open_view_upgrade_exception_tree"/>
 
-    <menuitem name="Docker" id="menu_dockerfile" parent="runbot_menu_root" action="open_view_dockerfile_tree" sequence="800"/>
+    <menuitem name="Docker" id="menu_dockerfile" parent="runbot_menu_root" sequence="800"/>
+    <menuitem name="Docker files" id="menu_dockerfiles" parent="menu_dockerfile" action="open_view_dockerfile_tree" sequence="801"/>
+    <menuitem name="Docker build results" id="menu_docker_results" parent="menu_dockerfile" action="open_view_docker_result_tree" sequence="802"/>
 
     <menuitem name="Manage errors" id="runbot_menu_manage_errors" parent="runbot_menu_root" sequence="900"/>
     <menuitem name="Build errors" id="runbot_menu_build_error_tree" parent="runbot_menu_manage_errors" sequence="10" action="open_view_build_error_tree"/>

runbot_builder/builder.py

@@ -21,7 +21,7 @@ class BuilderClient(RunbotClient):
                 repo._update(force=True)
 
     def loop_turn(self):
-        if self.count == 1: # cleanup at second iteration
+        if self.count == 1:  # cleanup at second iteration
             self.env['runbot.runbot']._source_cleanup()
             self.env['runbot.build']._local_cleanup()
             self.env['runbot.runbot']._docker_cleanup()

runbot_builder/tools.py

@@ -230,7 +230,7 @@ def prepare_stats_log(dest, previous_stats, current_stats):
     system_delta = (current_cpu_time - previous_cpu_time) / CPU_COUNT
     cpu_percent = cpu_delta / system_delta * 100
     cpu_percent_delta = (cpu_percent - logged_cpu_percent)
-    cpu_percent_ratio =  cpu_percent_delta / (logged_cpu_percent or 0.000001) * 100
+    cpu_percent_ratio = cpu_percent_delta / (logged_cpu_percent or 0.000001) * 100
 
     log_lines = []
     date_time = datetime.fromtimestamp(current_time).astimezone(tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]