[IMP] runbot_merge: support arbitrary tunnel scripts

Rather than add individual tunnel methods to conftest, just allows
specifying a tunnel script and have that do whatever.

Protocol is uncomplicated: workers run the `$tunnel` with an arbitrary
port, script should create a tunnel to `localhost:$port`, print the
ingress of the tunnel to `STDOUT` with a terminating newline then
close `STDOUT`, and wait for `SIGINT` or `SIGTERM`, doing  whatever
cleanup they need when receiving either of those.

`ngrok` and `localtunnel` adapter scripts are provided out of the box,
though the ngrok one doesn't *really* work when using xdist without a
pre-spawned ngrok worker. Then again using xdist against github actual
is suicidal (because concurrency limits + rate limits) so likely
irrelevant.

Fixes #729

conftest.py

@@ -63,7 +63,6 @@ import itertools
 import os
 import pathlib
 import pprint
-import random
 import re
 import socket
 import subprocess
@@ -101,14 +100,11 @@ def pytest_addoption(parser):
     parser.addoption('--coverage', action='store_true')
 
     parser.addoption(
-        '--tunnel', action="store", choices=['', 'ngrok', 'localtunnel'], default='',
+        '--tunnel', action="store", default='',
-        help="Which tunneling method to use to expose the local Odoo server "
+        help="Tunneling script, should take a port as argv[1] and output the "
-             "to hook up github's webhook. ngrok is more reliable, but "
+             "public address to stdout (with a newline) before closing it. "
-             "creating a free account is necessary to avoid rate-limiting "
+             "The tunneling script should respond gracefully to SIGINT and "
-             "issues (anonymous limiting is rate-limited at 20 incoming "
+             "SIGTERM.")
-             "queries per minute, free is 40, multi-repo batching tests will "
-             "blow through the former); localtunnel has no rate-limiting but "
-             "the servers are way less reliable")
 
 def is_manager(config):
     return not hasattr(config, 'workerinput')
@@ -232,87 +228,22 @@ def users(partners, rolemap):
 
 @pytest.fixture(scope='session')
 def tunnel(pytestconfig: pytest.Config, port: int):
-    """ Creates a tunnel to localhost:<port> using ngrok or localtunnel, should yield the
+    """ Creates a tunnel to localhost:<port>, should yield the
     publicly routable address & terminate the process at the end of the session
     """
-    tunnel = pytestconfig.getoption('--tunnel')
+    if tunnel := pytestconfig.getoption('--tunnel'):
-    if tunnel == '':
+        with subprocess.Popen(
-        yield f'http://localhost:{port}'
+            [tunnel, str(port)],
-    elif tunnel == 'ngrok':
+            stdout=subprocess.PIPE,
-        own = None
+            encoding="utf-8",
-        web_addr = 'http://localhost:4040/api'
+        ) as p:
-        addr = 'localhost:%d' % port
+            # read() blocks forever and I don't know why, read things about the
-        # try to find out if ngrok is running, and if it's not attempt
+            # write end of the stdout pipe still being open here?
-        # to start it
+            yield p.stdout.readline().strip()
-        try:
-            # FIXME: this is for xdist to avoid workers running ngrok at the
-            #        exact same time, use lockfile instead
-            time.sleep(random.SystemRandom().randint(1, 10))
-            requests.get(web_addr)
-        except requests.exceptions.ConnectionError:
-            own = subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
-            for _ in range(5):
-                time.sleep(1)
-                with contextlib.suppress(requests.exceptions.ConnectionError):
-                    requests.get(web_addr)
-                    break
-            else:
-                raise Exception("Unable to connect to ngrok")
-
-        requests.post(f'{web_addr}/tunnels', json={
-            'name': str(port),
-            'proto': 'http',
-            'addr': addr,
-            'schemes': ['https'],
-            'inspect': True,
-        }).raise_for_status()
-
-        tunnel = f'{web_addr}/tunnels/{port}'
-        for _ in range(10):
-            time.sleep(2)
-            r = requests.get(tunnel)
-            # not created yet, wait and retry
-            if r.status_code == 404:
-                continue
-            # check for weird responses
-            r.raise_for_status()
-            try:
-                yield r.json()['public_url']
-            finally:
-                requests.delete(tunnel)
-                for _ in range(10):
-                    time.sleep(1)
-                    r = requests.get(tunnel)
-                    # check if deletion is done
-                    if r.status_code == 404:
-                        break
-                    r.raise_for_status()
-                else:
-                    raise TimeoutError("ngrok tunnel deletion failed")
-
-                r = requests.get(f'{web_addr}/tunnels')
-                assert r.ok, f'{r.reason} {r.text}'
-                # there are still tunnels in the list -> bail
-                if not own or r.json()['tunnels']:
-                    return
-
-                # no more tunnels and we started ngrok -> try to kill it
-                own.terminate()
-                own.wait(30)
-        else:
-            raise TimeoutError("ngrok tunnel creation failed (?)")
-    elif tunnel == 'localtunnel':
-        p = subprocess.Popen(['lt', '-p', str(port)], stdout=subprocess.PIPE)
-        try:
-            r = p.stdout.readline()
-            m = re.match(br'your url is: (https://.*\.localtunnel\.me)', r)
-            assert m, "could not get the localtunnel URL"
-            yield m.group(1).decode('ascii')
-        finally:
             p.terminate()
             p.wait(30)
     else:
-        raise ValueError("Unsupported %s tunnel method" % tunnel)
+        yield f'http://localhost:{port}'
 
 class DbDict(dict):
     def __init__(self, adpath, shared_dir):

runbot_merge/localtunnel

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import re
+import signal
+import subprocess
+import sys
+import threading
+
+port = int(sys.argv[1])
+p = subprocess.Popen(['lt', '-p', str(port)], stdout=subprocess.PIPE, encoding="utf-8")
+r = p.stdout.readline()
+m = re.match(r'your url is: (https://.*\.localtunnel\.me)', r)
+assert m, "could not get the localtunnel URL"
+print(m[1], flush=True)
+sys.stdout.close()
+
+shutdown = threading.Event()
+def cleanup(_sig, _frame):
+    p.terminate()
+    p.wait(30)
+    shutdown.set()
+
+signal.signal(signal.SIGTERM, cleanup)
+signal.signal(signal.SIGINT, cleanup)
+
+shutdown.wait()

runbot_merge/ngrok

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+import contextlib
+import random
+import signal
+import subprocess
+import threading
+import time
+
+import requests
+import sys
+
+port = int(sys.argv[1])
+
+NGROK_CLI = [
+    'ngrok', 'start', '--none', '--region', 'eu',
+]
+
+own = None
+web_addr = 'http://localhost:4040/api'
+addr = 'localhost:%d' % port
+
+# FIXME: this is for xdist to avoid workers running ngrok at the
+#        exact same time, use lockfile instead
+time.sleep(random.SystemRandom().randint(1, 10))
+# try to find out if ngrok is running, and if it's not attempt
+# to start it
+try:
+    requests.get(web_addr)
+except requests.exceptions.ConnectionError:
+    own = subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
+    for _ in range(5):
+        time.sleep(1)
+        with contextlib.suppress(requests.exceptions.ConnectionError):
+            requests.get(web_addr)
+            break
+    else:
+        sys.exit("Unable to connect to ngrok")
+
+requests.post(f'{web_addr}/tunnels', json={
+    'name': str(port),
+    'proto': 'http',
+    'addr': addr,
+    'schemes': ['https'],
+    'inspect': True,
+}).raise_for_status()
+
+
+tunnel = f'{web_addr}/tunnels/{port}'
+for _ in range(10):
+    time.sleep(2)
+    r = requests.get(tunnel)
+    # not created yet, wait and retry
+    if r.status_code == 404:
+        continue
+    # check for weird responses
+    r.raise_for_status()
+
+    print("opened tunnel", file=sys.stderr)
+    print(r.json()['public_url'], flush=True)
+    sys.stdout.close()
+    break
+else:
+    sys.exit("ngrok tunnel creation failed (?)")
+
+shutdown = threading.Event()
+def cleanup(_sig, _frame):
+    requests.delete(tunnel)
+    for _ in range(10):
+        time.sleep(1)
+        r = requests.get(tunnel)
+        # check if deletion is done
+        if r.status_code == 404:
+            break
+        r.raise_for_status()
+    else:
+        raise sys.exit("ngrok tunnel deletion failed")
+
+    r = requests.get(f'{web_addr}/tunnels')
+    if not r.ok:
+        sys.exit(f'{r.reason} {r.text}')
+    # FIXME: if we started ngrok, we should probably wait for all tunnels to be
+    #        closed then terminate ngrok? This is likely a situation where the
+    #        worker which created the ngrok instance finished early...
+    if own and not r.json()['tunnels']:
+        # no more tunnels and we started ngrok -> try to kill it
+        own.terminate()
+        own.wait(30)
+    shutdown.set()
+
+# don't know why but signal.sigwait doesn't seem to take SIGTERM in account so
+# we need the cursed version
+signal.signal(signal.SIGTERM, cleanup)
+signal.signal(signal.SIGINT, cleanup)
+
+print("wait for signal", file=sys.stderr)
+shutdown.wait()