From d6e1516f31caaa7c982ee49bedb815d623bfe3ff Mon Sep 17 00:00:00 2001
From: Xavier Morel <xmo@odoo.com>
Date: Mon, 9 Dec 2024 12:41:29 +0100
Subject: [PATCH] [IMP] runbot_merge: support arbitrary tunnel scripts
Rather than add individual tunnel methods to conftest, just allows
specifying a tunnel script and have that do whatever.
Protocol is uncomplicated: workers run the `$tunnel` with an arbitrary
port, script should create a tunnel to `localhost:$port`, print the
ingress of the tunnel to `STDOUT` with a terminating newline then
close `STDOUT`, and wait for `SIGINT` or `SIGTERM`, doing whatever
cleanup they need when receiving either of those.
`ngrok` and `localtunnel` adapter scripts are provided out of the box,
though the ngrok one doesn't *really* work when using xdist without a
pre-spawned ngrok worker. Then again using xdist against github actual
is suicidal (because concurrency limits + rate limits) so likely
irrelevant.
Fixes #729
---
conftest.py | 101 +++++++--------------------------------
runbot_merge/localtunnel | 25 ++++++++++
runbot_merge/ngrok | 96 +++++++++++++++++++++++++++++++++++++
3 files changed, 137 insertions(+), 85 deletions(-)
create mode 100644 runbot_merge/localtunnel
create mode 100755 runbot_merge/ngrok
diff --git a/conftest.py b/conftest.py
index 7b42dd87..1125b4a0 100644
--- a/conftest.py
+++ b/conftest.py
@@ -63,7 +63,6 @@ import itertools
import os
import pathlib
import pprint
-import random
import re
import socket
import subprocess
@@ -101,14 +100,11 @@ def pytest_addoption(parser):
parser.addoption('--coverage', action='store_true')
parser.addoption(
- '--tunnel', action="store", choices=['', 'ngrok', 'localtunnel'], default='',
- help="Which tunneling method to use to expose the local Odoo server "
- "to hook up github's webhook. ngrok is more reliable, but "
- "creating a free account is necessary to avoid rate-limiting "
- "issues (anonymous limiting is rate-limited at 20 incoming "
- "queries per minute, free is 40, multi-repo batching tests will "
- "blow through the former); localtunnel has no rate-limiting but "
- "the servers are way less reliable")
+ '--tunnel', action="store", default='',
+ help="Tunneling script, should take a port as argv[1] and output the "
+ "public address to stdout (with a newline) before closing it. "
+ "The tunneling script should respond gracefully to SIGINT and "
+ "SIGTERM.")
def is_manager(config):
return not hasattr(config, 'workerinput')
@@ -232,87 +228,22 @@ def users(partners, rolemap):
@pytest.fixture(scope='session')
def tunnel(pytestconfig: pytest.Config, port: int):
- """ Creates a tunnel to localhost:<port> using ngrok or localtunnel, should yield the
+ """ Creates a tunnel to localhost:<port>, should yield the
publicly routable address & terminate the process at the end of the session
"""
- tunnel = pytestconfig.getoption('--tunnel')
- if tunnel == '':
- yield f'http://localhost:{port}'
- elif tunnel == 'ngrok':
- own = None
- web_addr = 'http://localhost:4040/api'
- addr = 'localhost:%d' % port
- # try to find out if ngrok is running, and if it's not attempt
- # to start it
- try:
- # FIXME: this is for xdist to avoid workers running ngrok at the
- # exact same time, use lockfile instead
- time.sleep(random.SystemRandom().randint(1, 10))
- requests.get(web_addr)
- except requests.exceptions.ConnectionError:
- own = subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
- for _ in range(5):
- time.sleep(1)
- with contextlib.suppress(requests.exceptions.ConnectionError):
- requests.get(web_addr)
- break
- else:
- raise Exception("Unable to connect to ngrok")
-
- requests.post(f'{web_addr}/tunnels', json={
- 'name': str(port),
- 'proto': 'http',
- 'addr': addr,
- 'schemes': ['https'],
- 'inspect': True,
- }).raise_for_status()
-
- tunnel = f'{web_addr}/tunnels/{port}'
- for _ in range(10):
- time.sleep(2)
- r = requests.get(tunnel)
- # not created yet, wait and retry
- if r.status_code == 404:
- continue
- # check for weird responses
- r.raise_for_status()
- try:
- yield r.json()['public_url']
- finally:
- requests.delete(tunnel)
- for _ in range(10):
- time.sleep(1)
- r = requests.get(tunnel)
- # check if deletion is done
- if r.status_code == 404:
- break
- r.raise_for_status()
- else:
- raise TimeoutError("ngrok tunnel deletion failed")
-
- r = requests.get(f'{web_addr}/tunnels')
- assert r.ok, f'{r.reason} {r.text}'
- # there are still tunnels in the list -> bail
- if not own or r.json()['tunnels']:
- return
-
- # no more tunnels and we started ngrok -> try to kill it
- own.terminate()
- own.wait(30)
- else:
- raise TimeoutError("ngrok tunnel creation failed (?)")
- elif tunnel == 'localtunnel':
- p = subprocess.Popen(['lt', '-p', str(port)], stdout=subprocess.PIPE)
- try:
- r = p.stdout.readline()
- m = re.match(br'your url is: (https://.*\.localtunnel\.me)', r)
- assert m, "could not get the localtunnel URL"
- yield m.group(1).decode('ascii')
- finally:
+ if tunnel := pytestconfig.getoption('--tunnel'):
+ with subprocess.Popen(
+ [tunnel, str(port)],
+ stdout=subprocess.PIPE,
+ encoding="utf-8",
+ ) as p:
+ # read() blocks forever and I don't know why, read things about the
+ # write end of the stdout pipe still being open here?
+ yield p.stdout.readline().strip()
p.terminate()
p.wait(30)
else:
- raise ValueError("Unsupported %s tunnel method" % tunnel)
+ yield f'http://localhost:{port}'
class DbDict(dict):
def __init__(self, adpath, shared_dir):
diff --git a/runbot_merge/localtunnel b/runbot_merge/localtunnel
new file mode 100644
index 00000000..b0e2c067
--- /dev/null
+++ b/runbot_merge/localtunnel
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import re
+import signal
+import subprocess
+import sys
+import threading
+
+port = int(sys.argv[1])
+p = subprocess.Popen(['lt', '-p', str(port)], stdout=subprocess.PIPE, encoding="utf-8")
+r = p.stdout.readline()
+m = re.match(r'your url is: (https://.*\.localtunnel\.me)', r)
+assert m, "could not get the localtunnel URL"
+print(m[1], flush=True)
+sys.stdout.close()
+
+shutdown = threading.Event()
+def cleanup(_sig, _frame):
+ p.terminate()
+ p.wait(30)
+ shutdown.set()
+
+signal.signal(signal.SIGTERM, cleanup)
+signal.signal(signal.SIGINT, cleanup)
+
+shutdown.wait()
diff --git a/runbot_merge/ngrok b/runbot_merge/ngrok
new file mode 100755
index 00000000..33984100
--- /dev/null
+++ b/runbot_merge/ngrok
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+import contextlib
+import random
+import signal
+import subprocess
+import threading
+import time
+
+import requests
+import sys
+
+port = int(sys.argv[1])
+
+NGROK_CLI = [
+ 'ngrok', 'start', '--none', '--region', 'eu',
+]
+
+own = None
+web_addr = 'http://localhost:4040/api'
+addr = 'localhost:%d' % port
+
+# FIXME: this is for xdist to avoid workers running ngrok at the
+# exact same time, use lockfile instead
+time.sleep(random.SystemRandom().randint(1, 10))
+# try to find out if ngrok is running, and if it's not attempt
+# to start it
+try:
+ requests.get(web_addr)
+except requests.exceptions.ConnectionError:
+ own = subprocess.Popen(NGROK_CLI, stdout=subprocess.DEVNULL)
+ for _ in range(5):
+ time.sleep(1)
+ with contextlib.suppress(requests.exceptions.ConnectionError):
+ requests.get(web_addr)
+ break
+ else:
+ sys.exit("Unable to connect to ngrok")
+
+requests.post(f'{web_addr}/tunnels', json={
+ 'name': str(port),
+ 'proto': 'http',
+ 'addr': addr,
+ 'schemes': ['https'],
+ 'inspect': True,
+}).raise_for_status()
+
+
+tunnel = f'{web_addr}/tunnels/{port}'
+for _ in range(10):
+ time.sleep(2)
+ r = requests.get(tunnel)
+ # not created yet, wait and retry
+ if r.status_code == 404:
+ continue
+ # check for weird responses
+ r.raise_for_status()
+
+ print("opened tunnel", file=sys.stderr)
+ print(r.json()['public_url'], flush=True)
+ sys.stdout.close()
+ break
+else:
+ sys.exit("ngrok tunnel creation failed (?)")
+
+shutdown = threading.Event()
+def cleanup(_sig, _frame):
+ requests.delete(tunnel)
+ for _ in range(10):
+ time.sleep(1)
+ r = requests.get(tunnel)
+ # check if deletion is done
+ if r.status_code == 404:
+ break
+ r.raise_for_status()
+ else:
+ raise sys.exit("ngrok tunnel deletion failed")
+
+ r = requests.get(f'{web_addr}/tunnels')
+ if not r.ok:
+ sys.exit(f'{r.reason} {r.text}')
+ # FIXME: if we started ngrok, we should probably wait for all tunnels to be
+ # closed then terminate ngrok? This is likely a situation where the
+ # worker which created the ngrok instance finished early...
+ if own and not r.json()['tunnels']:
+ # no more tunnels and we started ngrok -> try to kill it
+ own.terminate()
+ own.wait(30)
+ shutdown.set()
+
+# don't know why but signal.sigwait doesn't seem to take SIGTERM in account so
+# we need the cursed version
+signal.signal(signal.SIGTERM, cleanup)
+signal.signal(signal.SIGINT, cleanup)
+
+print("wait for signal", file=sys.stderr)
+shutdown.wait()