update

scraper_test/README.md

@@ -0,0 +1,20 @@
+# Scraper Test
+> [!WARNING]
+> This module is purely experimental and for educational purpose use only.
+>
+> Do not use it in any environment but in an experimental one, definitely not in a production environment.
+>
+> I'm not responsible for any damage or harm by the use of anything from this repo.
+>
+> Use it at your own risk.
+
+> [!CAUTION]
+> AI might generate commands that negatively impact your data.
+>
+> Do not use this module unless you have reviewed the source codes thoroughly, understand what it does and in an experimental environment.
+
+This module show the issues when running Crawl4AI, Playwright and Selenium inside Odoo.
+
+Please watch this video for more details:
+
+[![EXPLORING_ODOO](https://img.youtube.com/vi/dogjC_0P53A/0.jpg)](https://youtu.be/dogjC_0P53A)

scraper_test/__init__.py

@@ -0,0 +1 @@
+from . import wizard

scraper_test/__manifest__.py

@@ -0,0 +1,18 @@
+{
+    "name": "Scraper Test",
+    "version": "18.0.1.0.0",
+    "depends": ["web", "bus", "ai_chat_base"],
+    "author": "Yoni Tjio",
+    "category": "Customizations",
+    "description": """
+    Scraper Test
+    """,
+    "data": [
+        "security/ir.model.access.csv",
+        "wizard/scrap.xml",
+    ],
+    "application": True,
+    "installable": True,
+    "auto_install": False,
+    "license": "Other proprietary",
+}

scraper_test/security/ir.model.access.csv

@@ -0,0 +1,2 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_scrap_wizard,access_scrap_wizard,model_scrap_wizard,base.group_system,1,1,1,1

scraper_test/test.py

@@ -0,0 +1,386 @@
+import os
+import sys
+import signal
+import resource
+import argparse
+import logging
+import asyncio
+from multiprocessing import Process, get_context
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from playwright.async_api import async_playwright
+from selenium import webdriver
+
+_logger = logging.getLogger(__name__)
+
+def we_are_frozen():
+    # All of the modules are built-in to the interpreter, e.g., by py2exe
+    return hasattr(sys, "frozen")
+
+def module_path():
+    if we_are_frozen():
+        return os.path.dirname(sys.executable)
+    return os.path.dirname(__file__)
+
+CWD = module_path()
+TEST_WEBSITE = "http://example.com"
+
+# region utils
+def exit():
+    loop = asyncio.get_event_loop()
+    print("Stop")
+    loop.stop()
+
+def ask_exit():
+    for task in asyncio.Task.all_tasks():
+        task.cancel()
+    asyncio.ensure_future(exit())
+
+def run_async_function(func_to_run, *args):
+    new_loop = asyncio.new_event_loop()
+    try:
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            new_loop.add_signal_handler(sig, ask_exit)
+        asyncio.set_event_loop(new_loop)
+        return new_loop.run_until_complete(func_to_run(*args))
+    finally:
+        new_loop.close()
+# endregion
+
+# region base tests
+async def test_crawl4ai_async(ref_url):
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig()
+
+    crawler = AsyncWebCrawler(config=browser_config)
+    await crawler.start()
+    await crawler.arun(url=ref_url, config=run_config )
+    await crawler.close()
+
+def test_crawl4ai(ref_url):
+    try:
+        run_async_function(test_crawl4ai_async, ref_url)
+    except Exception as e:
+        err = str(e)
+        print("Error! " + err[:75] + '..' * (len(err) > 75))
+
+async def test_playwright_async(url):
+    async with async_playwright() as playwright:
+        await asyncio.sleep(0)
+        chromium = playwright.chromium
+        browser = await chromium.launch(headless=True)
+        page = await browser.new_page()
+        await page.goto(url)
+        title = await page.title()
+        print(title)
+        await browser.close()
+
+def test_playwright(ref_url):
+    try:
+        run_async_function(test_playwright_async, ref_url)
+    except Exception as e:
+        err = str(e)
+        print("Error! " + err[:75] + '..' * (len(err) > 75))
+
+async def test_selenium_async(url):
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    driver = webdriver.Chrome(options)
+    driver.get(url)
+    print(driver.title)
+    driver.quit()
+
+def test_selenium(ref_url):
+    try:
+        run_async_function(test_selenium_async, ref_url)
+    except Exception as e:
+        err = str(e)
+        print("Error! " + err[:75] + '..' * (len(err) > 75))
+# endregion
+
+# region subprocess
+async def test_playwright_subprocess(ref_url):
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_playwright; test_playwright('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    return line
+
+async def test_selenium_subprocess(ref_url):
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_selenium; test_selenium('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    try:
+        await asyncio.wait(0)
+    except:
+        pass
+    return line
+
+async def test_chromium_subprocess(ref_url):
+    try:
+        exec = "/home/yoni/.cache/ms-playwright/chromium_headless_shell-1155/chrome-linux/headless_shell"
+        proc = await asyncio.create_subprocess_exec(
+            exec,
+            "--single-process",
+            "--no-sandbox",
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--headless",
+            "--mute-audio",
+            "--dump-dom",
+            ref_url,
+            stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+        data = await proc.stdout.readline()
+        line = data.decode('utf-8').rstrip()
+
+        await proc.wait()
+        if line == '':
+            return "No output."
+        return line
+    except:
+        return "Error."
+# endregion
+
+# region subprocess with unlimited resource
+async def test_playwright_subprocess_with_unlimited_limit(ref_url):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+    resource.prlimit(0, rlimit, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_playwright; test_playwright('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    return line
+
+def test_playwright_subprocess_with_unlimited_limit_sync(ref_url):
+    line = run_async_function(test_playwright_subprocess_with_unlimited_limit, ref_url)
+    print(line)
+
+async def test_playwright_sub_subprocess(ref_url):
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_playwright_subprocess_with_unlimited_limit_sync; test_playwright_subprocess_with_unlimited_limit_sync('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    return line
+
+async def test_selenium_subprocess_with_unlimited_limit(ref_url):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+    resource.prlimit(0, rlimit, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_selenium; test_selenium('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    return line
+
+def test_selenium_subprocess_with_unlimited_limit_sync(ref_url):
+    line = run_async_function(test_selenium_subprocess_with_unlimited_limit, ref_url)
+    print(line)
+
+async def test_selenium_sub_subprocess(ref_url):
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable, '-c', f"from test import test_selenium_subprocess_with_unlimited_limit_sync; test_selenium_subprocess_with_unlimited_limit_sync('{ref_url}')",
+        stdout=asyncio.subprocess.PIPE, cwd=CWD)
+
+    data = await proc.stdout.readline()
+    line = data.decode('utf-8').rstrip()
+
+    await proc.wait()
+    return line
+
+# endregion
+
+# region multiprocess
+def test_playwright_with_unlimited_limit_multiprocess_child(q, ref_url):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+    resource.prlimit(0, rlimit, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+    line = run_async_function(test_playwright_subprocess, ref_url)
+
+    q.put(line)
+
+def test_playwright_with_unlimited_limit_multiprocess_parent(ref_url):
+    ctx = get_context('spawn')
+    q = ctx.Queue()
+
+    p = Process(target=test_playwright_with_unlimited_limit_multiprocess_child, args=(q, ref_url))
+    p.start()
+    print(q.get())
+    p.join()
+
+def test_selenium_with_unlimited_limit_multiprocess_child(q, ref_url):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+    resource.prlimit(0, rlimit, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+
+    line = run_async_function(test_selenium_subprocess, ref_url)
+
+    q.put(line)
+
+def test_selenium_with_unlimited_limit_multiprocess_parent(ref_url):
+    ctx = get_context('spawn')
+    q = ctx.Queue()
+
+    p = Process(target=test_selenium_with_unlimited_limit_multiprocess_child, args=(q, ref_url))
+    p.start()
+    print(q.get())
+    p.join()
+# endregion
+
+def run_tests():
+    print("----------crawl4ai")
+    test_crawl4ai(TEST_WEBSITE)
+
+    print("----------playwright")
+    test_playwright(TEST_WEBSITE)
+
+    print("----------selenium")
+    test_selenium(TEST_WEBSITE)
+
+def run_tests_subprocess():
+    print("----------Playwright inside sub process")
+    line = run_async_function(test_playwright_subprocess, TEST_WEBSITE)
+    print(line)
+
+    print("----------Selenium inside sub process")
+    line = run_async_function(test_selenium_subprocess, TEST_WEBSITE)
+    print(line)
+
+    print("----------Chromium inside sub process")
+    line = run_async_function(test_chromium_subprocess, TEST_WEBSITE)
+    print(line)
+
+def run_tests_subprocess_with_limit(limit):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+    resource.setrlimit(rlimit, (limit, hard))
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit before: ", soft, " : ", hard)
+
+    run_tests_subprocess()
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    resource.setrlimit(rlimit, (soft, hard))
+
+def run_tests_with_unlimited_limit(limit):
+    rlimit = resource.RLIMIT_AS
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("----------Playwright inside sub process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    line = run_async_function(test_playwright_subprocess_with_unlimited_limit, TEST_WEBSITE)
+    print(line)
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    print("----------Playwright inside sub sub process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    line = run_async_function(test_playwright_sub_subprocess, TEST_WEBSITE)
+    print(line)
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    print("----------Playright inside multi process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    test_playwright_with_unlimited_limit_multiprocess_parent(TEST_WEBSITE)
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    print("----------Selenium inside sub process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    line = run_async_function(test_selenium_subprocess_with_unlimited_limit, TEST_WEBSITE)
+    print(line)
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    print("----------Selenium inside sub sub process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    line = run_async_function(test_selenium_sub_subprocess, TEST_WEBSITE)
+    print(line)
+
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+    print("----------Selenium inside multi process")
+    resource.setrlimit(rlimit, (limit, hard))
+    soft, hard = resource.getrlimit(rlimit)
+
+    print("Limit before: ", soft, " : ", hard)
+    test_playwright_with_unlimited_limit_multiprocess_parent(TEST_WEBSITE)
+    soft, hard = resource.getrlimit(rlimit)
+    print("Limit after: ", soft, " : ", hard)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("operation", choices=['base', 'sub', 'slim', 'slila', 'multi'])
+
+    args = parser.parse_args()
+    if args.operation == "base":
+        print("##### Base Test")
+        run_tests()
+    elif args.operation == "sub":
+        print("##### Subprocess Test")
+        run_tests_subprocess()
+    elif args.operation == "slim":
+        limit = 2684354560
+        print("##### Test with limit")
+        run_tests_subprocess_with_limit(limit)
+    elif args.operation == "slila":
+        # limit = 8589934592 #64
+        limit = 137438953472 #128
+        print("##### Test with limit")
+        run_tests_subprocess_with_limit(limit)
+    elif args.operation == "multi":
+        limit = 2684354560
+        print("##### Test with unlimited limit")
+        run_tests_with_unlimited_limit(limit)
+
+if __name__ == "__main__":
+    main()

scraper_test/wizard/__init__.py

@@ -0,0 +1 @@
+from . import scrap

scraper_test/wizard/async_utils.py

@@ -0,0 +1,11 @@
+import logging
+_logger = logging.getLogger(__name__)
+import asyncio
+
+def run_async_function(func_to_run, *args):
+    new_loop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(new_loop)
+        return new_loop.run_until_complete(func_to_run(*args))
+    finally:
+        new_loop.close()

scraper_test/wizard/crawl_utils.py

@@ -0,0 +1,53 @@
+import logging
+_logger = logging.getLogger(__name__)
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+from playwright.async_api import async_playwright
+from selenium import webdriver
+
+from . import async_utils
+
+async def test_crawl4ai_async(ref_url):
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig()
+
+    crawler = AsyncWebCrawler(config=browser_config)
+    await crawler.start()
+    await crawler.arun(
+        url=ref_url,
+        config=run_config
+    )
+    await crawler.close()
+
+    
+def test_crawl4ai(ref_url):
+    async_utils.run_async_function(test_crawl4ai_async, ref_url)
+        
+async def test_playwright_async(url):
+    async with async_playwright() as playwright:
+        await asyncio.sleep(0)
+        chromium = playwright.chromium
+        browser = await chromium.launch(
+            timeout=0,
+            headless=True,
+            traces_dir="traces"
+        )
+        page = await browser.new_page()
+        await page.goto(url)
+        await page.title()
+        await browser.close()
+                
+def test_playwright(ref_url):
+    async_utils.run_async_function(test_playwright_async, ref_url)
+    
+async def test_selenium_async(url):
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    driver = webdriver.Chrome(options)
+    driver.get(url)
+    driver.quit()
+                
+def test_selenium(ref_url):
+    async_utils.run_async_function(test_selenium_async, ref_url)

scraper_test/wizard/scrap.py

@@ -0,0 +1,79 @@
+import logging
+
+_logger = logging.getLogger(__name__)
+
+import re
+from odoo import _, models, fields
+
+from .crawl_utils import test_playwright, test_selenium, test_crawl4ai
+
+class ScraperWizard(models.TransientModel):
+    _name = "scrap.wizard"
+    _description = "Scraper Test"
+
+    url = fields.Char(
+        "Url",
+        required=True,
+        help="Url",
+        default="http://localhost:8069/ai_doc/static/html/index.html",
+    )
+    
+    def _get_result(self, message = None):
+        if not message:
+            res = {
+                'type': 'ir.actions.act_window',
+                'name': "Scraper Test",
+                'target': 'new',
+                'view_mode': 'form',
+                'res_model': 'scrap.wizard',
+                'res_id': self.id,
+            }
+        if message:
+            res = {
+                'type': 'ir.actions.client',
+                'tag': 'display_notification',
+                'params': {
+                    'type': 'info',
+                    'title': 'Result',
+                    'message': message
+                },
+            }
+        
+        return res
+        
+    def action_test_crawl4ai(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_crawl4ai(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)
+
+
+    def action_test_playwright(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_playwright(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)
+
+    def action_test_selenium(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_selenium(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)

scraper_test/wizard/scrap.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record id="scrap_view_form" model="ir.ui.view">
+        <field name="name">scrap.wizard.view.form</field>
+        <field name="model">scrap.wizard</field>
+        <field name="arch" type="xml">
+            <form>
+                <group>
+                    <field name="id"/>
+                    <field name="url"/>
+                </group>
+                <footer>
+                    <button name="action_test_crawl4ai" string="Crawl4AI" type="object" class="oe_highlight"/>
+                    <button name="action_test_playwright" string="Playwright" type="object" class="oe_highlight"/>
+                    <button name="action_test_selenium" string="Selenium" type="object" class="oe_highlight"/>
+                    <button string="Close" class="oe_link" special="cancel"/>
+                </footer>
+            </form>
+        </field>
+    </record>
+
+    <record id="action_scrap_test" model="ir.actions.act_window">
+        <field name="name">Scraper Test</field>
+        <field name="type">ir.actions.act_window</field>
+        <field name="res_model">scrap.wizard</field>
+        <field name="view_mode">form</field>
+        <field name="target">new</field>
+        <field name="view_id" ref="scrap_view_form"/>
+    </record>
+
+    <menuitem name="Scraper Test" id="menu_doc_ref" parent="base.menu_custom" sequence="9000"
+        action="action_scrap_test"/>
+
+</odoo>