update

scraper_test/wizard/__init__.py

@@ -0,0 +1 @@
+from . import scrap

scraper_test/wizard/async_utils.py

@@ -0,0 +1,11 @@
+import logging
+_logger = logging.getLogger(__name__)
+import asyncio
+
+def run_async_function(func_to_run, *args):
+    new_loop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(new_loop)
+        return new_loop.run_until_complete(func_to_run(*args))
+    finally:
+        new_loop.close()

scraper_test/wizard/crawl_utils.py

@@ -0,0 +1,53 @@
+import logging
+_logger = logging.getLogger(__name__)
+
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+from playwright.async_api import async_playwright
+from selenium import webdriver
+
+from . import async_utils
+
+async def test_crawl4ai_async(ref_url):
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig()
+
+    crawler = AsyncWebCrawler(config=browser_config)
+    await crawler.start()
+    await crawler.arun(
+        url=ref_url,
+        config=run_config
+    )
+    await crawler.close()
+
+    
+def test_crawl4ai(ref_url):
+    async_utils.run_async_function(test_crawl4ai_async, ref_url)
+        
+async def test_playwright_async(url):
+    async with async_playwright() as playwright:
+        await asyncio.sleep(0)
+        chromium = playwright.chromium
+        browser = await chromium.launch(
+            timeout=0,
+            headless=True,
+            traces_dir="traces"
+        )
+        page = await browser.new_page()
+        await page.goto(url)
+        await page.title()
+        await browser.close()
+                
+def test_playwright(ref_url):
+    async_utils.run_async_function(test_playwright_async, ref_url)
+    
+async def test_selenium_async(url):
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    driver = webdriver.Chrome(options)
+    driver.get(url)
+    driver.quit()
+                
+def test_selenium(ref_url):
+    async_utils.run_async_function(test_selenium_async, ref_url)

scraper_test/wizard/scrap.py

@@ -0,0 +1,79 @@
+import logging
+
+_logger = logging.getLogger(__name__)
+
+import re
+from odoo import _, models, fields
+
+from .crawl_utils import test_playwright, test_selenium, test_crawl4ai
+
+class ScraperWizard(models.TransientModel):
+    _name = "scrap.wizard"
+    _description = "Scraper Test"
+
+    url = fields.Char(
+        "Url",
+        required=True,
+        help="Url",
+        default="http://localhost:8069/ai_doc/static/html/index.html",
+    )
+    
+    def _get_result(self, message = None):
+        if not message:
+            res = {
+                'type': 'ir.actions.act_window',
+                'name': "Scraper Test",
+                'target': 'new',
+                'view_mode': 'form',
+                'res_model': 'scrap.wizard',
+                'res_id': self.id,
+            }
+        if message:
+            res = {
+                'type': 'ir.actions.client',
+                'tag': 'display_notification',
+                'params': {
+                    'type': 'info',
+                    'title': 'Result',
+                    'message': message
+                },
+            }
+        
+        return res
+        
+    def action_test_crawl4ai(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_crawl4ai(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)
+
+
+    def action_test_playwright(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_playwright(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)
+
+    def action_test_selenium(self):
+        msg = "Ok!"
+        try:
+            text = self.url
+            test_selenium(text)
+        except Exception as e:
+            _logger.error(e);
+            err = str(e)
+            msg = "Error! " + err[:75] + '..' * (len(err) > 75)
+
+        return self._get_result(msg)

scraper_test/wizard/scrap.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record id="scrap_view_form" model="ir.ui.view">
+        <field name="name">scrap.wizard.view.form</field>
+        <field name="model">scrap.wizard</field>
+        <field name="arch" type="xml">
+            <form>
+                <group>
+                    <field name="id"/>
+                    <field name="url"/>
+                </group>
+                <footer>
+                    <button name="action_test_crawl4ai" string="Crawl4AI" type="object" class="oe_highlight"/>
+                    <button name="action_test_playwright" string="Playwright" type="object" class="oe_highlight"/>
+                    <button name="action_test_selenium" string="Selenium" type="object" class="oe_highlight"/>
+                    <button string="Close" class="oe_link" special="cancel"/>
+                </footer>
+            </form>
+        </field>
+    </record>
+
+    <record id="action_scrap_test" model="ir.actions.act_window">
+        <field name="name">Scraper Test</field>
+        <field name="type">ir.actions.act_window</field>
+        <field name="res_model">scrap.wizard</field>
+        <field name="view_mode">form</field>
+        <field name="target">new</field>
+        <field name="view_id" ref="scrap_view_form"/>
+    </record>
+
+    <menuitem name="Scraper Test" id="menu_doc_ref" parent="base.menu_custom" sequence="9000"
+        action="action_scrap_test"/>
+
+</odoo>