diff --git a/docs/browser/ASCII_BROWSER.md b/docs/browser/ASCII_BROWSER.md new file mode 100644 index 00000000..eaaf8f4e --- /dev/null +++ b/docs/browser/ASCII_BROWSER.md @@ -0,0 +1,55 @@ +# ASCII Browser Integration + +This document describes the ASCII browser integration in Devika using w3m. + +## Overview + +The ASCII browser implementation provides a lightweight alternative to API-based web access and screenshot-based browsing. It uses w3m to render web pages in ASCII format, which is both efficient and suitable for LLM processing. + +## Requirements + +- w3m must be installed on the system (`apt-get install w3m`) +- Python 3.6+ with subprocess module + +## Usage + +```python +from src.browser.w3m_browser import W3MBrowser + +# Initialize browser +browser = W3MBrowser() + +# Navigate to a URL +success, content = browser.navigate("http://example.com") + +# Get current content +current_content = browser.get_current_content() +``` + +## Benefits + +1. No API costs or rate limits +2. No third-party dependencies for basic web access +3. Efficient text-based content suitable for LLM processing +4. Reduced bandwidth usage compared to full browser rendering + +## Limitations + +1. No JavaScript support +2. Limited rendering of complex layouts +3. No image support in ASCII mode + +## Error Handling + +The browser implementation includes robust error handling for: +- Missing w3m installation +- Navigation failures +- Invalid URLs + +## Installation + +To install w3m on Ubuntu/Debian: +```bash +sudo apt-get update +sudo apt-get install w3m +``` diff --git a/requirements.txt b/requirements.txt index 91666960..bf522945 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ flask flask-cors toml urllib3 -requests +requests>=2.31.0 colorama fastlogging Jinja2 @@ -12,7 +12,7 @@ pdfminer.six playwright pytest-playwright tiktoken -ollama +ollama>=0.1.6 openai anthropic google-generativeai @@ -31,3 +31,5 @@ orjson gevent gevent-websocket curl_cffi +pytest>=7.4.0 +pytest-mock>=3.12.0 diff --git a/src/browser/w3m_browser.py b/src/browser/w3m_browser.py new file mode 100644 index 00000000..de74ade3 --- /dev/null +++ b/src/browser/w3m_browser.py @@ -0,0 +1,55 @@ +""" +ASCII-based browser implementation using w3m. + +This module provides a lightweight terminal-based browser implementation +using w3m for web access without requiring expensive API calls or screenshots. +""" +import subprocess +import logging +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + +class W3MBrowser: + """Terminal-based browser using w3m.""" + + def __init__(self): + """Initialize the W3M browser wrapper.""" + self._verify_w3m_installation() + self.current_url: Optional[str] = None + self.current_content: Optional[str] = None + + def _verify_w3m_installation(self) -> None: + """Verify w3m is installed and accessible.""" + try: + subprocess.run(['w3m', '-version'], + check=True, + capture_output=True, + text=True) + except (subprocess.CalledProcessError, FileNotFoundError) as e: + raise RuntimeError("w3m is not installed or not accessible") from e + + def navigate(self, url: str) -> Tuple[bool, str]: + """Navigate to a URL and return the page content in ASCII format.""" + self.current_url = url + try: + result = subprocess.run( + ['w3m', '-dump', url], + check=True, + capture_output=True, + text=True + ) + self.current_content = result.stdout + return True, self.current_content + except subprocess.CalledProcessError as e: + error_msg = f"Failed to load URL {url}: {str(e)}" + logger.error(error_msg) + return False, error_msg + + def get_current_content(self) -> Optional[str]: + """Get the current page content.""" + return self.current_content + + def get_current_url(self) -> Optional[str]: + """Get the current URL.""" + return self.current_url diff --git a/tests/test_w3m_browser.py b/tests/test_w3m_browser.py new file mode 100644 index 00000000..2977fc20 --- /dev/null +++ b/tests/test_w3m_browser.py @@ -0,0 +1,54 @@ +"""Tests for the W3M browser implementation.""" +import pytest +from unittest.mock import patch, MagicMock +import subprocess +from src.browser.w3m_browser import W3MBrowser + +def test_w3m_browser_init(): + """Test W3MBrowser initialization.""" + with patch('subprocess.run') as mock_run: + mock_run.return_value = MagicMock(returncode=0) + browser = W3MBrowser() + assert browser.get_current_url() is None + assert browser.get_current_content() is None + mock_run.assert_called_once() + +def test_w3m_browser_init_failure(): + """Test W3MBrowser initialization failure.""" + with patch('subprocess.run') as mock_run: + mock_run.side_effect = FileNotFoundError() + with pytest.raises(RuntimeError, match="w3m is not installed"): + W3MBrowser() + +def test_navigate_success(): + """Test successful navigation.""" + with patch('subprocess.run') as mock_run: + # Mock successful initialization + mock_run.return_value = MagicMock(returncode=0) + browser = W3MBrowser() + + # Mock successful navigation + mock_run.return_value = MagicMock( + returncode=0, + stdout="Test Content" + ) + + success, content = browser.navigate("http://example.com") + assert success is True + assert content == "Test Content" + assert browser.get_current_url() == "http://example.com" + assert browser.get_current_content() == "Test Content" + +def test_navigate_failure(): + """Test navigation failure.""" + with patch('subprocess.run') as mock_run: + # Mock successful initialization + mock_run.return_value = MagicMock(returncode=0) + browser = W3MBrowser() + + # Mock failed navigation + mock_run.side_effect = subprocess.CalledProcessError(1, 'w3m') + + success, content = browser.navigate("http://invalid.example") + assert success is False + assert "Failed to load URL" in content