smart_fetcher.py

import asyncio
from playwright.async_api import async_playwright
import requests
from fake_useragent import UserAgent
import urllib.robotparser as rp
from urllib.parse import urljoin
import json

ua = UserAgent()

def robots_allowed(url):
    base = "/".join(url.split("/")[:3])
    parser = rp.RobotFileParser()
    parser.set_url(urljoin(base, "/robots.txt"))
    try:
        parser.read()
        return parser.can_fetch("*", url)
    except:
        return True

async def fetch_dynamic(url, timeout=20):
    try:
        async with async_playwright() as p:
            browser = await p.chromium.launch(headless=True)
            page = await browser.new_page(user_agent=ua.random)
            await page.goto(url, timeout=timeout * 1000)
            html = await page.content()
            await browser.close()
            return html
    except Exception as e:
        print(f"[JS Fetch ERROR] {url}: {e}")
        return None

def fetch_static(url, timeout=10):
    try:
        headers = {"User-Agent": ua.random}
        r = requests.get(url, headers=headers, timeout=timeout)
        if r.status_code == 200:
            return r.text
    except Exception as e:
        print(f"[Static Fetch ERROR] {url}: {e}")
    return None

async def smart_fetch(url):
    if not robots_allowed(url):
        return {"url": url, "status": "blocked_by_robots", "content": None}

    html = fetch_static(url)
    if html:
        return {"url": url, "status": "ok", "method": "static", "content": html}

    html = await fetch_dynamic(url)
    if html:
        return {"url": url, "status": "ok", "method": "js_rendered", "content": html}

    return {"url": url, "status": "failed", "content": None}