main.py

import asyncio
import json
from extractor_engine import extract_all
from smart_fetcher import smart_fetch


async def run_pipeline(urls):
    results = []
    for url in urls:
        page = await smart_fetch(url)
        if page["status"] == "ok":
            extracted = extract_all(page["content"], url)
            results.append(extracted)
        else:
            results.append({"url": url, "status": page["status"]})
    with open("data/final_output.json", "w", encoding="utf-8") as f:
        json.dump(results, f, indent=2, ensure_ascii=False)

asyncio.run(run_pipeline([
    "https://www.vsa-nj.com/about-us",
    "https://fcmalagacity.com/about/",
    "https://www.spanishprofootball.com/"
]))