import requests
from bs4 import BeautifulSoup
import time

def extract_product(url):
    headers = {
        "User-Agent": "GrahamMirandaBot/1.0 (+https://grahammiranda.com/scraping-policy)"
    }
    response = requests.get(url, headers=headers, timeout=30)
    soup = BeautifulSoup(response.text, "html.parser")

    return {
        "title": soup.select_one("h1.product-title").get_text(strip=True),
        "price": soup.select_one("span.price").get_text(strip=True),
        "availability": soup.select_one("div.availability").get_text(strip=True),
        "url": url,
        "scraped_at": time.strftime("%Y-%m-%d %H:%M:%S")
    }

# Usage
for url in product_urls:
    data = extract_product(url)
    save_to_database(data)
    time.sleep(1.0)  # Respectful rate limiting

from playwright.sync_api import sync_playwright
import time

def extract_dynamic(url):
    with sync_playwright() as p:
        browser = p.firefox.launch(headless=True)
        page = browser.new_page()

        # Anti-detection
        page.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
        """)

        page.goto(url, wait_until="networkidle")
        time.sleep(2)  # Wait for JS hydration

        # Extract after render
        title = page.locator("h1").inner_text()
        price = page.locator("[data-testid='price']").inner_text()

        browser.close()
        return {"title": title, "price": price, "url": url}

import requests

# Method 1: Check page source for API calls
response = requests.get("https://example.com/products")
# Look for `window.__INITIAL_STATE__` or similar JS variables

# Method 2: Network inspection (manual, then script)
# Open DevTools → Network → XHR → find the API endpoint
# Usually: /api/v1/products, /graphql, /json/products

# Method 3: Check for GraphQL endpoints
# Common patterns: /graphql, /api/graphql, /gql

from celery import Celery
import redis

app = Celery('web_intel', broker='redis://localhost:6379/0')
redis_client = redis.Redis(host='localhost', port=6379, db=1)

@app.task
def scrape_url(url):
    # Your extraction logic here
    data = extract_product(url)
    redis_client.hset(f"product:{url}", mapping=data)
    return data

# Queue 10,000 URLs
for url in url_list:
    scrape_url.delay(url)

# PDF extraction
import fitz  # PyMuPDF

def extract_pdf_text(url):
    response = requests.get(url)
    doc = fitz.open(stream=response.content, filetype="pdf")
    text = "\n".join(page.get_text() for page in doc)
    return text

# DOCX extraction
from docx import Document

def extract_docx_text(filepath):
    doc = Document(filepath)
    return "\n".join(p.text for p in doc.paragraphs)

def robust_extract(url, max_retries=3):
    for attempt in range(max_retries):
        try:
            return extract_product(url)
        except requests.exceptions.Timeout:
            time.sleep(5 * (attempt + 1))  # Exponential backoff
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 429:
                time.sleep(60)  # Rate limited — long wait
            elif e.response.status_code == 403:
                return {"error": "blocked", "url": url}
            else:
                raise
        except Exception as e:
            if attempt == max_retries - 1:
                return {"error": str(e), "url": url}
            time.sleep(2 ** attempt)

    return {"error": "max_retries_exceeded", "url": url}

import time
from functools import wraps

def rate_limited(max_per_second=1.0):
    min_interval = 1.0 / max_per_second
    last_called = [0.0]

    def decorator(fn):
        @wraps(fn)
        def wrapper(*args, **kwargs):
            elapsed = time.time() - last_called[0]
            if elapsed < min_interval:
                time.sleep(min_interval - elapsed)
            result = fn(*args, **kwargs)
            last_called[0] = time.time()
            return result
        return wrapper
    return decorator

@rate_limited(max_per_second=0.5)  # 1 request per 2 seconds
def fetch_url(url):
    return requests.get(url, timeout=30)