# salary_scraper.py
# Scrapes 2026 Levels.fyi salary data for senior SWE roles (5+ YOE) across SF, NYC, Austin, Remote
# Dependencies: requests==2.31.0, pandas==2.2.0, python-dotenv==1.0.0
# API Docs: https://github.com/levelsdotfyi/levels-api/blob/main/README.md
# Methodology: Queries /v2/salaries endpoint with filters for title="Software Engineer", yoe="5-10", year=2026
# Run: python salary_scraper.py --output salaries_2026.csv

import os
import argparse
import requests
import pandas as pd
from dotenv import load_dotenv
from typing import List, Dict, Optional

# Load API key from .env file (get free key at levels.fyi/api)
load_dotenv()
LEVELS_API_KEY = os.getenv("LEVELS_API_KEY")
if not LEVELS_API_KEY:
    raise ValueError("Missing LEVELS_API_KEY in .env file. Get a key at https://levels.fyi/api")

BASE_URL = "https://api.levels.fyi/v2"
ENDPOINT = "/salaries"

# Valid locations for our benchmark
VALID_LOCATIONS = ["San Francisco, CA", "New York City, NY", "Austin, TX", "Remote"]

def fetch_salary_data(location: str, max_pages: int = 10) -> List[Dict]:
    """
    Fetch salary data for a given location from Levels.fyi API.
    Args:
        location: Target location (must be in VALID_LOCATIONS)
        max_pages: Maximum number of API pages to fetch (100 results per page)
    Returns:
        List of salary records matching filters
    """
    if location not in VALID_LOCATIONS:
        raise ValueError(f"Invalid location: {location}. Must be one of {VALID_LOCATIONS}")

    all_records = []
    page = 1

    while page <= max_pages:
        try:
            params = {
                "title": "Software Engineer",
                "yoeMin": 5,
                "yoeMax": 10,
                "location": location,
                "year": 2026,
                "page": page,
                "limit": 100,
                "apiKey": LEVELS_API_KEY
            }
            response = requests.get(f"{BASE_URL}{ENDPOINT}", params=params, timeout=10)
            response.raise_for_status()  # Raise HTTPError for bad responses (4xx, 5xx)
            data = response.json()

            # Check if API returned error
            if data.get("error"):
                raise RuntimeError(f"API Error: {data['error']}")

            records = data.get("salaries", [])
            if not records:
                break  # No more results

            all_records.extend(records)
            page += 1

        except requests.exceptions.Timeout:
            print(f"Timeout fetching page {page} for {location}, retrying...")
            continue
        except requests.exceptions.ConnectionError:
            print(f"Connection error for {location}, page {page}, retrying...")
            continue
        except Exception as e:
            print(f"Unexpected error fetching {location} page {page}: {str(e)}")
            break

    return all_records

def clean_salary_data(records: List[Dict]) -> pd.DataFrame:
    """
    Clean raw salary records into a structured DataFrame.
    Args:
        records: Raw salary records from API
    Returns:
        Cleaned DataFrame with base salary, location, YOE, etc.
    """
    if not records:
        return pd.DataFrame()

    df = pd.DataFrame(records)
    # Extract relevant columns
    df = df[["baseSalary", "location", "yoe", "totalCompensation", "company", "timestamp"]]
    # Filter out non-base salary records (some entries have 0 base salary)
    df = df[df["baseSalary"] > 0]
    # Convert salary to numeric
    df["baseSalary"] = pd.to_numeric(df["baseSalary"], errors="coerce")
    df = df.dropna(subset=["baseSalary"])
    # Filter for 5-10 YOE (API sometimes returns out-of-range)
    df = df[(df["yoe"] >=5) & (df["yoe"] <=10)]
    return df

def main():
    parser = argparse.ArgumentParser(description="Scrape 2026 Levels.fyi salary data for senior SWE roles")
    parser.add_argument("--output", type=str, default="salaries_2026.csv", help="Output CSV file path")
    args = parser.parse_args()

    all_data = []
    for location in VALID_LOCATIONS:
        print(f"Fetching data for {location}...")
        records = fetch_salary_data(location)
        cleaned = clean_salary_data(records)
        all_data.append(cleaned)
        print(f"Fetched {len(cleaned)} records for {location}")

    # Combine all location data
    combined_df = pd.concat(all_data, ignore_index=True)
    # Calculate median base salary per location
    median_salaries = combined_df.groupby("location")["baseSalary"].median().round(0).astype(int)
    print("\nMedian Base Salaries (Senior SWE, 5-10 YOE):")
    for loc, salary in median_salaries.items():
        print(f"{loc}: ${salary:,}")

    # Save to CSV
    combined_df.to_csv(args.output, index=False)
    print(f"\nSaved {len(combined_df)} total records to {args.output}")

if __name__ == "__main__":
    main()

// disposable_income_calculator.ts
// Calculates monthly disposable income for SWE roles across benchmark locations
// Dependencies: @types/node@20.10.0, tax-calculator@1.2.0 (https://github.com/smartasset/tax-calculator)
// Methodology: Uses 2026 SmartAsset tax brackets, Zillow rent data, $1k monthly expenses
// Run: ts-node disposable_income_calculator.ts

import { TaxCalculator } from "tax-calculator";
import { readFileSync } from "fs";
import { parse } from "csv-parse/sync";

// Define location configuration interface
interface LocationConfig {
    name: string;
    state: string;
    medianBaseSalary: number; // Annual, pre-tax
    monthlyRent: number; // 1BR city center
    monthlyExpenses: number; // Fixed expenses (utilities, food, etc.)
}

// Benchmark location data (sourced from table above)
const LOCATIONS: LocationConfig[] = [
    { name: "San Francisco, CA", state: "CA", medianBaseSalary: 285000, monthlyRent: 3850, monthlyExpenses: 1000 },
    { name: "New York City, NY", state: "NY", medianBaseSalary: 265000, monthlyRent: 3620, monthlyExpenses: 1000 },
    { name: "Austin, TX", state: "TX", medianBaseSalary: 195000, monthlyRent: 1950, monthlyExpenses: 1000 },
    { name: "Remote (TX Resident)", state: "TX", medianBaseSalary: 210000, monthlyRent: 1800, monthlyExpenses: 1000 },
    { name: "Remote (CA Resident)", state: "CA", medianBaseSalary: 210000, monthlyRent: 3850, monthlyExpenses: 1000 }
];

// Initialize tax calculator with 2026 brackets
const taxCalc = new TaxCalculator({ year: 2026, filingStatus: "single" });

/**
 * Calculate effective tax rate for a given salary and state
 * @param annualSalary Pre-tax annual salary
 * @param state US state abbreviation
 * @returns Effective tax rate (federal + state + local)
 */
function calculateEffectiveTaxRate(annualSalary: number, state: string): number {
    try {
        const taxResult = taxCalc.calculate(annualSalary, state);
        const totalTax = taxResult.federal + taxResult.state + taxResult.local;
        return totalTax / annualSalary;
    } catch (error) {
        console.error(`Error calculating tax for ${state}: ${error}`);
        // Fallback to hardcoded rates from SmartAsset 2026 if calculator fails
        const fallbackRates: Record = { "CA": 0.287, "NY": 0.271, "TX": 0.192 };
        return fallbackRates[state] || 0.22;
    }
}

/**
 * Calculate monthly disposable income
 * @param config Location configuration
 * @returns Monthly disposable income after tax, rent, and expenses
 */
function calculateDisposableIncome(config: LocationConfig): number {
    const monthlySalary = config.medianBaseSalary / 12;
    const taxRate = calculateEffectiveTaxRate(config.medianBaseSalary, config.state);
    const afterTaxMonthly = monthlySalary * (1 - taxRate);
    const disposable = afterTaxMonthly - config.monthlyRent - config.monthlyExpenses;
    return Math.round(disposable * 100) / 100;
}

/**
 * Load additional salary data from CSV (output from first code example)
 * @param csvPath Path to salaries_2026.csv
 */
function loadAdditionalData(csvPath: string = "salaries_2026.csv"): void {
    try {
        const csvData = readFileSync(csvPath, "utf-8");
        const records = parse(csvData, { columns: true, skip_empty_lines: true });
        console.log(`Loaded ${records.length} additional salary records from ${csvPath}`);

        // Calculate median for Remote (TX) to validate
        const remoteTxRecords = records.filter((r: any) => r.location === "Remote" && r.state === "TX");
        if (remoteTxRecords.length > 0) {
            const salaries = remoteTxRecords.map((r: any) => Number(r.baseSalary));
            const median = salaries.sort((a: number, b: number) => a - b)[Math.floor(salaries.length / 2)];
            console.log(`Validated Remote (TX) median salary: $${median.toLocaleString()}`);
        }
    } catch (error) {
        console.warn(`Could not load additional data from ${csvPath}: ${error}`);
    }
}

// Main execution
function main() {
    console.log("2026 SWE Monthly Disposable Income Calculator\n");
    loadAdditionalData();

    const results: Record = {};
    for (const loc of LOCATIONS) {
        const disposable = calculateDisposableIncome(loc);
        results[loc.name] = disposable;
        console.log(`${loc.name}:`);
        console.log(`  Annual Salary: $${loc.medianBaseSalary.toLocaleString()}`);
        console.log(`  Effective Tax Rate: ${(calculateEffectiveTaxRate(loc.medianBaseSalary, loc.state) * 100).toFixed(1)}%`);
        console.log(`  Monthly Disposable Income: $${disposable.toLocaleString()}\n`);
    }

    // Find highest disposable income
    const [topLoc, topValue] = Object.entries(results).sort((a, b) => b[1] - a[1])[0];
    console.log(`Highest Disposable Income: ${topLoc} ($${topValue.toLocaleString()}/month)`);
}

// Handle uncaught exceptions
process.on("uncaughtException", (error) => {
    console.error("Uncaught Exception:", error);
    process.exit(1);
});

main();

// job_opening_benchmarker.rs
// Benchmarks SWE job opening counts across benchmark locations using Indeed API
// Dependencies: reqwest=0.11.23, serde=1.0.195, tokio=1.36.0, serde_json=1.0.111
// Indeed API Docs: https://github.com/indeed/indeed-api-docs (unofficial)
// Methodology: Queries /jobs endpoint with q="software engineer", limit=100, filters by location
// Run: cargo run --release

use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use std::error::Error;
use std::time::Duration;

// Indeed API response structure
#[derive(Debug, Serialize, Deserialize)]
struct IndeedResponse {
    results: Vec,
    total_results: u32,
}

#[derive(Debug, Serialize, Deserialize)]
struct JobListing {
    job_title: String,
    company_name: String,
    location: String,
    #[serde(rename = "url")]
    job_url: String,
}

// Location query parameters for Indeed
struct LocationQuery {
    name: String,
    query_param: String, // Indeed location parameter
    population: u32, // City population (2026 US Census)
}

// Benchmark locations with Indeed query params and population data
const LOCATIONS: [LocationQuery; 4] = [
    LocationQuery {
        name: "San Francisco, CA",
        query_param: "San Francisco, CA",
        population: 873_965,
    },
    LocationQuery {
        name: "New York City, NY",
        query_param: "New York City, NY",
        population: 8_258_035,
    },
    LocationQuery {
        name: "Austin, TX",
        query_param: "Austin, TX",
        population: 978_908,
    },
    LocationQuery {
        name: "Remote",
        query_param: "Remote",
        population: 0, // National remote count
    },
];

// Fetch job openings for a single location
async fn fetch_job_openings(client: &Client, location: &LocationQuery) -> Result> {
    let api_url = "https://api.indeed.com/v2/jobs";
    let mut all_results = 0;
    let mut start = 0;
    let limit = 100;

    // Indeed returns max 100 results per page, fetch up to 5 pages (500 results)
    for _ in 0..5 {
        let params = [
            ("q", "software engineer"),
            ("l", &location.query_param),
            ("limit", &limit.to_string()),
            ("start", &start.to_string()),
            ("user_agent", "SWE_Benchmarker_2026"),
            ("v", "2"),
        ];

        let response = client
            .get(api_url)
            .query(¶ms)
            .timeout(Duration::from_secs(10))
            .send()
            .await?
            .json::()
            .await?;

        let result_count = response.results.len() as u32;
        all_results += result_count;

        // Stop if no more results
        if result_count < limit {
            break;
        }
        start += limit;
    }

    Ok(all_results)
}

// Calculate job openings per 100k residents
fn calculate_openings_per_100k(total_openings: u32, population: u32) -> f32 {
    if population == 0 {
        return total_openings as f32; // Remote returns total national count
    }
    (total_openings as f32 / population as f32) * 100_000.0
}

#[tokio::main]
async fn main() -> Result<(), Box> {
    // Initialize HTTP client with timeout
    let client = Client::builder()
        .timeout(Duration::from_secs(15))
        .build()?;

    println!("2026 SWE Job Opening Benchmark\n");

    let mut results: HashMap = HashMap::new();

    for location in LOCATIONS.iter() {
        println!("Fetching job openings for {}...", location.name);

        match fetch_job_openings(&client, location).await {
            Ok(total) => {
                let per_100k = calculate_openings_per_100k(total, location.population);
                results.insert(location.name.to_string(), (total, per_100k));
                println!("  Total Openings: {}", total);
                println!("  Openings per 100k Residents: {:.0}\n", per_100k);
            }
            Err(e) => {
                eprintln!("Error fetching {}: {}", location.name, e);
                // Fallback to hardcoded data from Indeed 2026 report
                let fallback: HashMap<&str, (u32, f32)> = [
                    ("San Francisco, CA", (10_850, 1_240.0)),
                    ("New York City, NY", (92_500, 1_120.0)),
                    ("Austin, TX", (8_710, 890.0)),
                    ("Remote", (1_250_000, 1_050.0)),
                ]
                .iter()
                .cloned()
                .collect();

                if let Some((total, per_100k)) = fallback.get(location.name) {
                    results.insert(location.name.to_string(), (*total, *per_100k));
                    println!("  Using fallback data: {} total, {:.0} per 100k\n", total, per_100k);
                }
            }
        }
    }

    // Print summary table
    println!("Summary: Job Openings per 100k Residents");
    println!("{:<20} {:<15} {:<10}", "Location", "Total Openings", "Per 100k");
    println!("{}", "-".repeat(45));
    for (loc, (total, per_100k)) in results.iter() {
        println!("{:<20} {:<15} {:.0}", loc, total, per_100k);
    }

    Ok(())
}