arxiv-scholar/
├── main.py                          # Full ingestion pipeline orchestrator
├── app.py                           # Streamlit chat UI
├── configs/
│   └── config.py                    # Centralized env-var-backed configuration
├── src/arxiv_scholar/
│   ├── schema.py                    # Core data models (Document, Chunk)
│   ├── api/
│   │   ├── schema.py                # REST API request/response models (SSE events)
│   │   └── server.py                # FastAPI streaming endpoint (POST /api/v1/query)
│   ├── chunking/
│   │   ├── base.py                  # Abstract BaseChunker interface
│   │   ├── layout.py                # Docling-based layout-aware chunker
│   │   └── sliding_window.py        # Fixed-size sliding window fallback chunker
│   ├── download/
│   │   └── arxiv_ingestion.py       # GCS-backed PDF downloader with cursor state
│   ├── embedding/
│   │   ├── base.py                  # Abstract BaseEmbedder interface
│   │   ├── fastembed_embedder.py    # ONNX CPU embedder (dense + sparse BM25)
│   │   └── st_embedder.py           # SentenceTransformer embedder (GPU)
│   ├── ingestion/
│   │   ├── base.py                  # Abstract DocumentReader interface
│   │   ├── local.py                 # Local filesystem PDF reader (PyMuPDF)
│   │   └── gcs.py                   # In-memory GCS bucket reader (serverless)
│   ├── llm/
│   │   └── service.py               # LLM client (decomposition, HyDE, synthesis)
│   ├── retrieval/
│   │   ├── retrieval.py             # Hybrid retriever with weighted fusion
│   │   ├── orchestrator.py          # Query orchestrator (routes → retrieves → fuses)
│   │   └── router.py                # ML + heuristic query router
│   └── storage/
│       ├── base.py                  # Abstract BaseVectorStore interface
│       └── qdrant_store.py          # Qdrant client (upsert, search, hybrid search)
├── scripts/
│   ├── generate_arxiv_manifest.py   # Paper selection criteria & manifest generator
│   ├── download_qdrant.sh           # Qdrant binary installer
│   ├── generate_eval_dataset.py     # Evaluation dataset generator
│   ├── run_benchmarks.py            # Retrieval benchmark runner
│   └── ...                          # Various ingestion and import utilities
├── colab/
│   ├── batch_gcs_to_drive.py        # Colab script: batch download from GCS
│   └── generate_embedded_dataset.py # Colab script: embed and push to Qdrant Cloud
├── notebooks/                       # Jupyter notebooks for development & testing
├── tests/                           # Unit and integration tests
├── docs/                            # GitHub Pages website
├── Dockerfile                       # Production container (HF Spaces / Cloud Run)
├── docker-compose.yml               # Local Qdrant service definition
└── pyproject.toml                   # Project metadata and dependencies

# Clone the repository
git clone https://github.com/dubeyaayush07/arxiv-scholar.git
cd arxiv-scholar

# Create a virtual environment and install dependencies
uv venv && source .venv/bin/activate
uv pip install -e .

# Required — Qdrant Cloud connection
export QDRANT_URL="your_qdrant_cloud_url"
export QDRANT_API_KEY="your_qdrant_api_key"
export QDRANT_COLLECTION="Arxiv-Scholar"

# Required for LLM features (decomposition, HyDE, answer synthesis)
export LLM_API_KEY="your_key_here"
export LLM_BASE_URL="https://generativelanguage.googleapis.com/v1beta/openai/"  # or any OpenAI-compatible endpoint
export LLM_MODEL="claude-haiku-4-5"

# Optional overrides (defaults shown)
export EMBEDDING_BACKEND="fastembed"            # or "sentence-transformers"
export EMBEDDING_MODEL="BAAI/bge-m3"
export SPARSE_EMBEDDING_MODEL="Qdrant/bm25"
export USE_RERANKER="False"                     # Disabled — causes performance degradation
export DENSE_WEIGHT="0.6"
export SPARSE_WEIGHT="0.4"

# For local Qdrant (alternative to cloud)
export QDRANT_HOST="localhost"
export QDRANT_PORT="6333"

# Trial run (downloads 2 PDFs, processes in-memory Qdrant)
python main.py --trial

# Production run (continuous batch ingestion)
python main.py

curl -N -X POST "https://trinetra-dev-arxiv-scholar.hf.space/api/v1/query" \
  -H "Content-Type: application/json" \
  -d '{
    "query": "What is contrastive learning?",
    "limit": 5,
    "use_reranker": false
  }'

# Start the server
uvicorn src.arxiv_scholar.api.server:app --reload

# Or via Docker
docker build -t arxiv-scholar .
docker run -p 7860:7860 --env-file .env arxiv-scholar

# Query your local instance
curl -N -X POST http://localhost:8000/api/v1/query \
  -H "Content-Type: application/json" \
  -d '{"query": "attention mechanisms in transformers", "limit": 10, "use_reranker": false}'

# Generate evaluation dataset
python scripts/generate_eval_dataset.py

# Run benchmarks
python scripts/run_benchmarks.py