from sentence_transformers import SentenceTransformer
import numpy as np

# Load a sentence embedding model
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

# Embed some sentences
sentences = [
    "The cat sat on the mat.",
    "A feline rested on the rug.",
    "Dogs love to play fetch.",
    "Machine learning is a subset of AI.",
    "Artificial intelligence includes ML.",
]

embeddings = model.encode(sentences)

print(f"Embedding shape: {embeddings.shape}")
print(f"Each sentence → {embeddings.shape[1]}-dimensional vector")
print(f"\nFirst embedding (first 8 dims): {embeddings[0][:8].round(4)}")

Embedding shape: (5, 384)
Each sentence → 384-dimensional vector

First embedding (first 8 dims): [ 0.0234 -0.1823  0.0912  0.3421 -0.0541  0.2134 -0.0823  0.1234]

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

def cosine_sim(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

# Compare all pairs
print("Cosine similarity between sentences:")
print(f"{'Pair':<55} {'Similarity'}")
print("-" * 70)

pairs = [
    (0, 1, "cat on mat vs feline on rug"),
    (0, 2, "cat on mat vs dogs play fetch"),
    (3, 4, "ML subset AI vs AI includes ML"),
    (0, 3, "cat on mat vs ML is AI"),
]

for i, j, desc in pairs:
    sim = cosine_sim(embeddings[i], embeddings[j])
    print(f"{desc:<55} {sim:.4f}")

Cosine similarity between sentences:
Pair                                                    Similarity
----------------------------------------------------------------------
cat on mat vs feline on rug                             0.8341
cat on mat vs dogs play fetch                           0.4123
ML subset AI vs AI includes ML                          0.8912
cat on mat vs ML is AI                                  0.1234

from sentence_transformers import SentenceTransformer

# Popular embedding models

models_info = {
    'all-MiniLM-L6-v2': {
        'dim': 384,
        'size': '80MB',
        'speed': 'very fast',
        'quality': 'good',
        'note': 'Best starting point. Fast and accurate.'
    },
    'all-mpnet-base-v2': {
        'dim': 768,
        'size': '420MB',
        'speed': 'medium',
        'quality': 'excellent',
        'note': 'Best quality for semantic search.'
    },
    'paraphrase-multilingual-MiniLM-L12-v2': {
        'dim': 384,
        'size': '470MB',
        'speed': 'fast',
        'quality': 'good',
        'note': 'Supports 50+ languages.'
    },
    'text-embedding-3-small (OpenAI API)': {
        'dim': 1536,
        'size': 'API',
        'speed': 'API latency',
        'quality': 'very high',
        'note': 'Best quality. Costs per token.'
    }
}

print(f"{'Model':<45} {'Dim':<6} {'Size':<10} {'Quality'}")
print("-" * 70)
for name, info in models_info.items():
    print(f"{name:<45} {info['dim']:<6} {info['size']:<10} {info['quality']}")

# Load the recommended default
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

# A knowledge base of documents
documents = [
    "Python is a high-level programming language known for its simplicity and readability.",
    "Machine learning algorithms learn patterns from data without being explicitly programmed.",
    "Neural networks are computing systems inspired by biological neural networks.",
    "The transformer architecture uses self-attention mechanisms to process sequential data.",
    "BERT is a bidirectional transformer pretrained on masked language modeling.",
    "GPT uses a decoder-only transformer trained on next-token prediction.",
    "Fine-tuning adapts a pretrained model to a specific task using domain data.",
    "LoRA reduces the number of trainable parameters by using low-rank decomposition.",
    "Vector databases store embeddings and support fast nearest-neighbor search.",
    "RAG combines retrieval with generation to give LLMs access to external knowledge.",
    "Cosine similarity measures the angle between two vectors in embedding space.",
    "Tokenization breaks text into smaller units called tokens before feeding to a model.",
    "Backpropagation computes gradients by applying the chain rule backward through a network.",
    "Overfitting occurs when a model learns the training data too well and fails on new data.",
    "Cross-validation gives a more reliable estimate of model performance than a single split.",
]

class SemanticSearch:
    def __init__(self, model_name='sentence-transformers/all-MiniLM-L6-v2'):
        self.model     = SentenceTransformer(model_name)
        self.documents = []
        self.embeddings = None

    def index(self, documents):
        self.documents  = documents
        print(f"Encoding {len(documents)} documents...")
        self.embeddings = self.model.encode(documents, show_progress_bar=True)
        print(f"Indexed {len(documents)} documents. Embedding shape: {self.embeddings.shape}")

    def search(self, query, top_k=3):
        # Encode the query
        query_embedding = self.model.encode([query])

        # Compute cosine similarity with all documents
        similarities = cosine_similarity(query_embedding, self.embeddings)[0]

        # Get top-k results
        top_indices = np.argsort(similarities)[::-1][:top_k]

        results = []
        for idx in top_indices:
            results.append({
                'document': self.documents[idx],
                'score':    similarities[idx],
                'index':    idx
            })
        return results

# Build the search engine
search_engine = SemanticSearch()
search_engine.index(documents)

# Test queries
queries = [
    "How do transformers work?",
    "What is the difference between BERT and GPT?",
    "How can I make training more efficient?",
    "What happens when a model memorizes training data?",
]

for query in queries:
    print(f"\nQuery: '{query}'")
    print("-" * 60)
    results = search_engine.search(query, top_k=3)
    for i, r in enumerate(results):
        print(f"  {i+1}. [{r['score']:.3f}] {r['document'][:80]}...")

Query: 'How do transformers work?'
------------------------------------------------------------
  1. [0.712] The transformer architecture uses self-attention mechanisms...
  2. [0.634] BERT is a bidirectional transformer pretrained on masked...
  3. [0.601] GPT uses a decoder-only transformer trained on next-token...

Query: 'What is the difference between BERT and GPT?'
------------------------------------------------------------
  1. [0.823] BERT is a bidirectional transformer pretrained on masked...
  2. [0.798] GPT uses a decoder-only transformer trained on next-token...
  3. [0.612] The transformer architecture uses self-attention mechanisms...

Query: 'How can I make training more efficient?'
------------------------------------------------------------
  1. [0.651] LoRA reduces the number of trainable parameters by using...
  2. [0.589] Fine-tuning adapts a pretrained model to a specific task...
  3. [0.534] Machine learning algorithms learn patterns from data...

Query: 'What happens when a model memorizes training data?'
------------------------------------------------------------
  1. [0.714] Overfitting occurs when a model learns the training data...
  2. [0.543] Cross-validation gives a more reliable estimate of model...
  3. [0.498] Fine-tuning adapts a pretrained model to a specific task...

pip install faiss-cpu   # or faiss-gpu for GPU support

import faiss
import numpy as np
from sentence_transformers import SentenceTransformer

# Generate sample embeddings (simulating a large corpus)
model       = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
dimension   = 384   # all-MiniLM-L6-v2 embedding size

# Simulate 10,000 documents
np.random.seed(42)
fake_embeddings = np.random.randn(10000, dimension).astype('float32')
# Normalize for cosine similarity (FAISS uses inner product)
faiss.normalize_L2(fake_embeddings)

# Build FAISS index
# IndexFlatIP: exact inner product search (cosine similarity after L2 normalization)
index = faiss.IndexFlatIP(dimension)
index.add(fake_embeddings)
print(f"FAISS index size: {index.ntotal} vectors")

# Search
query_embedding = np.random.randn(1, dimension).astype('float32')
faiss.normalize_L2(query_embedding)

k = 5
distances, indices = index.search(query_embedding, k)

print(f"\nTop {k} nearest neighbors:")
for dist, idx in zip(distances[0], indices[0]):
    print(f"  Index {idx}: similarity={dist:.4f}")

# For very large datasets: use IVF index (approximate, faster)
# IVF = Inverted File Index, partitions space into clusters

n_clusters = 100   # number of partitions (sqrt of dataset size is a good rule)
quantizer  = faiss.IndexFlatIP(dimension)
ivf_index  = faiss.IndexIVFFlat(quantizer, dimension, n_clusters, faiss.METRIC_INNER_PRODUCT)

# Must train IVF index before adding vectors
ivf_index.train(fake_embeddings)
ivf_index.add(fake_embeddings)

# Tune nprobe: how many clusters to search (higher = more accurate, slower)
ivf_index.nprobe = 10

distances_ivf, indices_ivf = ivf_index.search(query_embedding, k)
print(f"\nIVF index results (approximate but faster):")
for dist, idx in zip(distances_ivf[0], indices_ivf[0]):
    print(f"  Index {idx}: similarity={dist:.4f}")

# Benchmark: exact vs approximate
import time

# Exact search
start = time.time()
for _ in range(100):
    index.search(query_embedding, k)
exact_time = (time.time() - start) / 100

# Approximate search
start = time.time()
for _ in range(100):
    ivf_index.search(query_embedding, k)
approx_time = (time.time() - start) / 100

print(f"\nSearch time per query:")
print(f"  Exact (IndexFlatIP): {exact_time*1000:.2f}ms")
print(f"  Approximate (IVF):   {approx_time*1000:.2f}ms")
print(f"  Speedup: {exact_time/approx_time:.1f}x")

pip install chromadb

import chromadb
from sentence_transformers import SentenceTransformer

# Create a ChromaDB client
client = chromadb.Client()   # in-memory; use chromadb.PersistentClient('./chroma_db') for persistence

# Create a collection
collection = client.create_collection(
    name='ml_knowledge_base',
    metadata={'hnsw:space': 'cosine'}   # use cosine similarity
)

# Your documents with metadata
docs = [
    {
        'id': 'doc1',
        'text': 'Python is a high-level programming language known for simplicity.',
        'metadata': {'topic': 'programming', 'difficulty': 'beginner'}
    },
    {
        'id': 'doc2',
        'text': 'Machine learning algorithms learn patterns from data.',
        'metadata': {'topic': 'ml', 'difficulty': 'intermediate'}
    },
    {
        'id': 'doc3',
        'text': 'Neural networks are inspired by biological neural networks.',
        'metadata': {'topic': 'deep_learning', 'difficulty': 'intermediate'}
    },
    {
        'id': 'doc4',
        'text': 'BERT is a bidirectional transformer pretrained on MLM.',
        'metadata': {'topic': 'nlp', 'difficulty': 'advanced'}
    },
    {
        'id': 'doc5',
        'text': 'LoRA reduces trainable parameters using low-rank decomposition.',
        'metadata': {'topic': 'fine_tuning', 'difficulty': 'advanced'}
    },
]

# Add documents (ChromaDB can use its own embedding model or you provide embeddings)
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

collection.add(
    ids       = [d['id'] for d in docs],
    documents = [d['text'] for d in docs],
    embeddings= [model.encode(d['text']).tolist() for d in docs],
    metadatas = [d['metadata'] for d in docs]
)

print(f"Collection size: {collection.count()}")

# Basic query
results = collection.query(
    query_embeddings=[model.encode("How do transformers work?").tolist()],
    n_results=3
)

print("\nQuery: 'How do transformers work?'")
for i, (doc, dist) in enumerate(zip(results['documents'][0], results['distances'][0])):
    print(f"  {i+1}. [{1-dist:.3f}] {doc}")   # ChromaDB returns distance, convert to similarity

# Filter by metadata
results_filtered = collection.query(
    query_embeddings=[model.encode("machine learning concepts").tolist()],
    n_results=3,
    where={'difficulty': 'advanced'}   # only return advanced documents
)

print("\nQuery with filter (difficulty=advanced):")
for doc, meta in zip(results_filtered['documents'][0], results_filtered['metadatas'][0]):
    print(f"  [{meta['topic']}] {doc}")

# Update and delete
collection.update(
    ids=['doc1'],
    documents=['Python is a versatile high-level programming language.'],
    embeddings=[model.encode('Python is a versatile high-level programming language.').tolist()]
)

collection.delete(ids=['doc5'])
print(f"\nAfter update and delete: {collection.count()} documents")

from sentence_transformers import SentenceTransformer
import numpy as np
import time

model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

# Simulate a large dataset
large_corpus = [f"This is document number {i} about topic {i % 10}." for i in range(5000)]

# Efficient batch encoding
print("Encoding 5000 documents...")
start = time.time()

embeddings = model.encode(
    large_corpus,
    batch_size=64,           # process 64 at a time
    show_progress_bar=True,
    normalize_embeddings=True  # L2 normalize for cosine similarity
)

elapsed = time.time() - start
print(f"\nDone in {elapsed:.1f}s")
print(f"Speed: {len(large_corpus)/elapsed:.0f} docs/second")
print(f"Embeddings shape: {embeddings.shape}")

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

def evaluate_embeddings(model_name, test_pairs):
    """
    test_pairs: list of (sent1, sent2, label) where label=1 means similar, 0 means different
    """
    model = SentenceTransformer(model_name)

    sents1 = [p[0] for p in test_pairs]
    sents2 = [p[1] for p in test_pairs]
    labels = [p[2] for p in test_pairs]

    emb1 = model.encode(sents1)
    emb2 = model.encode(sents2)

    similarities = [cosine_similarity([e1], [e2])[0][0] for e1, e2 in zip(emb1, emb2)]

    # Threshold at 0.5 to predict similar/different
    preds = [1 if s > 0.5 else 0 for s in similarities]
    accuracy = sum(p == l for p, l in zip(preds, labels)) / len(labels)

    return accuracy, similarities

test_pairs = [
    ("cheap hotel", "affordable accommodation", 1),
    ("machine learning", "artificial intelligence", 1),
    ("cat on the mat", "deep learning model", 0),
    ("how to code in python", "python programming tutorial", 1),
    ("stock market crash", "cooking recipes", 0),
    ("neural network", "deep learning", 1),
    ("fix bug in code", "debug software", 1),
    ("the weather today", "quantum physics research", 0),
]

for model_name in ['sentence-transformers/all-MiniLM-L6-v2',
                    'sentence-transformers/all-mpnet-base-v2']:
    acc, sims = evaluate_embeddings(model_name, test_pairs)
    print(f"\n{model_name.split('/')[-1]}:")
    print(f"  Accuracy on test pairs: {acc:.1%}")
    for (s1, s2, label), sim in zip(test_pairs, sims):
        status = 'correct' if (sim > 0.5) == label else 'WRONG'
        print(f"  [{status}] sim={sim:.3f} | '{s1[:25]}' vs '{s2[:25]}'")

# Pattern 1: Asymmetric search (query and documents use different models)
# Useful when queries are short questions and documents are long passages

from sentence_transformers import SentenceTransformer

bi_encoder = SentenceTransformer('sentence-transformers/msmarco-distilbert-base-v4')

# Documents
passages = [
    "LoRA stands for Low-Rank Adaptation and is used for efficient fine-tuning.",
    "The Eiffel Tower is a famous landmark in Paris, France.",
    "Python was created by Guido van Rossum and first released in 1991.",
]

# Short query
query = "What is LoRA?"

query_emb    = bi_encoder.encode(query)
passage_embs = bi_encoder.encode(passages)

sims = cosine_similarity([query_emb], passage_embs)[0]
top  = np.argmax(sims)
print(f"Query: '{query}'")
print(f"Best match [{sims[top]:.3f}]: '{passages[top]}'")

# Pattern 2: Clustering embeddings to find topics
from sklearn.cluster import KMeans

sentences = [
    "Python is great for data science.",
    "R is used for statistical computing.",
    "Machine learning requires lots of data.",
    "Deep learning uses neural networks.",
    "Java is widely used in enterprise software.",
    "JavaScript powers the web frontend.",
    "Supervised learning uses labeled data.",
    "Unsupervised learning finds hidden patterns.",
]

model      = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
embeddings = model.encode(sentences)

kmeans = KMeans(n_clusters=3, random_state=42, n_init=10)
labels = kmeans.fit_predict(embeddings)

print("\nClustered sentences:")
for cluster_id in range(3):
    print(f"\nCluster {cluster_id}:")
    for sent, label in zip(sentences, labels):
        if label == cluster_id:
            print(f"  - {sent}")