1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

import pandas as pd
import numpy as np
from typing import List, Tuple
import warnings
warnings.filterwarnings('ignore')

# Critical pandas performance tip from archive:
# Using & and | operators is MUCH faster than zip() with any() or all()

def slow_pandas_filtering(df: pd.DataFrame) -> pd.DataFrame:
    """Inefficient: using zip with multiple conditions"""
    condition1 = df['column1'] > 100
    condition2 = df['column2'] < 50
    condition3 = df['column3'] == 'target'

    # SLOW: Don't do this!
    combined_condition = [
        any([c1, c2, c3])
        for c1, c2, c3 in zip(condition1, condition2, condition3)
    ]

    return df[combined_condition]

def fast_pandas_filtering(df: pd.DataFrame) -> pd.DataFrame:
    """Efficient: using pandas boolean operators"""
    # FAST: Use & and | operators directly
    condition = (
        (df['column1'] > 100) |
        (df['column2'] < 50) |
        (df['column3'] == 'target')
    )

    return df[condition]

def even_faster_pandas_filtering(df: pd.DataFrame) -> pd.DataFrame:
    """Even faster: use .loc with intermediate DataFrames if needed"""
    # For complex filtering, sometimes multiple .loc calls are faster
    filtered = df.loc[df['column1'] > 100]  # First filter
    filtered = filtered.loc[filtered['column2'] < 50]  # Second filter
    filtered = filtered.loc[filtered['column3'] == 'target']  # Third filter

    return filtered

# Advanced pandas optimization techniques
class PandasOptimizer:
    """Collection of pandas optimization techniques"""

    @staticmethod
    def optimize_dtypes(df: pd.DataFrame) -> pd.DataFrame:
        """Optimize DataFrame memory usage by choosing appropriate dtypes"""
        optimized_df = df.copy()

        for column in df.columns:
            col_type = df[column].dtype

            if col_type != 'object':
                # Optimize numeric columns
                col_min = df[column].min()
                col_max = df[column].max()

                if col_type == 'int64':
                    if col_min > np.iinfo(np.int8).min and col_max < np.iinfo(np.int8).max:
                        optimized_df[column] = df[column].astype(np.int8)
                    elif col_min > np.iinfo(np.int16).min and col_max < np.iinfo(np.int16).max:
                        optimized_df[column] = df[column].astype(np.int16)
                    elif col_min > np.iinfo(np.int32).min and col_max < np.iinfo(np.int32).max:
                        optimized_df[column] = df[column].astype(np.int32)

                elif col_type == 'float64':
                    if col_min > np.finfo(np.float32).min and col_max < np.finfo(np.float32).max:
                        optimized_df[column] = df[column].astype(np.float32)

            else:
                # Optimize object columns (strings)
                num_unique_values = len(df[column].unique())
                num_total_values = len(df[column])

                if num_unique_values / num_total_values < 0.5:
                    optimized_df[column] = df[column].astype('category')

        return optimized_df

    @staticmethod
    def efficient_groupby_operations(df: pd.DataFrame) -> pd.DataFrame:
        """Demonstrate efficient groupby patterns"""

        # Slow: Multiple separate groupby operations
        # result1 = df.groupby('category')['value'].mean()
        # result2 = df.groupby('category')['value'].sum()
        # result3 = df.groupby('category')['value'].count()

        # Fast: Single groupby with agg
        result = df.groupby('category')['value'].agg(['mean', 'sum', 'count'])

        # Even faster for multiple columns
        multi_result = df.groupby('category').agg({
            'value1': ['mean', 'sum'],
            'value2': ['max', 'min'],
            'value3': 'count'
        })

        return result, multi_result

    @staticmethod
    def vectorized_string_operations(df: pd.DataFrame, column: str) -> pd.DataFrame:
        """Use vectorized string operations instead of apply"""

        # Slow: apply with lambda
        # df['processed'] = df[column].apply(lambda x: x.upper().replace(' ', '_'))

        # Fast: vectorized string operations
        df['processed'] = df[column].str.upper().str.replace(' ', '_', regex=False)

        # Complex string processing
        df['cleaned'] = (df[column]
                        .str.strip()
                        .str.lower()
                        .str.replace(r'[^\w\s]', '', regex=True)
                        .str.replace(r'\s+', ' ', regex=True))

        return df

    @staticmethod
    def efficient_merge_operations(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame:
        """Optimize DataFrame merge operations"""

        # Set index for faster merges if doing multiple merges on same keys
        df1_indexed = df1.set_index('key_column')
        df2_indexed = df2.set_index('key_column')

        # Fast merge using indices
        result = df1_indexed.join(df2_indexed, how='inner')

        # For large datasets, consider using merge with sorted data
        df1_sorted = df1.sort_values('key_column')
        df2_sorted = df2.sort_values('key_column')

        result_sorted = pd.merge(df1_sorted, df2_sorted, on='key_column', how='inner')

        return result

# Performance testing for pandas operations
def pandas_performance_comparison():
    """Compare different pandas operation approaches"""

    # Create test data
    np.random.seed(42)
    n_rows = 100000

    df = pd.DataFrame({
        'column1': np.random.randint(0, 200, n_rows),
        'column2': np.random.randint(0, 100, n_rows),
        'column3': np.random.choice(['target', 'other1', 'other2'], n_rows),
        'value': np.random.randn(n_rows)
    })

    # Test filtering performance
    import time

    print("Pandas Performance Comparison:")
    print("-" * 40)

    # Test slow method
    start_time = time.perf_counter()
    slow_result = slow_pandas_filtering(df)
    slow_time = time.perf_counter() - start_time

    # Test fast method
    start_time = time.perf_counter()
    fast_result = fast_pandas_filtering(df)
    fast_time = time.perf_counter() - start_time

    # Test fastest method
    start_time = time.perf_counter()
    fastest_result = even_faster_pandas_filtering(df)
    fastest_time = time.perf_counter() - start_time

    print(f"Slow method (zip):     {slow_time:.4f}s")
    print(f"Fast method (&, |):    {fast_time:.4f}s ({slow_time/fast_time:.1f}x faster)")
    print(f"Fastest method (.loc): {fastest_time:.4f}s ({slow_time/fastest_time:.1f}x faster)")

    # Memory optimization test
    original_memory = df.memory_usage(deep=True).sum()
    optimized_df = PandasOptimizer.optimize_dtypes(df)
    optimized_memory = optimized_df.memory_usage(deep=True).sum()

    print(f"\nMemory optimization:")
    print(f"Original:  {original_memory / 1024 / 1024:.2f} MB")
    print(f"Optimized: {optimized_memory / 1024 / 1024:.2f} MB")
    print(f"Reduction: {(1 - optimized_memory/original_memory)*100:.1f}%")