1
import pyarrow.parquet as pq
2
 
3
# This 10GB file? It's now queryable instantly
4
# No loading time. No memory explosion. Just works.
5
table = pq.read_table('massive_file.parquet')

1
import polars as pl
2
 
3
# Lazy evaluation, automatic parallelization, 10-50x faster
4
df = pl.scan_parquet('*.parquet')  # Scan all files, lazily
5
result = (
6
    df.filter(pl.col('revenue') > 1000)
7
    .group_by('customer_id')
8
    .agg([
9
        pl.sum('revenue').alias('total_revenue'),
10
        pl.count('order_id').alias('order_count')
11
    ])
12
    .sort('total_revenue', descending=True)
13
    .collect()  # Only now does it actually run
14
)

1
import pandas as pd
2
 
3
# Looks like Pandas, runs like Polars
4
df = pd.read_parquet('data.parquet', engine='pyarrow')
5
# Copy-on-write by default, PyArrow dtypes, actual performance

1
import duckdb
2
 
3
# Query Parquet files directly with SQL
4
result = duckdb.sql("""
5
    SELECT 
6
        customer_segment,
7
        COUNT(*) as customer_count,
8
        AVG(lifetime_value) as avg_ltv
9
    FROM 'customers/*.parquet'
10
    WHERE signup_date >= '2023-01-01'
11
    GROUP BY customer_segment
12
    ORDER BY avg_ltv DESC
13
""").df()  # Returns a Pandas DataFrame

1
import duckdb
2
import polars as pl
3
import pyarrow.parquet as pq
4
 
5
# Read with PyArrow
6
table = pq.read_table('data.parquet')
7
 
8
# Convert to Polars for manipulation
9
df = pl.from_arrow(table)
10
processed = df.filter(pl.col('value') > 100)
11
 
12
# Query with DuckDB
13
result = duckdb.sql("SELECT * FROM processed WHERE category = 'A'").pl()
14
 
15
# It's all Arrow under the hood. Zero copy. Maximum speed.

1
import pandas as pd
2
import duckdb
3
 
4
# Just read the parquet files with pandas
5
orders = pd.read_parquet('orders_2024.parquet')
6
 
7
# Your existing pandas code just... works
8
daily_revenue = (
9
    orders
10
    .groupby(orders['date'].dt.date)
11
    .agg({'revenue': 'sum'})
12
    .sort_index()
13
)
14
 
15
# Need SQL? DuckDB reads your pandas dataframes directly
16
result = duckdb.sql("""
17
    SELECT 
18
        c.segment,
19
        DATE_TRUNC('month', o.date) as month,
20
        SUM(o.revenue) as revenue
21
    FROM orders o
22
    JOIN pd.read_parquet('customers.parquet') c ON o.customer_id = c.id
23
    GROUP BY 1, 2
24
""").df()