pip install polars

import pandas as pd
import polars as pl
import time

start = time.time()
df = pd.read_csv('flights.csv', dtype={"TAIL_NUMBER": "string", "ORIGIN_AIRPORT": "string", "DESTINATION_AIRPORT": "string"})
df = df[(df['MONTH'] == 12) & (df['ORIGIN_AIRPORT'] == 'SEA') & (df['DESTINATION_AIRPORT'] == 'DFW')]
end = time.time()
print(end - start)
df

6.601531744003296

start = time.time()
df = pl.read_csv('flights.csv').filter(
        (pl.col('MONTH') == 12) & (pl.col('ORIGIN_AIRPORT') == 'SEA') & (pl.col('DESTINATION_AIRPORT') == 'DFW'))
end = time.time()
print(end - start)
display(df)

0.6479246616363525

start = time.time()
df = pl.scan_csv('flights.csv').filter(
        (pl.col('MONTH') == 11) & (pl.col('ORIGIN_AIRPORT') == 'SEA') & (pl.col('DESTINATION_AIRPORT') == 'DFW')).collect()
end = time.time()
print(end - start)
display(df)

0.7065532207489014

lf = pl.scan_parquet("events/*.parquet")
fast = (
    lf.filter(pl.col("country") == "IN")
      .filter(pl.col("ts") >= pl.datetime(2025, 9, 1))
      .select(["ts", "country", "path"])
      .collect()
)

# Planificar acceso
q1 = (
    pl.scan_parquet("Parquet/**/*.parquet")
    .filter(pl.col("passenger_count") > 0) # Filter
    .with_columns([
        # Derive date/time features
        (pl.col("tpep_dropoff_datetime") - pl.col("tpep_pickup_datetime"))
        .dt.total_minutes()
        .alias("trip_duration_minutes")
    ])
    .group_by("passenger_count")
    .agg([
        pl.mean("trip_duration_minutes"),
        pl.mean("total_amount").alias("avg_fare")
    ])
    .sort("passenger_count")
)

# Ejecutar consulta
r1 = q1.collect()
print(r1)

┌─────────────────┬───────────────────────┬────────────┐
│ passenger_count ┆ trip_duration_minutes ┆ avg_fare   │
│ ---             ┆ ---                   ┆ ---        │
│ i64             ┆ f64                   ┆ f64        │
╞═════════════════╪═══════════════════════╪════════════╡
│ 1               ┆ 15.547899             ┆ 27.008371  │
│ 2               ┆ 17.243722             ┆ 30.352485  │
│ 3               ┆ 16.934745             ┆ 29.482887  │
│ 4               ┆ 17.764046             ┆ 32.436805  │
│ 5               ┆ 20.555213             ┆ 26.760919  │
│ 6               ┆ 23.465055             ┆ 26.841099  │
│ 7               ┆ 18.3                  ┆ 83.639     │
│ 8               ┆ 9.911765              ┆ 102.342059 │
│ 9               ┆ 11.666667             ┆ 62.724444  │
└─────────────────┴───────────────────────┴────────────┘

lf = pl.scan_parquet("path/YYYY=2025/MM=10/*/*.parquet")
today = lf.filter(pl.col("DD") == "08").collect()

q2 = (
    pl.scan_parquet("Parquet/*/*/*.parquet")
    .join(pl.scan_csv("taxi_zone_lookup.csv"), left_on="PULocationID", right_on="LocationID")
    .filter(pl.col("total_amount") > 25)
    .group_by("Zone")
    .agg(
        (pl.col("total_amount") /
        (pl.col("tpep_dropoff_datetime") - pl.col("tpep_pickup_datetime")).dt.total_minutes()
        ).mean().alias("cost_per_minute")
    ).sort("cost_per_minute",descending=True)
)

# Collect the results (executes the query)
r2 = q2.collect()
print(r2)

┌─────────────────────────────────┬─────────────────┐
│ Zone                            ┆ cost_per_minute │
│ ---                             ┆ ---             │
│ str                             ┆ f64             │
╞═════════════════════════════════╪═════════════════╡
│ Co-Op City                      ┆ inf             │
│ Whitestone                      ┆ inf             │
│ UN/Turtle Bay South             ┆ inf             │
│ Eltingville/Annadale/Prince's … ┆ inf             │
│ Pelham Bay Park                 ┆ inf             │
│ …                               ┆ …               │
│ New Dorp/Midland Beach          ┆ 3.958707        │
│ Governor's Island/Ellis Island… ┆ 3.238929        │
│ Rikers Island                   ┆ 2.802784        │
│ Green-Wood Cemetery             ┆ 1.832354        │
│ Crotona Park                    ┆ 1.514066        │
└─────────────────────────────────┴─────────────────┘

lf = pl.scan_csv("raw/*.csv", dtypes={"ts": pl.Datetime, "user_id": pl.Int64})
(
    lf.filter(pl.col("ts") >= pl.datetime(2025, 10, 1))
      .with_columns(pl.col("path").str.slice(0, 64).alias("path_64"))
      .sink_parquet("curated/2025-10.parquet")