shboost-cmd-plot-and-animation

name	shboost_cmd_plot_and_animation
title	ShBoost CMD Plot and Population Animation
description	Cache selected columns from the ShBoost 2024 S3 dataset, create a hexbin CMD with log-scaled density, add optimized non‑overlapping stellar‑population annotations, and generate both a high‑resolution PNG and an MP4 animation.
author	hermes

When to Use

Cache selected columns from the ShBoost 2024 S3 dataset, create a hexbin CMD with log-scaled density, add optimized non‑overlapping stellar‑population annotations, and generate both a high‑resolution PNG and an MP4 animation.

Canonical Routing

This is a specialized or legacy example skill. For new work, start with astro-data-access-umbrella and route through:

• s3-parquet-astro-access
• astro-catalog-plotting-cache

Keep this skill for dataset-specific examples, but prefer the canonical skills for new implementations, live probes, REANA execution, and plotting/cache conventions.

Pitfalls

• Do not hardcode credentials, tokens, or personal secrets.
• Verify external service URLs, paths, and permissions before making changes.
• Keep generated outputs reproducible and record input assumptions.

Verification

• Confirm required inputs and credentials are available.
• Run the smallest safe command or example before scaling up.
• Check produced files, API responses, or plots before reporting success.

Overview

Automates the workflow for visualising the ShBoost 2024 colour‑magnitude diagram (CMD) from a massive S3 dataset. Handles efficient caching, sampling, hexbin density plotting, colour‑coded annotations, legend creation, and animation generation.

Prerequisites

• Python 3.12+ with packages: dask[dataframe], pandas, matplotlib, seaborn.
• Access to the public S3 bucket s3://shboost2024/shboost_08july2024_pub.parq/.
• ffmpeg (optional; falls back to GIF via PillowWriter).

Constants (adjust as needed)

S3_ENDPOINT = "https://s3.data.aip.de:9000"
S3_PARQUET_GLOB = "s3://shboost2024/shboost_08july2024_pub.parq/*.parquet"
STORAGE_OPTS = {"use_ssl": True, "anon": True, "client_kwargs": {"endpoint_url": S3_ENDPOINT}}
CACHE_PATH = "shboost_full_cmd.parquet"   # local cache of selected columns
TARGET_ROWS = 10**12  # placeholder – full cache will be written regardless of size
PLOT_ROWS = 10_000_000  # rows used for the actual figure (adjust for RAM)

1. Load or Create Cache

def load_or_fetch(force_refresh: bool) -> pd.DataFrame:
    if os.path.isfile(CACHE_PATH) and not force_refresh:
        print(f"🔄 Loading cached full data from {CACHE_PATH}")
        full_dd = dd.read_parquet(CACHE_PATH)
    else:
        print("⏬ Reading metadata from S3 …")
        full_dd = dd.read_parquet(S3_PARQUET_GLOB, storage_options=STORAGE_OPTS)
        full_dd = full_dd[["bprp0", "mg0"]]
        total_rows = full_dd.shape[0].compute()
        print(f"Dataset size: {total_rows:,} rows (full cache will be written)")
        full_dd.to_parquet(CACHE_PATH, write_index=False)
        print("Full cache saved.")
        full_dd = dd.read_parquet(CACHE_PATH)

    # Sample for plotting
    total_rows = full_dd.shape[0].compute()
    frac = min(PLOT_ROWS / total_rows, 1.0)
    print(f"Sampling fraction for plot: {frac:.6%} → ~{min(PLOT_ROWS, total_rows):,} rows")
    sample_dd = full_dd if frac >= 1.0 else full_dd.sample(frac=frac, random_state=42)
    pdf = sample_dd.compute()
    print(f"Sampled rows for plot: {len(pdf):,}")
    return pdf

2. Static CMD Plot (plot_cmd)

def plot_cmd(df: pd.DataFrame, png_path: str = "shboost_cmd.png"):
    sns.set_style("whitegrid")
    plt.rcParams.update({
        "font.size": 12,
        "axes.titlesize": 14,
        "axes.labelsize": 13,
        "legend.fontsize": 11,
        "figure.figsize": (8, 6),
        "savefig.dpi": 300,
        "savefig.bbox": "tight",
    })
    mesh = plt.hexbin(df["bprp0"], df["mg0"], gridsize=512, cmap="viridis",
                     mincnt=1, linewidths=0.2, norm=LogNorm())
    cb = plt.colorbar(mesh, label="Counts")
    cb.ax.set_ylabel("Counts", rotation=270, labelpad=15)
    cb.ax.set_yscale('log')

    # Colour map for population labels
    color_map = {
        "Main Sequence": "tab:blue",
        "Turn‑off / Subgiants": "tab:orange",
        "Red Giant Branch": "tab:red",
        "Red Clump": "tab:purple",
        "Horizontal Branch": "tab:brown",
        "AGB": "tab:pink",
        "White Dwarfs": "tab:gray",
        : ,
    }
     ():
        plt.annotate(txt, xy=xy, xytext=offset, textcoords=,
                     fontsize=, color=color_map.get(txt, ), ha=, va=,
                     bbox=(boxstyle=, fc=, alpha=),
                     arrowprops=(arrowstyle=, color=, lw=))
    
    add_label(, xy=(, ), offset=(-, -))
    add_label(, xy=(, ), offset=(-, -))
    add_label(, xy=(, ), offset=(, ))
    add_label(, xy=(, ), offset=(, -))
    add_label(, xy=(, -), offset=(-, ))
    add_label(, xy=(, -), offset=(, ))
    add_label(, xy=(-, ), offset=(-, -))
    add_label(, xy=(-, ), offset=(-, ))

    
     matplotlib.lines  Line2D
    pop_legend = [Line2D([],[],marker=,color=,label=name,
                         markerfacecolor=color_map[name],markersize=)  name  color_map]
    plt.legend(handles=pop_legend, loc=, fontsize=, framealpha=, title=)

    plt.title()
    plt.xlabel()
    plt.ylabel()
    plt.xlim(-, )
    plt.gca().invert_yaxis()
    plt.tight_layout()
    plt.savefig(png_path, dpi=)
    plt.close()
    ()

3. Animation (create_animation)

def create_animation(df, png_path="shboost_cmd.png"):
    import matplotlib.animation as animation
    from matplotlib.colors import LogNorm
    fig, ax = plt.subplots()
    sns.set_style("whitegrid")
    plt.rcParams.update({"font.size": 12, "axes.titlesize": 14, "axes.labelsize": 13,
                         "legend.fontsize": 11, "figure.figsize": (8, 6),
                         "savefig.dpi": 300, "savefig.bbox": "tight"})
    mesh = ax.hexbin(df["bprp0"], df["mg0"], gridsize=512, cmap="viridis",
                     mincnt=1, linewidths=0.2, norm=LogNorm())
    cb = fig.colorbar(mesh, label="Counts")
    cb.ax.set_ylabel("Counts", rotation=270, labelpad=15)
    cb.ax.set_yscale('log')
    ax.set_xlabel("bprp0")
    ax.set_ylabel("mg0")
    ax.set_xlim(-4, 8)
    ax.invert_yaxis()
    ax.set_title(f"ShBoost 2024 Colour‑Magnitude Diagram ({len(df):,} stars)")

    populations = [
        ("Main Sequence", (1.5, 8)),
        ("Turn‑off / Subgiants", (0.8, )),
        (, (, )),
        (, (, )),
        (, (, -)),
        (, (, -)),
        (, (-, )),
        (, (-, )),
    ]
    color_map = {name: col  name, col  ([p[]  p  populations],
                     [,,,,,
                      ,,])}
     matplotlib.lines  Line2D
    pop_legend = [Line2D([],[],marker=,color=,label=name,
                         markerfacecolor=color_map[name],markersize=)  name  color_map]
    ax.legend(handles=pop_legend, loc=, fontsize=, framealpha=, title=)

    annotation = ax.annotate(, xy=(,), xytext=(,), textcoords=,
                             bbox=(boxstyle=, fc=, alpha=),
                             arrowprops=(arrowstyle=, color=, lw=))
     ():
        annotation.set_visible()
         (annotation,)
     ():
        name, xy = populations[frame]
        annotation.set_visible()
        annotation.set_text(name)
        annotation.xy = xy
        annotation.set_position((, -))
        annotation.set_fontsize()
        annotation.set_color(color_map.get(name, ))
         (annotation,)
    anim = animation.FuncAnimation(fig, update, frames=(populations), init_func=init,
                                   interval=, blit=, repeat=, save_count=(populations))
    out_path = os.path.join(os.path.dirname(png_path), )
    :
         matplotlib.animation  FFMpegWriter
        writer = FFMpegWriter(fps=)
        anim.save(out_path, writer=writer)
     Exception:
        out_path = os.path.join(os.path.dirname(png_path), )
         matplotlib.animation  PillowWriter
        anim.save(out_path, writer=PillowWriter(fps=))
    plt.close(fig)
    ()

4. Execution

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--force-refresh", action="store_true",
                        help="Ignore existing cache and re‑download from S3.")
    args = parser.parse_args()
    df = load_or_fetch(force_refresh=args.force_refresh)
    plot_cmd(df)
    create_animation(df)

Tips & Gotchas

• Memory: The full cache (~200 M rows) occupies several GB on disk but stays lazy thanks to Dask. Only PLOT_ROWS rows are materialised.
• Adjust PLOT_ROWS if you hit RAM limits.
• Annotation offsets may need tweaking for different sample sizes.
• FFmpeg: Install ffmpeg for MP4 output; otherwise the fallback GIF works everywhere.
• Cache invalidation: Use --force-refresh when the upstream dataset updates.

---

Skill created by Hermes.