list-experiments

name	list-experiments
description	Use this skill whenever you need to list all of the experiment ideas tried and in progress for this research programme. It outputs 3 files organized by usefulness — merged winners, a compact results table, and full details for deep dives. Use when generating new experimental ideas to check what has already been tried.

List Experiments

Run the script below to fetch all experiment PRs from the advisor branch and organize them into 3 files.

The BASE_BRANCH should be set to the advisor branch (e.g. noam). Check the $ADVISOR_BRANCH env var or the PR base branch.

Code

import subprocess, json, os, re
from datetime import datetime

BASE_BRANCH = os.environ.get("ADVISOR_BRANCH", "noam")

def get_repo_info():
    r = subprocess.run(["gh", "repo", "view", "--json", "owner,name"], capture_output=True, text=True)
    if r.returncode:
        raise RuntimeError(r.stderr)
    info = json.loads(r.stdout)
    return info["owner"]["login"], info["name"]

def fetch_issue_comments(owner, name, number):
    """Fetch all PR conversation comments for a PR, across every REST page."""
    comments = []
    page = 1
    while True:
        endpoint = f"repos/{owner}/{name}/issues/{number}/comments?per_page=100&page={page}"
        r = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True)
        if r.returncode:
            raise RuntimeError(r.stderr)
        batch = json.loads(r.stdout)
        comments.extend((comment.get("body") or "") for comment in batch)
        if len(batch) < 100:
            return comments
        page += 1

def fetch():
    """Fetch PRs with complete comments via GraphQL plus REST overflow pages."""
    owner, name = get_repo_info()
    prs = []
    cursor = None
    while True:
        after = f', after: "{cursor}"' if cursor else ""
        query = f'''{{
          repository(owner: "{owner}", name: "{name}") {{
            pullRequests(first: 100, baseRefName: "{BASE_BRANCH}", states: [OPEN, CLOSED, MERGED]{after}) {{
              pageInfo {{ hasNextPage endCursor }}
              nodes {{
                number title body state mergedAt url
                labels(first: 10) {{ nodes {{ name }} }}
                comments(first: 100) {{
                  pageInfo {{ hasNextPage }}
                  nodes {{ body }}
                }}
              }}
            }}
          }}
        }}'''
        r = subprocess.run(["gh", "api", "graphql", "-f", f"query={query}"], capture_output=True, text=True)
        if r.returncode:
            raise RuntimeError(r.stderr)
        data = json.loads(r.stdout)["data"]["repository"]["pullRequests"]
        for node in data["nodes"]:
            first_comments = [c["body"] or "" for c in node["comments"]["nodes"]]
            comments = (
                fetch_issue_comments(owner, name, node["number"])
                if node["comments"]["pageInfo"]["hasNextPage"]
                else first_comments
            )
            prs.append({
                "number": node["number"],
                "title": node["title"],
                "body": node["body"] or "",
                "state": node["state"],
                "mergedAt": node["mergedAt"],
                "url": node["url"],
                "labels": [{"name": l["name"]} for l in node["labels"]["nodes"]],
                "comments": comments,
            })
        if not data["pageInfo"]["hasNextPage"]:
            break
        cursor = data["pageInfo"]["endCursor"]
    return prs

def extract_section(text, header):
    """Extract a markdown section (# or ##) by header name."""
    match = re.search(rf'(^#{{1,2}} {header}.*?)(?=^#{{1,2}} |\Z)', text, re.MULTILINE | re.DOTALL)
    return match.group(1).strip() if match else None

def find_results_text(pr):
    """Search body then comments (newest first) for the Results section."""
    body = pr.get("body", "") or ""
    section = extract_section(body, "Results")
    if section and "_To be filled" not in section:
        return section
    for comment in reversed(pr.get("comments", [])):
        section = extract_section(comment, "Results")
        if section and "_To be filled" not in section:
            return section
    return None

def extract_val_loss(text):
    if not text:
        return None
    patterns = [
        r'val[/_]loss[:\s]*\*?\*?(\d+\.\d+)',
        r'\*\*val/loss\*\*[:\s|]*\*?\*?(\d+\.\d+)',
        r'val/loss[:\s|]+(\d+\.\d+)',
    ]
    for p in patterns:
        m = re.search(p, text, re.IGNORECASE)
        if m:
            return float(m.group(1))
    return None

def has_results(pr):
    return find_results_text(pr) is not None

def labels_str(pr):
    return ", ".join(l["name"] for l in pr.get("labels", []))

def hypothesis_oneline(body):
    """Extract first sentence of hypothesis."""
    sec = extract_section(body or "", "Hypothesis")
    if not sec:
        return ""
    text = re.sub(r'^#{1,2} Hypothesis\n', '', sec).strip()
    m = re.match(r'(.+?[.!])\s', text)
    return m.group(1) if m else text[:120]

# --- Fetch ---
prs = fetch()
merged = sorted([p for p in prs if p.get("mergedAt")], key=lambda p: p["mergedAt"])
with_results = [p for p in prs if has_results(p) and not p.get("mergedAt")]
no_results = [p for p in prs if not has_results(p)]

ts = datetime.now().strftime("%Y-%m-%d_%H-%M")
outdir = "experiment_log"
os.makedirs(outdir, exist_ok=True)

# --- File 1: Merged winners (chronological) ---
merged_path = f"{outdir}/merged_{ts}.md"
with open(merged_path, "w") as f:
    f.write(f"# Merged Experiments ({len(merged)} winners)\n\n")
    f.write("These PRs beat baseline and were merged, in chronological order.\n\n")
    for pr in merged:
        body = pr.get("body", "") or ""
        hyp = extract_section(body, "Hypothesis") or ""
        results = find_results_text(pr) or ""
        val = extract_val_loss(results)
        val_str = f" | val_loss: {val}" if val else ""
        f.write(f"## #{pr['number']}: {pr['title']}{val_str}\n\n")
        if hyp:
            f.write(f"{hyp}\n\n")
        if results:
            f.write(f"{results}\n\n")
        f.write("---\n\n")

# --- File 2: Compact results table ---
table_path = f"{outdir}/results_table_{ts}.md"
with open(table_path, "w") as f:
    f.write(f"# Experiment Results Table\n\n")
    f.write(f"Total: {len(prs)} PRs | Merged: {len(merged)} | ")
    f.write(f"Ran (not merged): {len(with_results)} | Never ran: {len(no_results)}\n\n")

    f.write("## Merged\n\n")
    f.write("| PR | Title | val_loss |\n|---|---|---|\n")
    for pr in merged:
        results = find_results_text(pr)
        val = extract_val_loss(results)
        val_str = f"{val:.4f}" if val else "—"
        f.write(f"| #{pr['number']} | {pr['title']} | {val_str} |\n")

    f.write(f"\n## Ran but not merged ({len(with_results)})\n\n")
    f.write("| PR | Title | val_loss | Hypothesis (short) |\n|---|---|---|---|\n")
    for pr in sorted(with_results, key=lambda p: p["number"], reverse=True):
        body = pr.get("body", "") or ""
        results = find_results_text(pr)
        val = extract_val_loss(results)
        val_str = f"{val:.4f}" if val else "—"
        hyp = hypothesis_oneline(body)
        f.write(f"| #{pr['number']} | {pr['title']} | {val_str} | {hyp[:80]} |\n")

    f.write(f"\n## Never ran ({len(no_results)})\n\n")
    f.write("| PR | Title | Hypothesis (short) |\n|---|---|---|\n")
    for pr in sorted(no_results, key=lambda p: p["number"], reverse=True):
        hyp = hypothesis_oneline(pr.get("body", ""))
        f.write(f"| #{pr['number']} | {pr['title']} | {hyp[:80]} |\n")

# --- File 3: Full details (only PRs that ran) ---
full_path = f"{outdir}/full_details_{ts}.md"
with open(full_path, "w") as f:
    f.write(f"# Full Experiment Details ({len(merged) + len(with_results)} experiments)\n\n")
    for pr in merged + sorted(with_results, key=lambda p: p["number"], reverse=True):
        body = pr.get("body", "").strip() or "_No description._"
        state = "MERGED" if pr.get("mergedAt") else "CLOSED"
        f.write(f"# #{pr['number']}: {pr['title']} [{state}]\n\n{body}\n\n")
        results = find_results_text(pr)
        if results and results not in body:
            f.write(f"{results}\n\n")
        f.write("---\n\n")

print(f"Saved {len(prs)} PRs ({len(merged)} merged, {len(with_results)} ran, {len(no_results)} never ran):")
print(f"  {merged_path}         — merged winners with hypothesis + results")
print(f"  {table_path}   — compact table of all experiments")
print(f"  {full_path}   — full PR bodies (only experiments that ran)")