-
Confirm credentials and model availability:
echo $TOGETHER_API_KEY | head -c 8 && echo "..."
for id in google/gemma-3n-E4B-it zai-org/GLM-5.2 openai/gpt-oss-20b; do
echo -n "$id: "
curl -s https://api.together.xyz/v1/models \
-H "Authorization: Bearer $TOGETHER_API_KEY" \
| jq -r --arg id "$id" '.[] | select(.id==$id) | "✓ \(.type) ctx=\(.context_length)"'
done
-
Install the workshop model configs (idempotent):
for slot in modelA modelB modelC; do
cp .workshop/assets/together_${slot}.yaml.template \
configs/models/together_${slot}.yaml
done
-
ITERATION LOOP — smoke each model on structured_extraction (~30-90s):
for slot in modelA modelB modelC; do
.venv/bin/benchy eval \
--config configs/models/together_${slot}.yaml \
--tasks structured_extraction --limit 5 \
--run-id w2_smoke_${slot} --exit-policy smoke
done
-
Verify smoke statuses:
for slot in modelA modelB modelC; do
echo "=== $slot ==="
cat outputs/benchmark_outputs/w2_smoke_${slot}_LIMITED/*/run_outcome.json \
| jq '{status, duration_s, passed_subtasks: .counts.passed_subtasks}'
done
Acceptable: "passed" or "degraded". Investigate "failed" / "error" before proceeding.
-
SUBMISSION RUN — full structured_extraction without --limit (~2-5 min per model):
for slot in modelA modelB modelC; do
.venv/bin/benchy eval \
--config configs/models/together_${slot}.yaml \
--tasks structured_extraction \
--run-id w2_full_${slot} --exit-policy smoke
done
-
Confirm submission runs:
for slot in modelA modelB modelC; do
echo "=== $slot ==="
cat outputs/benchmark_outputs/w2_full_${slot}/*/run_outcome.json \
| jq '{status, duration_s, subtasks: .tasks.structured_extraction.subtasks | to_entries | map({name: .key, status: .value.status})}'
done
