[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill-sourcegraph-evaluate":3,"mdc-w8l0ww-key":35,"related-org-sourcegraph-evaluate":451,"related-repo-sourcegraph-evaluate":564},{"slug":4,"name":4,"fn":5,"description":6,"org":7,"tags":11,"stars":25,"repoUrl":26,"updatedAt":27,"license":28,"forks":29,"topics":30,"repo":31,"sourceUrl":33,"mdContent":34},"evaluate","score traces and evaluate benchmark results","Extract metrics, score traces, and evaluate benchmark task results.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},"sourcegraph","Sourcegraph","https:\u002F\u002Fpexgzepcugksgbtrxkhf.supabase.co\u002Fstorage\u002Fv1\u002Fobject\u002Fpublic\u002Forg-logos\u002Fsourcegraph.png",[12,16,19,22],{"name":13,"slug":14,"type":15},"Benchmarking","benchmarking","tag",{"name":17,"slug":18,"type":15},"Evals","evals",{"name":20,"slug":21,"type":15},"Data Analysis","data-analysis",{"name":23,"slug":24,"type":15},"Engineering","engineering",31,"https:\u002F\u002Fgithub.com\u002Fsourcegraph\u002FCodeScaleBench","2026-07-16T06:04:41.910821",null,4,[],{"repoUrl":26,"stars":25,"forks":29,"topics":32,"description":28},[],"https:\u002F\u002Fgithub.com\u002Fsourcegraph\u002FCodeScaleBench\u002Ftree\u002FHEAD\u002Fskills\u002Fevaluate","---\nname: evaluate\ndescription: Extract metrics, score traces, and evaluate benchmark task results.\n---\n\n# Skill: Evaluate Results\n\n## Scope\n\nUse this skill when the user asks to:\n- Extract and compute task metrics from completed runs\n- Score traces and verify dual verification\n- Re-extract metrics after trace updates\n- Compare verifier outputs and validate scoring\n- Generate evaluation reports\n\n## Canonical Commands\n\n```bash\n# Extract task metrics from a run\npython3 scripts\u002Fevaluation\u002Fextract_task_metrics.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Score all traces in a run\npython3 scripts\u002Fevaluation\u002Ftrace_quality_pipeline.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Reextract metrics for specific suite\npython3 scripts\u002Fevaluation\u002Freextract_all_metrics.py --filter csb_sdlc_debug\n\n# Compute retrieval metrics (IR tasks)\npython3 scripts\u002Fevaluation\u002Fcompute_retrieval_metrics.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Dual verification check\npython3 scripts\u002Fevaluation\u002Fcross_validate_oracles.py --run-dir runs\u002Fstaging\u002Frun_dir\n```\n\n## Metrics Pipeline\n\n1. **Trace extraction** → parse trajectory.json for tool calls, outputs, reasoning\n2. **Oracle verification** → compare against ground truth (local + Sourcegraph)\n3. **Dual scoring** → compute reward for baseline and MCP configs\n4. **Aggregation** → roll up to suite and model level\n5. **Quality gates** → flag inconsistencies and outliers\n\n## Verification Modes\n\n- **Local verification** — on-disk oracle checks\n- **Sourcegraph (SG) verification** — MCP-based symbol resolution\n- **Dual** — both pass (canonical tasks)\n\n## Related Skills\n\n- `\u002Frun` — launch runs to generate traces\n- `\u002Freport` — compile evaluation reports\n- `\u002Faudit` — validate trace quality and consistency\n",{"data":36,"body":37},{"name":4,"description":6},{"type":38,"children":39},"root",[40,49,56,62,92,98,303,309,364,370,403,409,445],{"type":41,"tag":42,"props":43,"children":45},"element","h1",{"id":44},"skill-evaluate-results",[46],{"type":47,"value":48},"text","Skill: Evaluate Results",{"type":41,"tag":50,"props":51,"children":53},"h2",{"id":52},"scope",[54],{"type":47,"value":55},"Scope",{"type":41,"tag":57,"props":58,"children":59},"p",{},[60],{"type":47,"value":61},"Use this skill when the user asks to:",{"type":41,"tag":63,"props":64,"children":65},"ul",{},[66,72,77,82,87],{"type":41,"tag":67,"props":68,"children":69},"li",{},[70],{"type":47,"value":71},"Extract and compute task metrics from completed runs",{"type":41,"tag":67,"props":73,"children":74},{},[75],{"type":47,"value":76},"Score traces and verify dual verification",{"type":41,"tag":67,"props":78,"children":79},{},[80],{"type":47,"value":81},"Re-extract metrics after trace updates",{"type":41,"tag":67,"props":83,"children":84},{},[85],{"type":47,"value":86},"Compare verifier outputs and validate scoring",{"type":41,"tag":67,"props":88,"children":89},{},[90],{"type":47,"value":91},"Generate evaluation reports",{"type":41,"tag":50,"props":93,"children":95},{"id":94},"canonical-commands",[96],{"type":47,"value":97},"Canonical Commands",{"type":41,"tag":99,"props":100,"children":105},"pre",{"className":101,"code":102,"language":103,"meta":104,"style":104},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# Extract task metrics from a run\npython3 scripts\u002Fevaluation\u002Fextract_task_metrics.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Score all traces in a run\npython3 scripts\u002Fevaluation\u002Ftrace_quality_pipeline.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Reextract metrics for specific suite\npython3 scripts\u002Fevaluation\u002Freextract_all_metrics.py --filter csb_sdlc_debug\n\n# Compute retrieval metrics (IR tasks)\npython3 scripts\u002Fevaluation\u002Fcompute_retrieval_metrics.py --run-dir runs\u002Fstaging\u002Frun_dir\n\n# Dual verification check\npython3 scripts\u002Fevaluation\u002Fcross_validate_oracles.py --run-dir runs\u002Fstaging\u002Frun_dir\n","bash","",[106],{"type":41,"tag":107,"props":108,"children":109},"code",{"__ignoreMap":104},[110,122,148,158,166,187,195,204,227,235,244,265,273,282],{"type":41,"tag":111,"props":112,"children":115},"span",{"class":113,"line":114},"line",1,[116],{"type":41,"tag":111,"props":117,"children":119},{"style":118},"--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic",[120],{"type":47,"value":121},"# Extract task metrics from a run\n",{"type":41,"tag":111,"props":123,"children":125},{"class":113,"line":124},2,[126,132,138,143],{"type":41,"tag":111,"props":127,"children":129},{"style":128},"--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B",[130],{"type":47,"value":131},"python3",{"type":41,"tag":111,"props":133,"children":135},{"style":134},"--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D",[136],{"type":47,"value":137}," scripts\u002Fevaluation\u002Fextract_task_metrics.py",{"type":41,"tag":111,"props":139,"children":140},{"style":134},[141],{"type":47,"value":142}," --run-dir",{"type":41,"tag":111,"props":144,"children":145},{"style":134},[146],{"type":47,"value":147}," runs\u002Fstaging\u002Frun_dir\n",{"type":41,"tag":111,"props":149,"children":151},{"class":113,"line":150},3,[152],{"type":41,"tag":111,"props":153,"children":155},{"emptyLinePlaceholder":154},true,[156],{"type":47,"value":157},"\n",{"type":41,"tag":111,"props":159,"children":160},{"class":113,"line":29},[161],{"type":41,"tag":111,"props":162,"children":163},{"style":118},[164],{"type":47,"value":165},"# Score all traces in a run\n",{"type":41,"tag":111,"props":167,"children":169},{"class":113,"line":168},5,[170,174,179,183],{"type":41,"tag":111,"props":171,"children":172},{"style":128},[173],{"type":47,"value":131},{"type":41,"tag":111,"props":175,"children":176},{"style":134},[177],{"type":47,"value":178}," scripts\u002Fevaluation\u002Ftrace_quality_pipeline.py",{"type":41,"tag":111,"props":180,"children":181},{"style":134},[182],{"type":47,"value":142},{"type":41,"tag":111,"props":184,"children":185},{"style":134},[186],{"type":47,"value":147},{"type":41,"tag":111,"props":188,"children":190},{"class":113,"line":189},6,[191],{"type":41,"tag":111,"props":192,"children":193},{"emptyLinePlaceholder":154},[194],{"type":47,"value":157},{"type":41,"tag":111,"props":196,"children":198},{"class":113,"line":197},7,[199],{"type":41,"tag":111,"props":200,"children":201},{"style":118},[202],{"type":47,"value":203},"# Reextract metrics for specific suite\n",{"type":41,"tag":111,"props":205,"children":207},{"class":113,"line":206},8,[208,212,217,222],{"type":41,"tag":111,"props":209,"children":210},{"style":128},[211],{"type":47,"value":131},{"type":41,"tag":111,"props":213,"children":214},{"style":134},[215],{"type":47,"value":216}," scripts\u002Fevaluation\u002Freextract_all_metrics.py",{"type":41,"tag":111,"props":218,"children":219},{"style":134},[220],{"type":47,"value":221}," --filter",{"type":41,"tag":111,"props":223,"children":224},{"style":134},[225],{"type":47,"value":226}," csb_sdlc_debug\n",{"type":41,"tag":111,"props":228,"children":230},{"class":113,"line":229},9,[231],{"type":41,"tag":111,"props":232,"children":233},{"emptyLinePlaceholder":154},[234],{"type":47,"value":157},{"type":41,"tag":111,"props":236,"children":238},{"class":113,"line":237},10,[239],{"type":41,"tag":111,"props":240,"children":241},{"style":118},[242],{"type":47,"value":243},"# Compute retrieval metrics (IR tasks)\n",{"type":41,"tag":111,"props":245,"children":247},{"class":113,"line":246},11,[248,252,257,261],{"type":41,"tag":111,"props":249,"children":250},{"style":128},[251],{"type":47,"value":131},{"type":41,"tag":111,"props":253,"children":254},{"style":134},[255],{"type":47,"value":256}," scripts\u002Fevaluation\u002Fcompute_retrieval_metrics.py",{"type":41,"tag":111,"props":258,"children":259},{"style":134},[260],{"type":47,"value":142},{"type":41,"tag":111,"props":262,"children":263},{"style":134},[264],{"type":47,"value":147},{"type":41,"tag":111,"props":266,"children":268},{"class":113,"line":267},12,[269],{"type":41,"tag":111,"props":270,"children":271},{"emptyLinePlaceholder":154},[272],{"type":47,"value":157},{"type":41,"tag":111,"props":274,"children":276},{"class":113,"line":275},13,[277],{"type":41,"tag":111,"props":278,"children":279},{"style":118},[280],{"type":47,"value":281},"# Dual verification check\n",{"type":41,"tag":111,"props":283,"children":285},{"class":113,"line":284},14,[286,290,295,299],{"type":41,"tag":111,"props":287,"children":288},{"style":128},[289],{"type":47,"value":131},{"type":41,"tag":111,"props":291,"children":292},{"style":134},[293],{"type":47,"value":294}," scripts\u002Fevaluation\u002Fcross_validate_oracles.py",{"type":41,"tag":111,"props":296,"children":297},{"style":134},[298],{"type":47,"value":142},{"type":41,"tag":111,"props":300,"children":301},{"style":134},[302],{"type":47,"value":147},{"type":41,"tag":50,"props":304,"children":306},{"id":305},"metrics-pipeline",[307],{"type":47,"value":308},"Metrics Pipeline",{"type":41,"tag":310,"props":311,"children":312},"ol",{},[313,324,334,344,354],{"type":41,"tag":67,"props":314,"children":315},{},[316,322],{"type":41,"tag":317,"props":318,"children":319},"strong",{},[320],{"type":47,"value":321},"Trace extraction",{"type":47,"value":323}," → parse trajectory.json for tool calls, outputs, reasoning",{"type":41,"tag":67,"props":325,"children":326},{},[327,332],{"type":41,"tag":317,"props":328,"children":329},{},[330],{"type":47,"value":331},"Oracle verification",{"type":47,"value":333}," → compare against ground truth (local + Sourcegraph)",{"type":41,"tag":67,"props":335,"children":336},{},[337,342],{"type":41,"tag":317,"props":338,"children":339},{},[340],{"type":47,"value":341},"Dual scoring",{"type":47,"value":343}," → compute reward for baseline and MCP configs",{"type":41,"tag":67,"props":345,"children":346},{},[347,352],{"type":41,"tag":317,"props":348,"children":349},{},[350],{"type":47,"value":351},"Aggregation",{"type":47,"value":353}," → roll up to suite and model level",{"type":41,"tag":67,"props":355,"children":356},{},[357,362],{"type":41,"tag":317,"props":358,"children":359},{},[360],{"type":47,"value":361},"Quality gates",{"type":47,"value":363}," → flag inconsistencies and outliers",{"type":41,"tag":50,"props":365,"children":367},{"id":366},"verification-modes",[368],{"type":47,"value":369},"Verification Modes",{"type":41,"tag":63,"props":371,"children":372},{},[373,383,393],{"type":41,"tag":67,"props":374,"children":375},{},[376,381],{"type":41,"tag":317,"props":377,"children":378},{},[379],{"type":47,"value":380},"Local verification",{"type":47,"value":382}," — on-disk oracle checks",{"type":41,"tag":67,"props":384,"children":385},{},[386,391],{"type":41,"tag":317,"props":387,"children":388},{},[389],{"type":47,"value":390},"Sourcegraph (SG) verification",{"type":47,"value":392}," — MCP-based symbol resolution",{"type":41,"tag":67,"props":394,"children":395},{},[396,401],{"type":41,"tag":317,"props":397,"children":398},{},[399],{"type":47,"value":400},"Dual",{"type":47,"value":402}," — both pass (canonical tasks)",{"type":41,"tag":50,"props":404,"children":406},{"id":405},"related-skills",[407],{"type":47,"value":408},"Related Skills",{"type":41,"tag":63,"props":410,"children":411},{},[412,423,434],{"type":41,"tag":67,"props":413,"children":414},{},[415,421],{"type":41,"tag":107,"props":416,"children":418},{"className":417},[],[419],{"type":47,"value":420},"\u002Frun",{"type":47,"value":422}," — launch runs to generate traces",{"type":41,"tag":67,"props":424,"children":425},{},[426,432],{"type":41,"tag":107,"props":427,"children":429},{"className":428},[],[430],{"type":47,"value":431},"\u002Freport",{"type":47,"value":433}," — compile evaluation reports",{"type":41,"tag":67,"props":435,"children":436},{},[437,443],{"type":41,"tag":107,"props":438,"children":440},{"className":439},[],[441],{"type":47,"value":442},"\u002Faudit",{"type":47,"value":444}," — validate trace quality and consistency",{"type":41,"tag":446,"props":447,"children":448},"style",{},[449],{"type":47,"value":450},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"items":452,"total":229},[453,466,473,488,502,516,530,540,551],{"slug":454,"name":454,"fn":455,"description":456,"org":457,"tags":458,"stars":25,"repoUrl":26,"updatedAt":465},"audit","audit repository health and benchmark integrity","Run repo health checks, validate benchmark tasks, and audit run integrity.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[459,461,462],{"name":460,"slug":454,"type":15},"Audit",{"name":13,"slug":14,"type":15},{"name":463,"slug":464,"type":15},"QA","qa","2026-07-17T06:07:07.220218",{"slug":4,"name":4,"fn":5,"description":6,"org":467,"tags":468,"stars":25,"repoUrl":26,"updatedAt":27},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[469,470,471,472],{"name":13,"slug":14,"type":15},{"name":20,"slug":21,"type":15},{"name":23,"slug":24,"type":15},{"name":17,"slug":18,"type":15},{"slug":474,"name":474,"fn":475,"description":476,"org":477,"tags":478,"stars":25,"repoUrl":26,"updatedAt":487},"infra","check infrastructure and system dependencies","Check infrastructure readiness, manage MCP tools, and audit system dependencies.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[479,480,481,484],{"name":460,"slug":454,"type":15},{"name":23,"slug":24,"type":15},{"name":482,"slug":483,"type":15},"Infrastructure","infrastructure",{"name":485,"slug":486,"type":15},"MCP","mcp","2026-07-16T06:04:41.188274",{"slug":489,"name":489,"fn":490,"description":491,"org":492,"tags":493,"stars":25,"repoUrl":26,"updatedAt":501},"next","plan benchmarking and coverage tasks","Plan upcoming work, analyze coverage gaps, and recommend next steps for benchmarking.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[494,495,498],{"name":13,"slug":14,"type":15},{"name":496,"slug":497,"type":15},"Planning","planning",{"name":499,"slug":500,"type":15},"Strategy","strategy","2026-07-17T06:06:57.69018",{"slug":503,"name":503,"fn":504,"description":505,"org":506,"tags":507,"stars":25,"repoUrl":26,"updatedAt":515},"report","generate CodeScaleBench evaluation reports","Generate evaluation reports, analyze run costs, and compare configurations.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[508,511,512],{"name":509,"slug":510,"type":15},"Analytics","analytics",{"name":13,"slug":14,"type":15},{"name":513,"slug":514,"type":15},"Reporting","reporting","2026-07-16T06:02:36.809556",{"slug":517,"name":517,"fn":518,"description":519,"org":520,"tags":521,"stars":25,"repoUrl":26,"updatedAt":529},"run","manage CodeScaleBench benchmark runs","Launch and manage CodeScaleBench benchmark runs with paired-run guardrails, quick reruns, and execution orchestration.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[522,525,526],{"name":523,"slug":524,"type":15},"Automation","automation",{"name":13,"slug":14,"type":15},{"name":527,"slug":528,"type":15},"Testing","testing","2026-07-16T06:04:40.848817",{"slug":531,"name":531,"fn":532,"description":533,"org":534,"tags":535,"stars":25,"repoUrl":26,"updatedAt":539},"scaffold","create and validate benchmark tasks","Create, mine, and validate new benchmark tasks and task suites.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[536,537,538],{"name":13,"slug":14,"type":15},{"name":23,"slug":24,"type":15},{"name":527,"slug":528,"type":15},"2026-07-16T06:04:41.525889",{"slug":541,"name":541,"fn":542,"description":543,"org":544,"tags":545,"stars":25,"repoUrl":26,"updatedAt":550},"status","monitor benchmark execution and task status","Monitor active runs, check task completion status, and watch benchmark execution progress.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[546,547],{"name":13,"slug":14,"type":15},{"name":548,"slug":549,"type":15},"Monitoring","monitoring","2026-07-16T06:02:37.137106",{"slug":552,"name":552,"fn":553,"description":554,"org":555,"tags":556,"stars":25,"repoUrl":26,"updatedAt":563},"triage","triage and analyze failed benchmark tasks","Investigate and triage failed benchmark tasks, analyze root causes, and plan reruns.",{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[557,558,561],{"name":13,"slug":14,"type":15},{"name":559,"slug":560,"type":15},"Debugging","debugging",{"name":562,"slug":552,"type":15},"Triage","2026-07-16T06:02:37.472337",{"items":565,"total":229},[566,572,579,586,592,598,604],{"slug":454,"name":454,"fn":455,"description":456,"org":567,"tags":568,"stars":25,"repoUrl":26,"updatedAt":465},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[569,570,571],{"name":460,"slug":454,"type":15},{"name":13,"slug":14,"type":15},{"name":463,"slug":464,"type":15},{"slug":4,"name":4,"fn":5,"description":6,"org":573,"tags":574,"stars":25,"repoUrl":26,"updatedAt":27},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[575,576,577,578],{"name":13,"slug":14,"type":15},{"name":20,"slug":21,"type":15},{"name":23,"slug":24,"type":15},{"name":17,"slug":18,"type":15},{"slug":474,"name":474,"fn":475,"description":476,"org":580,"tags":581,"stars":25,"repoUrl":26,"updatedAt":487},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[582,583,584,585],{"name":460,"slug":454,"type":15},{"name":23,"slug":24,"type":15},{"name":482,"slug":483,"type":15},{"name":485,"slug":486,"type":15},{"slug":489,"name":489,"fn":490,"description":491,"org":587,"tags":588,"stars":25,"repoUrl":26,"updatedAt":501},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[589,590,591],{"name":13,"slug":14,"type":15},{"name":496,"slug":497,"type":15},{"name":499,"slug":500,"type":15},{"slug":503,"name":503,"fn":504,"description":505,"org":593,"tags":594,"stars":25,"repoUrl":26,"updatedAt":515},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[595,596,597],{"name":509,"slug":510,"type":15},{"name":13,"slug":14,"type":15},{"name":513,"slug":514,"type":15},{"slug":517,"name":517,"fn":518,"description":519,"org":599,"tags":600,"stars":25,"repoUrl":26,"updatedAt":529},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[601,602,603],{"name":523,"slug":524,"type":15},{"name":13,"slug":14,"type":15},{"name":527,"slug":528,"type":15},{"slug":531,"name":531,"fn":532,"description":533,"org":605,"tags":606,"stars":25,"repoUrl":26,"updatedAt":539},{"slug":8,"name":9,"logoUrl":10,"githubOrg":8},[607,608,609],{"name":13,"slug":14,"type":15},{"name":23,"slug":24,"type":15},{"name":527,"slug":528,"type":15}]