SWE-Bench Lite recipe

import { loadSweBenchLite } from "@pleach/eval/benchmarks/swe-bench";

const cases = await loadSweBenchLite({ maxCases: 5 });
console.log(cases.length); // 5
console.log(cases[0].id);  // "swebench-lite/django__django-11099"

import { evalLab } from "@pleach/recipes/eval-lab";
import { loadSweBenchLite } from "@pleach/eval/benchmarks/swe-bench";
import { EvalSuite } from "@pleach/eval";

const lab = evalLab({
  suiteId: "swe-bench-lite-claude-sonnet-4-6",
  orchestratorConfig: {
    provider: "anthropic",
    model: "claude-sonnet-4-6",
    apiKey: process.env.ANTHROPIC_API_KEY!,
  },
  evalSuiteFactory: ({ suiteId, runtime }) =>
    new EvalSuite({ suiteId, runtime }),
});

const cases = await loadSweBenchLite({ maxCases: 10 });
for (const c of cases) lab.addCase(c);

const report = await lab.run();
console.log(`${report.summary.passed}/${report.summary.total}`);

import { runMatrixBatch, createModelSwapSubjects } from "@pleach/eval";
import { loadSweBenchLite } from "@pleach/eval/benchmarks/swe-bench";

const cases = await loadSweBenchLite({ maxCases: 10 });

const subjects = createModelSwapSubjects({
  models: ["claude-sonnet-4-6", "claude-opus-4-7", "claude-haiku-4-6"],
  buildSubject: (model) => ({
    id: model,
    run: async (scenario) => runCaseAgainstModel(model, scenario),
  }),
});

const matrixReport = await runMatrixBatch({
  subjects,
  scenarios,
  scorers,
});

import { createDivergenceReporter } from "@pleach/eval";

const reporter = createDivergenceReporter({ runs });

console.log(reporter.summary());
// {
//   subjectCount: 3,
//   caseCount: 10,
//   agreementRate: 0.6, // 6 of 10 cases all 3 subjects agreed
//   divergenceRate: 0.4,
// }

for (const row of reporter.perCase()) {
  if (row.divergences.length > 0) {
    console.log(row.caseId, row.divergences);
  }
}

// For a downstream chart:
const heatmap = reporter.export("markdown");

import { loadSweBenchLite } from "@pleach/eval/benchmarks/swe-bench";

const cases = await loadSweBenchLite();
const cases_with_real_scorer = cases.map((c) => ({
  ...c,
  scorer: {
    kind: "custom",
    fn: async (actualPatch, ctx) => {
      // `EvalCase.input` is typed `string | Record<string, unknown>`;
      // narrow to the SWE-Bench row shape before reading fields.
      const input = c.input as { repo: string; issueNumber: number };
      // Apply patch + run tests in your Docker harness.
      const result = await yourDockerHarness.grade({
        repo: input.repo,
        issueNumber: input.issueNumber,
        patch: actualPatch,
      });
      return {
        score: result.passToFail ? 1 : 0,
        explanation: result.summary,
      };
    },
  },
}));

import { createSandboxComposite } from "@pleach/coding-agent/sandbox";
import { createApplyDiffTool, createRunTestsTool } from "@pleach/coding-agent/tools";

const session = await provider.acquire();
const composite = createSandboxComposite({ session });

const cases_with_sandbox_scorer = cases.map((c) => ({
  ...c,
  scorer: {
    kind: "custom",
    fn: async (actualPatch, ctx) => {
      // `EvalCase.input` is typed `string | Record<string, unknown>`;
      // narrow to the SWE-Bench row shape before reading fields.
      const input = c.input as { repo: string; baseCommit: string };
      // 1. Clone repo at baseCommit.
      await composite.gitClone(`https://github.com/${input.repo}.git`, {
        ref: input.baseCommit,
      });
      // 2. Apply the model's patch via apply_diff tool.
      const apply = await applyDiff.execute(
        { path: ".", diff: actualPatch },
        { toolCallId: "swe-bench-grade" },
      );
      if (apply.conflicts.length > 0) {
        return { score: 0, explanation: `patch conflicts: ${apply.conflicts.join("; ")}` };
      }
      // 3. Run tests via run_tests tool.
      const tests = await runTests.execute(
        { command: "pytest -xvs", cwd: input.repo.split("/")[1] },
        { toolCallId: "swe-bench-grade" },
      );
      return {
        score: tests.failed === 0 && (tests.passed ?? 0) > 0 ? 1 : 0,
        explanation: `exit ${tests.exitCode}; passed ${tests.passed} failed ${tests.failed}`,
      };
    },
  },
}));