update

Author: Aslemammad

agents/claude-code.ts

@@ -107,11 +107,12 @@ const claudeCodeAgent: AgentDefinition = {
     const cacheKey = sessionKey(cwd, model);
     const existingSessionID = sessionCache.get(cacheKey);
 
-    const actions: string[] = [];
-    const usage = {
-      input: 0,
-      output: 0,
-    };
+  const actions: string[] = [];
+  const usage = {
+    input: 0,
+    output: 0,
+    cost: 0,
+  };
 
     try {
       const result = query({
@@ -134,6 +135,7 @@ const claudeCodeAgent: AgentDefinition = {
         if (message.type === "result" && "usage" in message) {
           usage.input += message.usage.input_tokens || 0;
           usage.output += message.usage.output_tokens || 0;
+          usage.cost += message.total_cost_usd || 0;
         }
 
         actions.push(JSON.stringify(message));

agents/codex.ts

@@ -21,13 +21,13 @@ const DEFAULT_SANDBOX: SandboxMode = "workspace-write";
 const codexClient = new Codex();
 const threadCache = new Map<string, Thread>();
 
-export const models: string[] = [
+export const models = [
   "gpt-5-codex",
   "gpt-5.1-codex",
   // "gpt-5",
   // "o3",
   // "o4-mini"
-];
+] as const;
 
 function sessionKey(cwd: string, model: string): string {
   return `${cwd}::${model}`;
@@ -106,9 +106,9 @@ function getOrCreateThread(model: string, cwd: string): Thread {
   return thread;
 }
 
-const codexAgent: AgentDefinition = {
+const codexAgent: AgentDefinition<(typeof models)[number]> = {
   async run(
-    model: string,
+    model: (typeof models)[number],
     prompt: string,
     cwd: string,
     options?: AgentRunOptions,
@@ -130,10 +130,32 @@ const codexAgent: AgentDefinition = {
 
     const actions: string[] = [];
     let usage: Usage;
+    let cost = 0;
     try {
+      const pricingKey = model;
+      const pricing = openai.models[pricingKey]?.cost;
       const turn = await thread.run(prompt);
       assert(turn.usage, "The agent did not emit the usage information.");
       usage = turn.usage;
+      if (!pricing) {
+        if (!missingPricing.has(pricingKey)) {
+          missingPricing.add(pricingKey);
+          console.warn(
+            `[codex] Pricing not found for ${pricingKey}; using $0 for cost calculation.`,
+          );
+        }
+      } else {
+        const billableInput =
+          (usage.input_tokens ?? 0) - (usage.cached_input_tokens ?? 0);
+        const cachedInput = usage.cached_input_tokens ?? 0;
+        const output = usage.output_tokens ?? 0;
+        cost =
+          (billableInput * pricing.input +
+            output * pricing.output +
+            cachedInput * pricing.cache_read) /
+          1_000_000;
+      }
+
       actions.push(...turn.items.map((item) => JSON.stringify(item)));
       logTurnItems(turn.items, options);
     } catch (error) {
@@ -147,9 +169,28 @@ const codexAgent: AgentDefinition = {
       usage: {
         input: usage.input_tokens,
         output: usage.output_tokens,
+        cost,
       },
     };
   },
 };
 
 export default codexAgent;
+
+
+const response = await fetch("https://models.dev/api.json");
+if (!response.ok) {
+  throw new Error(`models.dev responded with ${response.status}`);
+}
+
+const openai = (await response.json())["openai"] as {
+  models: Record<string, {
+    cost: {
+      input: number,
+      output: number,
+      cache_read: number
+    }
+  }>
+}
+
+const missingPricing = new Set<string>();

agents/index.ts

@@ -2,21 +2,21 @@ import { strict as assert } from "node:assert";
 
 import type { AgentDefinition } from "~/lib/createAgent.js";
 
-export interface AgentRegistration {
+export interface AgentRegistration<TModel extends string = string> {
   name: string;
-  definition: AgentDefinition;
-  models: string[];
+  definition: AgentDefinition<TModel>;
+  models: ReadonlyArray<TModel>;
 }
 
-interface AgentModuleShape {
-  default?: AgentDefinition;
-  models?: string[];
+interface AgentModuleShape<TModel extends string = string> {
+  default?: AgentDefinition<TModel>;
+  models?: ReadonlyArray<TModel>;
 }
 
-function createAgentRegistration(
+function createAgentRegistration<TModel extends string>(
   name: string,
-  module: AgentModuleShape,
-): AgentRegistration {
+  module: AgentModuleShape<TModel>,
+): AgentRegistration<TModel> {
   const definition = module.default;
   const models = module.models;
 
@@ -26,7 +26,7 @@ function createAgentRegistration(
   return { name, definition, models };
 }
 
-const agents: Record<string, AgentRegistration> = {
+const agents: Record<string, AgentRegistration<any>> = {
   codex: createAgentRegistration("codex", await import("~/agents/codex.js")),
   opencode: createAgentRegistration(
     "opencode",

agents/opencode.ts

@@ -169,11 +169,12 @@ const opencodeAgent: AgentDefinition = {
       sessionCache.set(cacheKey, sessionID);
     }
 
-    const actions: string[] = [];
-    const usage = {
-      input: 0,
-      output: 0,
-    };
+  const actions: string[] = [];
+  const usage = {
+    input: 0,
+    output: 0,
+    cost: 0,
+  };
     try {
       const [providerID, modelID] = model.split("/");
 
@@ -196,6 +197,7 @@ const opencodeAgent: AgentDefinition = {
       const info = data.info;
       usage.input = info?.tokens?.input ?? 0;
       usage.output = info?.tokens?.output ?? 0;
+      usage.cost = info?.cost ?? 0;
 
       if (info) {
         actions.push(JSON.stringify(info));

bun.lock

@@ -1,6 +1,5 @@
 {
   "lockfileVersion": 1,
-  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "opencode-bench",

cli.ts

@@ -37,7 +37,7 @@ interface EpisodeResult {
   logs: string[];
   actions: string[];
   usage: Usage;
-  durationMs: number;
+  duration: number;
 }
 
 const evalIds = dataset
@@ -172,15 +172,14 @@ cli.command(
             console.log(
               `${prefix} Starting episode (timeout: ${timeoutInMinutes} min)...`,
             );
-            const startedAt = Date.now();
             const result = await runEpisode(
               evalDef,
               agent,
               model,
               tasks,
               prefix,
             );
-            return { index, durationMs: Date.now() - startedAt, ...result };
+            return { index, ...result };
           },
           {
             timeoutMs: timeoutInMinutes * 60 * 1000,
@@ -226,17 +225,15 @@ cli.command(
       (prev, { usage }) => ({
         input: prev.input + usage.input / episodeResults.length,
         output: prev.output + usage.output / episodeResults.length,
+        cost: prev.cost + usage.cost / episodeResults.length,
       }),
-      { input: 0, output: 0 },
+      { input: 0, output: 0, cost: 0 },
     );
 
-    const totalDurationMs = episodeResults.reduce(
-      (prev, { durationMs }) => prev + durationMs,
+    const totalDuration = episodeResults.reduce(
+      (prev, { duration }) => prev + duration,
       0,
     );
-    const totalTokens = averageUsage.input + averageUsage.output;
-    const tokensPerSecond =
-      totalDurationMs > 0 ? totalTokens / (totalDurationMs / 1000) : 0;
 
     for (const result of episodeResults) {
       mergeAggregationInputs(aggregatedInputs, result.aggregation);
@@ -281,8 +278,7 @@ cli.command(
       episodeExports,
       averageUsage,
       summary,
-      totalDurationMs,
-      tokensPerSecond,
+      totalDuration,
     );
 
     printEvalResult(evaluationResult);
@@ -316,6 +312,7 @@ async function runEpisode(
 ) {
   const baselineCommit = evalDef.from;
   let cwd: string | undefined;
+  let episodeDuration = 0;
 
   try {
     console.log(`${prefix} Cloning repository...`);
@@ -345,15 +342,20 @@ async function runEpisode(
     }
 
     let tasksExecuted = 0;
-    let usage: Usage = { input: 0, output: 0 };
+    let usage: Usage = { input: 0, output: 0, cost: 0 };
     const episodeActions: string[] = [];
+    let episodeDuration = 0;
 
     for (const task of tasks) {
       const logPrefix = `${prefix} ${task.commit}`;
 
       try {
+        let successfulRunDuration = 0;
+        // TODO: retrying the agent runs here means if the agent did half of the work, the next agent would come up and continue those changes which is not correct.
+        // the agent should start from a clean state again and do the work. so the whole loop should be restarted.
         const result = await withRetries(
           async () => {
+            const startedAt = Date.now();
             const result = await agent.definition.run(
               model,
               task.prompt,
@@ -365,6 +367,7 @@ async function runEpisode(
                 logPrefix,
               },
             );
+            successfulRunDuration = Date.now() - startedAt;
             return result;
           },
           {
@@ -386,10 +389,12 @@ async function runEpisode(
             },
           },
         );
+        episodeDuration += successfulRunDuration;
 
         // Only accumulate usage from the successful result
         usage.input += result.usage.input;
         usage.output += result.usage.output;
+        usage.cost += result.usage.cost;
 
         // Collect actions from this task
         episodeActions.push(...result.actions);
@@ -447,6 +452,7 @@ async function runEpisode(
       logs: [],
       actions: episodeActions,
       usage,
+      duration: episodeDuration,
     };
   } finally {
     if (cwd) {
@@ -633,8 +639,7 @@ function summarizeAggregation(
   episodes: Episode[],
   usage: Usage,
   summary: string,
-  durationMs: number,
-  tokensPerSecond: number,
+  duration: number,
 ): { lines: string[]; exportData: EvaluationRunExport } {
   const evalId = datasetEval.repo;
   const runContext = contextLabel ? `${evalId} [${contextLabel}]` : evalId;
@@ -697,8 +702,7 @@ function summarizeAggregation(
     episodes,
     usage,
     summary,
-    durationMs,
-    tokensPerSecond,
+    duration,
   };
 
   return { lines, exportData };

lib/createAgent.ts

@@ -13,9 +13,9 @@ export type AgentExecutor = (
   prompt: AgentPrompt,
 ) => AgentCommandSpec | Promise<AgentCommandSpec>;
 
-export interface AgentDefinition {
+export interface AgentDefinition<TModel extends string = string> {
   run: (
-    model: string,
+    model: TModel,
     prompt: AgentPrompt,
     cwd: string,
     options?: AgentRunOptions,
@@ -29,6 +29,7 @@ export interface AgentRunResult {
   usage: {
     input: number;
     output: number;
+    cost: number;
   };
 }