{
  "model_name": "Claude-3.7-Sonnet",
  "model_organization": "Anthropic",
  "submitting_organization": "Sierra",
  "submission_date": "2025-06-09",
  "contact_info": {
    "email": "victor@sierra.ai, ben.s@sierra.ai",
    "name": "Sierra Research Team"
  },
  "is_new": false,
  "trajectories_available": false,
  "references": [
    {
      "title": "τ²-Bench: Evaluating Conversational Agents in a Dual-Control Environment",
      "url": "https://arxiv.org/abs/2506.07982",
      "type": "paper"
    }
  ],
  "results": {
    "retail": {
      "pass_1": 72.1,
      "pass_2": 66.8,
      "pass_3": 63.2,
      "pass_4": 59.7
    },
    "airline": {
      "pass_1": 64.2,
      "pass_2": 58.9,
      "pass_3": 55.4,
      "pass_4": 52.1
    },
    "telecom": {
      "pass_1": 49.0,
      "pass_2": 43.7,
      "pass_3": 40.1,
      "pass_4": 37.2
    }
  },
  "methodology": {
    "evaluation_date": "2025-06-09",
    "tau2_bench_version": "0.1.3",
    "user_simulator": "gpt-4.1-2025-04-14",
    "notes": "Original evaluation from tau2-bench paper",
    "verification": {
      "modified_prompts": false,
      "omitted_questions": false,
      "details": "Verified evaluation with full trajectory data available."
    }
  },
  "model_release": {
    "release_date": "2025-02-24",
    "announcement_url": "https://www.anthropic.com/news/claude-3-7-sonnet",
    "announcement_title": "Claude 3.7 Sonnet and Claude Code"
  }
}