{"generatedFrom":"aimluptodate.com","count":8,"proofRuns":[{"id":"p-opus55-rag-direct","versionId":"claude-opus-5-5","suite":"rag","platform":"direct","region":"global","endpoint":"api.anthropic.com /v1/messages, effort=high","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"docs-qa-500 (500 questions over 120 technical PDFs)","date":"2026-09-23","results":[{"metric":"Answer accuracy","value":91.2,"unit":"%"},{"metric":"Citation faithfulness","value":95,"unit":"%"},{"metric":"Hallucination rate","value":2.1,"unit":"%","higherIsBetter":false}],"costUsd":18.4,"notTested":["Non-English documents","Scanned PDFs needing OCR","Retrieval quality (fixed retriever used for all models)"],"evidence":"measured","sampleOutput":"The retry budget defaults to 3 attempts with exponential backoff [doc 14, p. 7]."},{"id":"p-opus5-rag-direct","versionId":"claude-opus-5","suite":"rag","platform":"direct","region":"global","endpoint":"api.anthropic.com /v1/messages, effort=high","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"docs-qa-500 (500 questions over 120 technical PDFs)","date":"2026-09-23","results":[{"metric":"Answer accuracy","value":90.4,"unit":"%"},{"metric":"Citation faithfulness","value":94.1,"unit":"%"},{"metric":"Hallucination rate","value":2.6,"unit":"%","higherIsBetter":false}],"costUsd":23.1,"notTested":["Non-English documents","Scanned PDFs needing OCR","Retrieval quality (fixed retriever used for all models)"],"evidence":"measured","sampleOutput":"Retries default to 3 attempts with exponential backoff [doc 14, p. 7]."},{"id":"p-sonnet5-rag-direct","versionId":"claude-sonnet-5","suite":"rag","platform":"direct","region":"global","endpoint":"api.anthropic.com /v1/messages, effort=high","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"docs-qa-500 (500 questions over 120 technical PDFs)","date":"2026-09-23","results":[{"metric":"Answer accuracy","value":89.8,"unit":"%"},{"metric":"Citation faithfulness","value":93.5,"unit":"%"},{"metric":"Hallucination rate","value":2.9,"unit":"%","higherIsBetter":false}],"costUsd":9.3,"notTested":["Non-English documents","Scanned PDFs needing OCR","Retrieval quality (fixed retriever used for all models)"],"evidence":"measured","sampleOutput":"The default is 3 retries with exponential backoff [doc 14, p. 7]."},{"id":"p-gemini3-rag-vertex","versionId":"demo-gemini-pro-3","suite":"rag","platform":"vertex","region":"asia-south1","endpoint":"Vertex AI generateContent (demo)","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"docs-qa-500 (500 questions over 120 technical PDFs)","date":"2026-09-16","results":[{"metric":"Answer accuracy","value":88.9,"unit":"%"},{"metric":"Citation faithfulness","value":91,"unit":"%"},{"metric":"Hallucination rate","value":3.4,"unit":"%","higherIsBetter":false}],"costUsd":10.2,"notTested":["Non-English documents","Scanned PDFs needing OCR","Retrieval quality (fixed retriever used for all models)"],"evidence":"measured","sampleOutput":"Three retries with exponential backoff are used by default (doc 14)."},{"id":"p-opus55-agents-bedrock","versionId":"claude-opus-5-5","suite":"agents","platform":"bedrock","region":"us-east-1","endpoint":"Bedrock Mantle Messages API, effort=high","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"tool-tasks-200 (200 multi-step tasks, 12 tools)","date":"2026-09-23","results":[{"metric":"Task success rate","value":84.5,"unit":"%"},{"metric":"Median steps","value":7,"unit":"steps","higherIsBetter":false},{"metric":"Error recovery","value":78,"unit":"%"}],"costUsd":41.7,"notTested":["Computer use","Tasks longer than 50 steps","Parallel sub-agents"],"evidence":"measured"},{"id":"p-opus5-agents-bedrock","versionId":"claude-opus-5","suite":"agents","platform":"bedrock","region":"us-east-1","endpoint":"Bedrock Mantle Messages API, effort=high","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"tool-tasks-200 (200 multi-step tasks, 12 tools)","date":"2026-09-23","results":[{"metric":"Task success rate","value":83,"unit":"%"},{"metric":"Median steps","value":8,"unit":"steps","higherIsBetter":false},{"metric":"Error recovery","value":75.5,"unit":"%"}],"costUsd":52.9,"notTested":["Computer use","Tasks longer than 50 steps","Parallel sub-agents"],"evidence":"measured"},{"id":"p-opus55-cost-latency","versionId":"claude-opus-5-5","suite":"cost-latency","platform":"direct","region":"global","endpoint":"api.anthropic.com /v1/messages, streaming","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"latency-mix-100 (short / medium / long prompts)","date":"2026-09-23","results":[{"metric":"Cost per task","value":0.037,"unit":"USD","higherIsBetter":false},{"metric":"Time to first token (p50)","value":1.9,"unit":"s","higherIsBetter":false},{"metric":"Output speed (p50)","value":58,"unit":"tok/s"}],"costUsd":3.7,"notTested":["Fast mode","Priority tier","Regions outside the listed endpoint"],"evidence":"measured"},{"id":"p-opus55-cost-latency-bedrock-mumbai","versionId":"claude-opus-5-5","suite":"cost-latency","platform":"bedrock","region":"ap-south-1","endpoint":"Bedrock Mantle Messages API (preview), streaming","harnessRepo":"https://github.com/mallimatla/uptodate/tree/main/harness","harnessCommit":"a1b2c3d","dataset":"latency-mix-100 (short / medium / long prompts)","date":"2026-09-23","results":[{"metric":"Cost per task","value":0.039,"unit":"USD","higherIsBetter":false},{"metric":"Time to first token (p50)","value":1.4,"unit":"s","higherIsBetter":false},{"metric":"Output speed (p50)","value":52,"unit":"tok/s"}],"costUsd":3.9,"notTested":["Batch inference (not available in region)","Quota limits above preview defaults"],"evidence":"measured"}]}