{ "suite": "Luna Practical v1", "product": "Luna 7 Mega", "date": "2026-09-21T18:50:37.989Z", "app_version": "0.8.28", "kind": "Internal component eval; not an independent benchmark or end-to-end installed app test", "conditions": { "repeats": 2, "order": "two sequential passes; no retries; fresh context per question", "profile": "unchanged production Provider, default low effort, neutral tone, brief response; no owner custom rules", "tools": "tool choices recorded but not executed; only generated JS pure functions executed in isolated VM", "machine": { "os": "win32", "arch": "x64", "cpu": "AMD Ryzen 9 9950X3D 16-Core Processor ", "node": "v24.19.0" }, "network": "local Windows client to cloud model; server hardware is unknown; latency includes network and queueing" }, "suite_sha256": "da6fa921ac99741883e06398f404dd48f284aab7588f70560306748ce5266ac7", "groups": { "code": { "passed": 12, "total": 12, "latency": { "n": 12, "median_ms": 9422, "p95_ms": 12907, "min_ms": 6836, "max_ms": 12907 } }, "analysis": { "passed": 8, "total": 8, "latency": { "n": 8, "median_ms": 2086.5, "p95_ms": 2796, "min_ms": 1684, "max_ms": 2796 } }, "assistant": { "passed": 6, "total": 6, "latency": { "n": 6, "median_ms": 1899.5, "p95_ms": 2549, "min_ms": 1477, "max_ms": 2549 } }, "tool": { "passed": 6, "total": 6, "latency": { "n": 6, "median_ms": 2664, "p95_ms": 4802, "min_ms": 2294, "max_ms": 4802 } } }, "passed": 32, "total": 32, "latency": { "n": 32, "median_ms": 2664, "p95_ms": 12379, "min_ms": 1477, "max_ms": 12907 }, "first_text": { "n": 23, "median_ms": 1828, "p95_ms": 6037, "min_ms": 1185, "max_ms": 7018 }, "tool_grading": { "app": "Exactly one business tool: list_installed_apps; query contains Audacity.", "search": "Exactly one business tool: search_web; query contains Одесс, Odesa or Odessa.", "browser": "Exactly one business tool: open_browser; URL exactly https://merchant.luna.invalid/." }, "rows": [ { "id": "intervals", "title": "Объединение интервалов", "category": "code", "repeat": 1, "passed": true, "total_ms": 8353, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "write_file", "args": { "path": "solve.js", "content": "function solve(input) {\n const sorted = input.map(([start, end]) => [start, end]);\n sorted.sort((a, b) => a[0] - b[0] || a[1] - b[1]);\n\n const result = [];\n for (const interval of sorted) {\n const last = result[result.length - 1];\n if (last && interval[0] <= last[1]) {\n last[1] = Math.max(last[1], interval[1]);\n } else {\n result.push(interval);\n }\n }\n return result;\n}\n" } } ], "code_source": "write_file payload (not executed as a tool)", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ [ 1, 7 ] ], "unchanged": true, "expected": [ [ 1, 7 ] ] }, { "passed": true, "actual": [ [ 1, 2 ], [ 4, 5 ] ], "unchanged": true, "expected": [ [ 1, 2 ], [ 4, 5 ] ] }, { "passed": true, "actual": [ [ -5, 0 ] ], "unchanged": true, "expected": [ [ -5, 0 ] ] }, { "passed": true, "actual": [ [ 1, 9 ] ], "unchanged": true, "expected": [ [ 1, 9 ] ] } ] }, { "id": "csv", "title": "CSV с кавычками", "category": "code", "repeat": 1, "passed": true, "total_ms": 9249, "first_text_ms": 5221, "answer": "function solve(input) {\n if (input === \"\") return [];\n\n const rows = [];\n let row = [];\n let cell = \"\";\n let quoted = false;\n\n for (let i = 0; i < input.length; i++) {\n const ch = input[i];\n\n if (quoted) {\n if (ch === '\"') {\n if (input[i + 1] === '\"') {\n cell += '\"';\n i++;\n } else {\n quoted = false;\n }\n } else {\n cell += ch;\n }\n } else if (ch === '\"') {\n quoted = true;\n } else if (ch === \",\") {\n row.push(cell);\n cell = \"\";\n } else if (ch === \"\\n\") {\n row.push(cell);\n rows.push(row);\n row = [];\n cell = \"\";\n } else {\n cell += ch;\n }\n }\n\n if (input[input.length - 1] !== \"\\n\") {\n row.push(cell);\n rows.push(row);\n }\n\n return rows;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ [ "a", "b" ], [ "c", "d" ] ], "unchanged": true, "expected": [ [ "a", "b" ], [ "c", "d" ] ] }, { "passed": true, "actual": [ [ "a,b", "c" ] ], "unchanged": true, "expected": [ [ "a,b", "c" ] ] }, { "passed": true, "actual": [ [ "a\"b", "x" ] ], "unchanged": true, "expected": [ [ "a\"b", "x" ] ] }, { "passed": true, "actual": [ [ "a\nb", "", "c" ], [ "x", "y", "z" ] ], "unchanged": true, "expected": [ [ "a\nb", "", "c" ], [ "x", "y", "z" ] ] }, { "passed": true, "actual": [ [ "", "" ] ], "unchanged": true, "expected": [ [ "", "" ] ] } ] }, { "id": "topology", "title": "Зависимости задач", "category": "code", "repeat": 1, "passed": true, "total_ms": 11576, "first_text_ms": 3139, "answer": "function solve(input) {\n const adjacency = new Map();\n const indegree = new Map();\n\n for (const node of input.nodes) {\n adjacency.set(node, new Set());\n indegree.set(node, 0);\n }\n\n for (const [a, b] of input.edges) {\n const neighbors = adjacency.get(a);\n if (!neighbors.has(b)) {\n neighbors.add(b);\n indegree.set(b, indegree.get(b) + 1);\n }\n }\n\n const heap = [];\n\n function push(value) {\n let i = heap.length;\n heap.push(value);\n while (i > 0) {\n const parent = (i - 1) >> 1;\n if (heap[parent] <= value) break;\n heap[i] = heap[parent];\n i = parent;\n }\n heap[i] = value;\n }\n\n function pop() {\n const minimum = heap[0];\n const last = heap.pop();\n if (heap.length > 0) {\n let i = 0;\n while (true) {\n let child = 2 * i + 1;\n if (child >= heap.length) break;\n if (child + 1 < heap.length && heap[child + 1] < heap[child]) {\n child++;\n }\n if (last <= heap[child]) break;\n heap[i] = heap[child];\n i = child;\n }\n heap[i] = last;\n }\n return minimum;\n }\n\n for (const node of input.nodes) {\n if (indegree.get(node) === 0) push(node);\n }\n\n const result = [];\n while (heap.length > 0) {\n const node = pop();\n result.push(node);\n for (const neighbor of adjacency.get(node)) {\n const degree = indegree.get(neighbor) - 1;\n indegree.set(neighbor, degree);\n if (degree === 0) push(neighbor);\n }\n }\n\n return result.length === input.nodes.length ? result : null;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ "a", "b", "c" ], "unchanged": true, "expected": [ "a", "b", "c" ] }, { "passed": true, "actual": null, "unchanged": true, "expected": null }, { "passed": true, "actual": [ "a", "b" ], "unchanged": true, "expected": [ "a", "b" ] }, { "passed": true, "actual": null, "unchanged": true, "expected": null } ] }, { "id": "money", "title": "Деньги без потери копеек", "category": "code", "repeat": 1, "passed": true, "total_ms": 9213, "first_text_ms": 7018, "answer": "function solve(input) {\n return input.map((value) => {\n if (typeof value !== \"string\") return null;\n\n const match = /^(-?)([0-9]+)(?:[.,]([0-9]{1,2}))?$/.exec(value.trim());\n if (!match) return null;\n\n const fraction = (match[3] || \"\").padEnd(2, \"0\");\n const cents = BigInt(match[2]) * 100n + BigInt(fraction);\n\n return Number(match[1] === \"-\" ? -cents : cents);\n });\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [ 0, 100, 110, 101, -1 ], "unchanged": true, "expected": [ 0, 100, 110, 101, -1 ] }, { "passed": true, "actual": [ 1230, 205, -200 ], "unchanged": true, "expected": [ 1230, 205, -200 ] }, { "passed": true, "actual": [ null, null, null, null, null, null, null ], "unchanged": true, "expected": [ null, null, null, null, null, null, null ] }, { "passed": true, "actual": [ 0, 0, 99999999 ], "unchanged": true, "expected": [ 0, 0, 99999999 ] } ] }, { "id": "ttl", "title": "Кэш со сроком действия", "category": "code", "repeat": 1, "passed": true, "total_ms": 7626, "first_text_ms": 5712, "answer": "function solve(input) {\n const store = new Map();\n const results = [];\n\n for (const operation of input) {\n if (operation.op === \"set\") {\n store.set(operation.key, {\n value: operation.value,\n expiresAt: operation.t + operation.ttl\n });\n } else if (operation.op === \"get\") {\n const entry = store.get(operation.key);\n\n if (entry === undefined) {\n results.push(null);\n } else if (operation.t >= entry.expiresAt) {\n store.delete(operation.key);\n results.push(null);\n } else {\n results.push(entry.value);\n }\n }\n }\n\n return results;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ 1, null ], "unchanged": true, "expected": [ 1, null ] }, { "passed": true, "actual": [ 2, null ], "unchanged": true, "expected": [ 2, null ] }, { "passed": true, "actual": [ 0, null ], "unchanged": true, "expected": [ 0, null ] } ] }, { "id": "path", "title": "Кратчайший путь на сетке", "category": "code", "repeat": 1, "passed": true, "total_ms": 11220, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "write_file", "args": { "path": "solve.js", "content": "function solve(input) {\n const rows = input.length;\n if (rows === 0) return -1;\n const cols = input[0].length;\n if (cols === 0 || input[0][0] !== 0 || input[rows - 1][cols - 1] !== 0) return -1;\n\n const seen = Array.from({ length: rows }, () => new Uint8Array(cols));\n const queue = [[0, 0, 0]];\n seen[0][0] = 1;\n const directions = [[1, 0], [-1, 0], [0, 1], [0, -1]];\n\n for (let head = 0; head < queue.length; head++) {\n const [r, c, distance] = queue[head];\n if (r === rows - 1 && c === cols - 1) return distance;\n for (const [dr, dc] of directions) {\n const nr = r + dr;\n const nc = c + dc;\n if (nr < 0 || nr >= rows || nc < 0 || nc >= cols) continue;\n if (seen[nr][nc] || input[nr][nc] !== 0) continue;\n seen[nr][nc] = 1;\n queue.push([nr, nc, distance + 1]);\n }\n }\n return -1;\n}\n" } } ], "code_source": "write_file payload (not executed as a tool)", "tests": [ { "passed": true, "actual": 0, "unchanged": true, "expected": 0 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 }, { "passed": true, "actual": 2, "unchanged": true, "expected": 2 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 }, { "passed": true, "actual": 4, "unchanged": true, "expected": 4 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 } ] }, { "id": "revenue", "title": "Выручка с возвратами", "category": "analysis", "repeat": 1, "passed": true, "total_ms": 2796, "first_text_ms": 2228, "answer": "{\"gross\":2500,\"refunds\":1000,\"net\":1500,\"paid_orders\":3}", "tool_calls": [], "actual": { "gross": 2500, "refunds": 1000, "net": 1500, "paid_orders": 3 } }, { "id": "conversion", "title": "Взвешенная конверсия", "category": "analysis", "repeat": 1, "passed": true, "total_ms": 2105, "first_text_ms": 1679, "answer": "{\"visitors\":1000,\"purchases\":65,\"conversion_pct\":6.5}", "tool_calls": [], "actual": { "visitors": 1000, "purchases": 65, "conversion_pct": 6.5 } }, { "id": "schedule", "title": "Поиск свободного времени", "category": "analysis", "repeat": 1, "passed": true, "total_ms": 1785, "first_text_ms": 1424, "answer": "{\"start\":\"10:00\",\"end\":\"10:30\"}", "tool_calls": [], "actual": { "start": "10:00", "end": "10:30" } }, { "id": "missing", "title": "Не выдумывать недостающие данные", "category": "analysis", "repeat": 1, "passed": true, "total_ms": 2335, "first_text_ms": 1795, "answer": "{\"can_compute\":false,\"profit\":null,\"missing\":[\"costs\"]}", "tool_calls": [], "actual": { "can_compute": false, "profit": null, "missing": [ "costs" ] } }, { "id": "card", "title": "Имя и отделение из карточки", "category": "assistant", "repeat": 1, "passed": true, "total_ms": 2549, "first_text_ms": 1828, "answer": "{\"first_name\":\"Анна\",\"last_name\":\"Иванова\",\"city\":\"Одесса\",\"branch_number\":\"165\",\"service\":\"Новая почта\"}", "tool_calls": [], "actual": { "first_name": "Анна", "last_name": "Иванова", "city": "Одесса", "branch_number": "165", "service": "Новая почта" } }, { "id": "memory", "title": "Предпочтение из памяти", "category": "assistant", "repeat": 1, "passed": true, "total_ms": 1477, "first_text_ms": 1185, "answer": "{\"drink\":\"чай\",\"sweetener\":\"без сахара\"}", "tool_calls": [], "actual": { "drink": "чай", "sweetener": "без сахара" } }, { "id": "app", "title": "Поиск установленного приложения", "category": "tool", "repeat": 1, "passed": true, "total_ms": 2611, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "list_installed_apps", "args": { "query": "Audacity" } } ] }, { "id": "search", "title": "Запрос актуальной погоды", "category": "tool", "repeat": 1, "passed": true, "total_ms": 2294, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "search_web", "args": { "query": "Одесса погода сейчас текущая температура сегодня" } } ] }, { "id": "browser", "title": "Поручение во встроенном браузере", "category": "tool", "repeat": 1, "passed": true, "total_ms": 4802, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "open_browser", "args": { "url": "https://merchant.luna.invalid/" } } ] }, { "id": "no_invent", "title": "Не приписывать доступ к камере", "category": "assistant", "repeat": 1, "passed": true, "total_ms": 2162, "first_text_ms": 1798, "answer": "{\"can_see\":false,\"object\":null}", "tool_calls": [], "actual": { "can_see": false, "object": null } }, { "id": "intervals", "title": "Объединение интервалов", "category": "code", "repeat": 2, "passed": true, "total_ms": 6836, "first_text_ms": 4120, "answer": "function solve(input) {\n const sorted = input.map(([start, end]) => [start, end]);\n sorted.sort((a, b) => a[0] - b[0] || a[1] - b[1]);\n\n const result = [];\n\n for (const [start, end] of sorted) {\n const last = result[result.length - 1];\n\n if (last && start <= last[1]) {\n last[1] = Math.max(last[1], end);\n } else {\n result.push([start, end]);\n }\n }\n\n return result;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ [ 1, 7 ] ], "unchanged": true, "expected": [ [ 1, 7 ] ] }, { "passed": true, "actual": [ [ 1, 2 ], [ 4, 5 ] ], "unchanged": true, "expected": [ [ 1, 2 ], [ 4, 5 ] ] }, { "passed": true, "actual": [ [ -5, 0 ] ], "unchanged": true, "expected": [ [ -5, 0 ] ] }, { "passed": true, "actual": [ [ 1, 9 ] ], "unchanged": true, "expected": [ [ 1, 9 ] ] } ] }, { "id": "csv", "title": "CSV с кавычками", "category": "code", "repeat": 2, "passed": true, "total_ms": 9595, "first_text_ms": 6037, "answer": "function solve(input) {\n if (input === \"\") return [];\n\n const rows = [];\n let row = [];\n let cell = \"\";\n let quoted = false;\n\n for (let i = 0; i < input.length; i++) {\n const ch = input[i];\n\n if (quoted) {\n if (ch === '\"') {\n if (input[i + 1] === '\"') {\n cell += '\"';\n i++;\n } else {\n quoted = false;\n }\n } else {\n cell += ch;\n }\n } else if (ch === '\"') {\n quoted = true;\n } else if (ch === \",\") {\n row.push(cell);\n cell = \"\";\n } else if (ch === \"\\n\") {\n row.push(cell);\n rows.push(row);\n row = [];\n cell = \"\";\n } else {\n cell += ch;\n }\n }\n\n if (input[input.length - 1] !== \"\\n\") {\n row.push(cell);\n rows.push(row);\n }\n\n return rows;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ [ "a", "b" ], [ "c", "d" ] ], "unchanged": true, "expected": [ [ "a", "b" ], [ "c", "d" ] ] }, { "passed": true, "actual": [ [ "a,b", "c" ] ], "unchanged": true, "expected": [ [ "a,b", "c" ] ] }, { "passed": true, "actual": [ [ "a\"b", "x" ] ], "unchanged": true, "expected": [ [ "a\"b", "x" ] ] }, { "passed": true, "actual": [ [ "a\nb", "", "c" ], [ "x", "y", "z" ] ], "unchanged": true, "expected": [ [ "a\nb", "", "c" ], [ "x", "y", "z" ] ] }, { "passed": true, "actual": [ [ "", "" ] ], "unchanged": true, "expected": [ [ "", "" ] ] } ] }, { "id": "topology", "title": "Зависимости задач", "category": "code", "repeat": 2, "passed": true, "total_ms": 12907, "first_text_ms": 4778, "answer": "function solve(input) {\n const adjacency = new Map();\n const indegree = new Map();\n\n for (const node of input.nodes) {\n adjacency.set(node, new Set());\n indegree.set(node, 0);\n }\n\n for (const [a, b] of input.edges) {\n const neighbors = adjacency.get(a);\n if (!neighbors.has(b)) {\n neighbors.add(b);\n indegree.set(b, indegree.get(b) + 1);\n }\n }\n\n const heap = [];\n\n function push(value) {\n let i = heap.length;\n heap.push(value);\n while (i > 0) {\n const parent = (i - 1) >> 1;\n if (heap[parent] <= value) break;\n heap[i] = heap[parent];\n i = parent;\n }\n heap[i] = value;\n }\n\n function pop() {\n const minimum = heap[0];\n const last = heap.pop();\n\n if (heap.length > 0) {\n let i = 0;\n while (2 * i + 1 < heap.length) {\n let child = 2 * i + 1;\n if (\n child + 1 < heap.length &&\n heap[child + 1] < heap[child]\n ) {\n child++;\n }\n if (last <= heap[child]) break;\n heap[i] = heap[child];\n i = child;\n }\n heap[i] = last;\n }\n\n return minimum;\n }\n\n for (const node of input.nodes) {\n if (indegree.get(node) === 0) push(node);\n }\n\n const result = [];\n\n while (heap.length > 0) {\n const node = pop();\n result.push(node);\n\n for (const neighbor of adjacency.get(node)) {\n const degree = indegree.get(neighbor) - 1;\n indegree.set(neighbor, degree);\n if (degree === 0) push(neighbor);\n }\n }\n\n return result.length === input.nodes.length ? result : null;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ "a", "b", "c" ], "unchanged": true, "expected": [ "a", "b", "c" ] }, { "passed": true, "actual": null, "unchanged": true, "expected": null }, { "passed": true, "actual": [ "a", "b" ], "unchanged": true, "expected": [ "a", "b" ] }, { "passed": true, "actual": null, "unchanged": true, "expected": null } ] }, { "id": "money", "title": "Деньги без потери копеек", "category": "code", "repeat": 2, "passed": true, "total_ms": 9620, "first_text_ms": 5687, "answer": "function solve(input) {\n return input.map((value) => {\n const match = /^(-?)([0-9]+)(?:[.,]([0-9]{1,2}))?$/.exec(value.trim());\n if (!match) return null;\n\n let rubles = 0;\n for (const digit of match[2]) {\n rubles = rubles * 10 + digit.charCodeAt(0) - 48;\n }\n\n const fraction = match[3] || \"\";\n let kopecks = rubles * 100;\n if (fraction.length >= 1) kopecks += (fraction.charCodeAt(0) - 48) * 10;\n if (fraction.length === 2) kopecks += fraction.charCodeAt(1) - 48;\n\n return match[1] && kopecks !== 0 ? -kopecks : kopecks;\n });\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [ 0, 100, 110, 101, -1 ], "unchanged": true, "expected": [ 0, 100, 110, 101, -1 ] }, { "passed": true, "actual": [ 1230, 205, -200 ], "unchanged": true, "expected": [ 1230, 205, -200 ] }, { "passed": true, "actual": [ null, null, null, null, null, null, null ], "unchanged": true, "expected": [ null, null, null, null, null, null, null ] }, { "passed": true, "actual": [ 0, 0, 99999999 ], "unchanged": true, "expected": [ 0, 0, 99999999 ] } ] }, { "id": "ttl", "title": "Кэш со сроком действия", "category": "code", "repeat": 2, "passed": true, "total_ms": 7541, "first_text_ms": 3492, "answer": "function solve(input) {\n const store = new Map();\n const results = [];\n\n for (const operation of input) {\n if (operation.op === \"set\") {\n store.set(operation.key, {\n value: operation.value,\n expiresAt: operation.t + operation.ttl\n });\n } else if (operation.op === \"get\") {\n const entry = store.get(operation.key);\n\n if (entry === undefined) {\n results.push(null);\n } else if (operation.t >= entry.expiresAt) {\n store.delete(operation.key);\n results.push(null);\n } else {\n results.push(entry.value);\n }\n }\n }\n\n return results;\n}", "tool_calls": [], "code_source": "message", "tests": [ { "passed": true, "actual": [], "unchanged": true, "expected": [] }, { "passed": true, "actual": [ 1, null ], "unchanged": true, "expected": [ 1, null ] }, { "passed": true, "actual": [ 2, null ], "unchanged": true, "expected": [ 2, null ] }, { "passed": true, "actual": [ 0, null ], "unchanged": true, "expected": [ 0, null ] } ] }, { "id": "path", "title": "Кратчайший путь на сетке", "category": "code", "repeat": 2, "passed": true, "total_ms": 12379, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "write_file", "args": { "path": "solve.js", "content": "function solve(input) {\n if (!Array.isArray(input) || input.length === 0 || input[0].length === 0) return -1;\n const rows = input.length;\n const cols = input[0].length;\n if (input[0][0] !== 0 || input[rows - 1][cols - 1] !== 0) return -1;\n const visited = new Uint8Array(rows * cols);\n const queue = [0];\n visited[0] = 1;\n let head = 0;\n let steps = 0;\n const directions = [[1, 0], [-1, 0], [0, 1], [0, -1]];\n while (head < queue.length) {\n const end = queue.length;\n while (head < end) {\n const id = queue[head++];\n if (id === rows * cols - 1) return steps;\n const row = Math.floor(id / cols);\n const col = id % cols;\n for (const [dr, dc] of directions) {\n const r = row + dr;\n const c = col + dc;\n if (r < 0 || r >= rows || c < 0 || c >= cols) continue;\n const next = r * cols + c;\n if (visited[next] || input[r][c] !== 0) continue;\n visited[next] = 1;\n queue.push(next);\n }\n }\n steps++;\n }\n return -1;\n}\n" } } ], "code_source": "write_file payload (not executed as a tool)", "tests": [ { "passed": true, "actual": 0, "unchanged": true, "expected": 0 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 }, { "passed": true, "actual": 2, "unchanged": true, "expected": 2 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 }, { "passed": true, "actual": 4, "unchanged": true, "expected": 4 }, { "passed": true, "actual": -1, "unchanged": true, "expected": -1 } ] }, { "id": "revenue", "title": "Выручка с возвратами", "category": "analysis", "repeat": 2, "passed": true, "total_ms": 2014, "first_text_ms": 1398, "answer": "{\"gross\":2500,\"refunds\":1000,\"net\":1500,\"paid_orders\":3}", "tool_calls": [], "actual": { "gross": 2500, "refunds": 1000, "net": 1500, "paid_orders": 3 } }, { "id": "conversion", "title": "Взвешенная конверсия", "category": "analysis", "repeat": 2, "passed": true, "total_ms": 1684, "first_text_ms": 1319, "answer": "{\"visitors\":1000,\"purchases\":65,\"conversion_pct\":6.5}", "tool_calls": [], "actual": { "visitors": 1000, "purchases": 65, "conversion_pct": 6.5 } }, { "id": "schedule", "title": "Поиск свободного времени", "category": "analysis", "repeat": 2, "passed": true, "total_ms": 2068, "first_text_ms": 1620, "answer": "{\"start\":\"10:00\",\"end\":\"10:30\"}", "tool_calls": [], "actual": { "start": "10:00", "end": "10:30" } }, { "id": "missing", "title": "Не выдумывать недостающие данные", "category": "analysis", "repeat": 2, "passed": true, "total_ms": 2448, "first_text_ms": 2130, "answer": "{\"can_compute\":false,\"profit\":null,\"missing\":[\"costs\"]}", "tool_calls": [], "actual": { "can_compute": false, "profit": null, "missing": [ "costs" ] } }, { "id": "card", "title": "Имя и отделение из карточки", "category": "assistant", "repeat": 2, "passed": true, "total_ms": 2065, "first_text_ms": 1396, "answer": "{\"first_name\":\"Анна\",\"last_name\":\"Иванова\",\"city\":\"Одесса\",\"branch_number\":\"165\",\"service\":\"Новая почта\"}", "tool_calls": [], "actual": { "first_name": "Анна", "last_name": "Иванова", "city": "Одесса", "branch_number": "165", "service": "Новая почта" } }, { "id": "memory", "title": "Предпочтение из памяти", "category": "assistant", "repeat": 2, "passed": true, "total_ms": 1486, "first_text_ms": 1202, "answer": "{\"drink\":\"чай\",\"sweetener\":\"без сахара\"}", "tool_calls": [], "actual": { "drink": "чай", "sweetener": "без сахара" } }, { "id": "app", "title": "Поиск установленного приложения", "category": "tool", "repeat": 2, "passed": true, "total_ms": 3352, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "list_installed_apps", "args": { "query": "Audacity" } } ] }, { "id": "search", "title": "Запрос актуальной погоды", "category": "tool", "repeat": 2, "passed": true, "total_ms": 2678, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "search_web", "args": { "query": "Одесса погода сейчас текущая температура" } } ] }, { "id": "browser", "title": "Поручение во встроенном браузере", "category": "tool", "repeat": 2, "passed": true, "total_ms": 2650, "first_text_ms": null, "answer": "", "tool_calls": [ { "name": "open_browser", "args": { "url": "https://merchant.luna.invalid/" } } ] }, { "id": "no_invent", "title": "Не приписывать доступ к камере", "category": "assistant", "repeat": 2, "passed": true, "total_ms": 1734, "first_text_ms": 1484, "answer": "{\"can_see\":false,\"object\":null}", "tool_calls": [], "actual": { "can_see": false, "object": null } } ] }