{ "schemaVersion": 1, "results": [ { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-kv-cpu1-paddle-vl", "suite": "ocr", "cohort": "ocr-9f24313acc9ad33554e2f513", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T19:41:41Z", "model": { "name": "PaddlePaddle/PaddleOCR-VL-1.6", "revision": "c5630abae1d940eafe0697512a0325494b02ab42", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Transformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "Greedy generation max_new_tokens=512; input TPS includes processor image placeholder tokens, excludes output tokens. Generation-limit count is in evidence.", "CPU generation explicitly enables the KV cache; the pinned Paddle generation config disables it. Separate cached-inference cohort." ], "backendCohort": "transformers-paddle-cpu-kv-cache-v1", "maxNewTokens": 512, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": 27.074259209407366, "accuracy": { "name": "Character accuracy", "value": 99.94252873563218 }, "ramPeakMb": 7443.578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 46538.66945183836, "pagesPerSecond": 0.021487507309053466, "cerPercent": 0.05747126436781609, "werPercent": 0.40650406504065045 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-kv-cpu1-paddle-vl.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-gpu24-r2-paddle-vl", "suite": "ocr", "cohort": "ocr-a61cd796bc2b589813ef2725", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-03T19:37:01Z", "model": { "name": "PaddlePaddle/PaddleOCR-VL-1.6", "revision": "c5630abae1d940eafe0697512a0325494b02ab42", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "Transformers / torch", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "Greedy generation max_new_tokens=512; input TPS includes processor image placeholder tokens, excludes output tokens. Generation-limit count is in evidence." ], "backendCohort": "transformers", "maxNewTokens": 512, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": 4.427333985718773, "accuracy": { "name": "Character accuracy", "value": 99.94252873563218 }, "ramPeakMb": 5414.296875, "vramPeakMb": 6677.7412109375, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 284595.6514833476, "pagesPerSecond": 0.0035137571315228358, "cerPercent": 0.05747126436781609, "werPercent": 0.40650406504065045 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-gpu24-r2-paddle-vl.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-gpu24-r2-glm", "suite": "ocr", "cohort": "ocr-a61cd796bc2b589813ef2725", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-03T18:39:22Z", "model": { "name": "zai-org/GLM-OCR", "revision": "2e85a62840ccac27daa451df36c736c4636b8628", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "Transformers / torch", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "Greedy generation max_new_tokens=512; input TPS includes processor image placeholder tokens, excludes output tokens. Generation-limit count is in evidence." ], "backendCohort": "transformers", "maxNewTokens": 512, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": 240.3688766836189, "accuracy": { "name": "Character accuracy", "value": 99.54022988505747 }, "ramPeakMb": 6284.734375, "vramPeakMb": 7477.15869140625, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5237.782933341805, "pagesPerSecond": 0.19092047393456624, "cerPercent": 0.45977011494252873, "werPercent": 1.6260162601626018 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-gpu24-r2-glm.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-igpu1-glm", "suite": "ocr", "cohort": "ocr-b03be2076d25459f87b723b7", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-03T18:35:37Z", "model": { "name": "ggml-org/GLM-OCR-GGUF", "revision": "65a42de1148dbed2297e922b5dbc7d9b70c36578", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GLM-OCR GGUF", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Warm page HTTP including PNG base64 serialization, multimodal preprocessing, inference and text decoding/transfer; excludes fixture rendering, loading, scoring and warmups. HTTP completion is synchronous.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Official llama.cpp GLM-OCR prompt is OCR before image.", "Actual prompt/completion usage only; absent usage remains null. Prompt usage includes server vision tokens, not a text-only estimate.", "Native GGUF backend cohort, not Transformers latency peers." ], "backendCohort": "llama.cpp / Vulkan / GLM-OCR GGUF", "maxNewTokens": 512, "prompt": "OCR", "temperature": 0, "seed": 0 }, "metrics": { "tokensPerSecond": 38.56705616018906, "accuracy": { "name": "Character accuracy", "value": 0 }, "ramPeakMb": 1367.70703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 32566.654680180363, "pagesPerSecond": 0.030706254904609126, "cerPercent": 515.5172413793103, "werPercent": 1248.780487804878 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-igpu1-glm.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-cpu1-glm", "suite": "ocr", "cohort": "ocr-0ab1b6b4e8da907e436615ec", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T18:28:15Z", "model": { "name": "zai-org/GLM-OCR", "revision": "2e85a62840ccac27daa451df36c736c4636b8628", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Transformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "Greedy generation max_new_tokens=512; input TPS includes processor image placeholder tokens, excludes output tokens. Generation-limit count is in evidence." ], "backendCohort": "transformers", "maxNewTokens": 512, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": 26.01927031640375, "accuracy": { "name": "Character accuracy", "value": 99.54022988505747 }, "ramPeakMb": 8252.296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 48387.21396450031, "pagesPerSecond": 0.02066661661350576, "cerPercent": 0.45977011494252873, "werPercent": 1.6260162601626018 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-cpu1-glm.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-gpu-o-medium", "suite": "ocr", "cohort": "ocr-81845926ea7cbee3f4379aee", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-11-03T18:23:46Z", "model": { "name": "PP-OCRv6 medium", "revision": "314c1d7063b5eb5b9e15e0efd4668bc14ec8d8297454ab751c3e93c05a2de155", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "RapidOCR / onnxruntime", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "RapidOCR 3.9.2 stock detector resizing heuristic, recognition batch 6, classification disabled. CPU stock pipeline loads the pinned unused classifier; CUDA composes upstream detector/crop/recognizer with profiled ORT sessions. No tokenizer: TPS null." ], "backendCohort": "onnxruntime", "maxNewTokens": null, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Character accuracy", "value": 100 }, "ramPeakMb": 1485.59375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 233.2672500051558, "pagesPerSecond": 4.286928404985686, "cerPercent": 0, "werPercent": 0 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-gpu-o-medium.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-gpu-o-small", "suite": "ocr", "cohort": "ocr-81845926ea7cbee3f4379aee", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-11-03T18:23:34Z", "model": { "name": "PP-OCRv6 small", "revision": "8c44847c942d6822f1cb043287680e27a70cbc0fbac9f6b90c6541132759ee9a", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "RapidOCR / onnxruntime", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "RapidOCR 3.9.2 stock detector resizing heuristic, recognition batch 6, classification disabled. CPU stock pipeline loads the pinned unused classifier; CUDA composes upstream detector/crop/recognizer with profiled ORT sessions. No tokenizer: TPS null." ], "backendCohort": "onnxruntime", "maxNewTokens": null, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Character accuracy", "value": 100 }, "ramPeakMb": 1376.59375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 141.4763166588576, "pagesPerSecond": 7.068320858333512, "cerPercent": 0, "werPercent": 0 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-gpu-o-small.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-cpu2-tesseract", "suite": "ocr", "cohort": "ocr-655995d98c003509e797b0f8", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T18:17:05Z", "model": { "name": "Tesseract 5.5.3 / tessdata_best", "revision": "8ff4a80dd662e249ca77ad6c721d36b9029e09c36a6a98a43dfa68c4fe27363b", "precision": "NATIVE" }, "engine": { "device": "cpu", "backend": "Tesseract / tesserocr", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "Tesseract native LSTM precision is not FP32; one language per page, OEM 1, PSM 3. In-process API including per-page language initialization" ], "backendCohort": "tesserocr", "maxNewTokens": null, "tesseractPsm": 3, "tesseractOem": 1 }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Character accuracy", "value": 99.94252873563218 }, "ramPeakMb": 124.55859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 341.0540051603069, "pagesPerSecond": 2.93208695652164, "cerPercent": 0.05747126436781609, "werPercent": 0.40650406504065045 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-cpu2-tesseract.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-cpu2-medium", "suite": "ocr", "cohort": "ocr-b038668dc2a3900b8e26b1b6", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T18:17:00Z", "model": { "name": "PP-OCRv6 medium", "revision": "314c1d7063b5eb5b9e15e0efd4668bc14ec8d8297454ab751c3e93c05a2de155", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "RapidOCR / onnxruntime", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "RapidOCR 3.9.2 stock detector resizing heuristic, recognition batch 6, classification disabled. CPU stock pipeline loads the pinned unused classifier; CUDA composes upstream detector/crop/recognizer with profiled ORT sessions. No tokenizer: TPS null." ], "backendCohort": "onnxruntime", "maxNewTokens": null, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Character accuracy", "value": 100 }, "ramPeakMb": 715.23046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2595.136348992431, "pagesPerSecond": 0.38533620801398466, "cerPercent": 0, "werPercent": 0 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-cpu2-medium.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ocr-01a1930a-cpu1-small", "suite": "ocr", "cohort": "ocr-0291c843c17927f59940d128", "cohortLabel": "Synthetic EN-DE-PL print OCR v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T18:16:38Z", "model": { "name": "PP-OCRv6 small", "revision": "8c44847c942d6822f1cb043287680e27a70cbc0fbac9f6b90c6541132759ee9a", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "RapidOCR / onnxruntime", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL print OCR v1", "revision": "e45eeb866be703a728530417daa022750b2ceaebbbd51dd89558de7e530c2608", "samples": 6, "languages": [ "de", "en", "pl" ], "public": true, "license": "CC0-1.0", "normalization": "NFC, whitespace collapse, case and punctuation preserved; no spelling modernization" }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Warm page recognition including PNG decode, preprocessing, model inference and text decoding; excludes fixture rendering, load, scoring and warmups; CUDA synchronized", "notes": [ "Six synthetic clean/small print pages, two per language; no scans, handwriting, tables or layout scoring. Smoke cohort only.", "NFC and whitespace collapse; case and punctuation preserved. jiwer micro CER/WER over all pages and repeats; spaces count in CER.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups where supported; non-torch GPU VRAM is unknown, not zero.", "RapidOCR 3.9.2 stock detector resizing heuristic, recognition batch 6, classification disabled. CPU stock pipeline loads the pinned unused classifier; CUDA composes upstream detector/crop/recognizer with profiled ORT sessions. No tokenizer: TPS null." ], "backendCohort": "onnxruntime", "maxNewTokens": null, "tesseractPsm": null, "tesseractOem": null }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Character accuracy", "value": 100 }, "ramPeakMb": 443.08203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 819.1717269752795, "pagesPerSecond": 1.2207452565439645, "cerPercent": 0, "werPercent": 0 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "ocr/evidence/r20261103-ocr-01a1930a-cpu1-small.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ojsh-01a1930a-cpu2-d-open-jev-2b-scifact-s2", "suite": "decision", "cohort": "decision-1486e060dd71d47e1076b790", "cohortLabel": "SciFact claims dev (all 300, v1) / shard 2/2 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T14:51:45Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "zefan PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1) / shard 2/2", "revision": "9ff796b5a35271b786eef31ce0471d4d6744fc149d9cd1563c3861555203a29c", "samples": 150, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cpu" }, "metrics": { "tokensPerSecond": 16.845852400363746, "accuracy": { "name": "Decision accuracy", "value": 76 }, "ramPeakMb": 3955.43359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 88926.34011014209, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.351696034551906 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-ojsh-01a1930a-cpu2-d-open-jev-2b-scifact-s2.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-ojsh-01a1930a-cpu1-d-open-jev-2b-scifact-s1", "suite": "decision", "cohort": "decision-a237ae523f89cd23bb4a21ae", "cohortLabel": "SciFact claims dev (all 300, v1) / shard 1/2 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T14:26:10Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "zefan PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1) / shard 1/2", "revision": "1826a32370a814482c02f6c23dec5c4ea6b891b357ac179bb6d1dff3159eb773", "samples": 150, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cpu" }, "metrics": { "tokensPerSecond": 18.15650891951067, "accuracy": { "name": "Decision accuracy", "value": 70.66666666666667 }, "ramPeakMb": 3878.47265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 80916.43644231993, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4232822719540545 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-ojsh-01a1930a-cpu1-d-open-jev-2b-scifact-s1.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-scifact", "suite": "decision", "cohort": "decision-498a28cce3d07e57aff9d694", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T10:20:06Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 88.48000011132345, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 11141.7265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5698.462922305916, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3453141796 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-xnli-de", "suite": "decision", "cohort": "decision-a6232233666e5a35de9355c4", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T09:50:25Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.02941017470434, "accuracy": { "name": "Decision accuracy", "value": 71 }, "ramPeakMb": 11243.19921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1496.0952326125698, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4056900391666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-anli-r3", "suite": "decision", "cohort": "decision-e2918a1fd0692d61e056b64d", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T09:42:30Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 82.1100106201955, "accuracy": { "name": "Decision accuracy", "value": 47 }, "ramPeakMb": 11105.0546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1880.7288599800086, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6832482440000001 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-malkuth-2b-scifact", "suite": "decision", "cohort": "decision-4dce3075f2f31048368c86f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T09:39:16Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 75.37905971539975, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 11301.29296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6516.398610629646, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2885460463333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-malkuth-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-boolq", "suite": "decision", "cohort": "decision-839560fc04d2c934bc8a965d", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T09:32:40Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 73.77147644124746, "accuracy": { "name": "Decision accuracy", "value": 89 }, "ramPeakMb": 11105.8671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2307.0343924711538, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11028465659999999 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261103-bal-01a1930a-cpu1-d-mapika-2b-v12-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-malkuth-2b-xnli-de", "suite": "decision", "cohort": "decision-687a2e91c91ad53438d451b9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T09:05:26Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 60.51197889330704, "accuracy": { "name": "Decision accuracy", "value": 72.66666666666667 }, "ramPeakMb": 11409.984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1739.2148671735777, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.37016998879999996 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-malkuth-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-malkuth-2b-anli-r3", "suite": "decision", "cohort": "decision-f4b9c93e3a04e6563e5a7c85", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:56:16Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 63.89013439372204, "accuracy": { "name": "Decision accuracy", "value": 45.666666666666664 }, "ramPeakMb": 11219.96875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2262.0080763862156, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7950413072666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-malkuth-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-malkuth-2b-boolq", "suite": "decision", "cohort": "decision-5ae8ea48d1f9a9183341fa25", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:44:29Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 70.05398003245108, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 11170.41796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2258.163394286142, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07455011256666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-malkuth-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-08b-v1-scifact", "suite": "decision", "cohort": "decision-4dce3075f2f31048368c86f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:29:15Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 157.87470033266408, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 6511.38671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3111.328154320945, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4187147084333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-08b-v1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-08b-v1-xnli-de", "suite": "decision", "cohort": "decision-687a2e91c91ad53438d451b9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:13:02Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 120.37107669269449, "accuracy": { "name": "Decision accuracy", "value": 58.333333333333336 }, "ramPeakMb": 5425.421875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 874.3241003153768, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5373063677333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-08b-v1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-08b-v1-anli-r3", "suite": "decision", "cohort": "decision-f4b9c93e3a04e6563e5a7c85", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:08:24Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 137.73297515863518, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 5236.890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1049.2766879793878, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8298923993999999 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-08b-v1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-08b-v1-boolq", "suite": "decision", "cohort": "decision-5ae8ea48d1f9a9183341fa25", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T08:02:54Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 145.75048936562405, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 5328.078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1085.370855507014, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.13742714756666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-08b-v1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-scifact", "suite": "decision", "cohort": "decision-982e6318be2f85f3a1299ca0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T07:51:54Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 86.75178501468129, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": 11271.32421875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5835.038436550142, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.31952417356639806 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-xnli-de", "suite": "decision", "cohort": "decision-b6f1c71b262bf5fc751fedb6", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T07:21:27Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 68.98685927815806, "accuracy": { "name": "Decision accuracy", "value": 69 }, "ramPeakMb": 11321.58203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1742.9889487867088, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4127984267902147 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-anli-r3", "suite": "decision", "cohort": "decision-eb8ed59772065b508e0d7ca2", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T07:12:19Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.1776481560513, "accuracy": { "name": "Decision accuracy", "value": 46.666666666666664 }, "ramPeakMb": 11138.265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2065.7103510631714, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7503270239378498 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-boolq", "suite": "decision", "cohort": "decision-c66276ea3286989cd1a41b2f", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T07:01:34Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 82.12043945094544, "accuracy": { "name": "Decision accuracy", "value": 90.66666666666667 }, "ramPeakMb": 11163.3203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2267.3202259780373, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07518003965761152 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-2b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-scifact", "suite": "decision", "cohort": "decision-982e6318be2f85f3a1299ca0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:48:10Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 184.9820439528645, "accuracy": { "name": "Decision accuracy", "value": 72.33333333333333 }, "ramPeakMb": 4833.1328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2963.5308827038775, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.39118776354021795 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-xnli-de", "suite": "decision", "cohort": "decision-b6f1c71b262bf5fc751fedb6", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:32:48Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 181.49319075511613, "accuracy": { "name": "Decision accuracy", "value": 71.66666666666667 }, "ramPeakMb": 5014.8046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 932.5051404363088, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3853451190393599 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-anli-r3", "suite": "decision", "cohort": "decision-eb8ed59772065b508e0d7ca2", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:27:55Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 197.8188120732816, "accuracy": { "name": "Decision accuracy", "value": 34 }, "ramPeakMb": 4824.21875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1053.624094100087, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.905443198743802 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-boolq", "suite": "decision", "cohort": "decision-c66276ea3286989cd1a41b2f", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:22:25Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "jev-style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 196.84507934006734, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 4954.8671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1062.7308238268597, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1244416410152801 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-jev-style-08b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-scifact", "suite": "decision", "cohort": "decision-4dce3075f2f31048368c86f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:16:11Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 264.1090850651967, "accuracy": { "name": "Decision accuracy", "value": 49 }, "ramPeakMb": 4536.3671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1863.9898985115967, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7088671039 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-xnli-de", "suite": "decision", "cohort": "decision-687a2e91c91ad53438d451b9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:06:27Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 261.67177942456465, "accuracy": { "name": "Decision accuracy", "value": 54.666666666666664 }, "ramPeakMb": 3621.875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 431.7240485329724, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6042865203333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-anli-r3", "suite": "decision", "cohort": "decision-f4b9c93e3a04e6563e5a7c85", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:04:07Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 284.13800859032347, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 3430.65625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 509.82267637717695, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8733028824666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-omni-600m-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:03:27Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 271.3485276255117, "accuracy": { "name": "Decision accuracy", "value": 64.66666666666667 }, "ramPeakMb": 1017.17578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1775.8219323441153, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4923046385949579 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-omni-600m-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-boolq", "suite": "decision", "cohort": "decision-5ae8ea48d1f9a9183341fa25", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T06:01:25Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 286.8354655237873, "accuracy": { "name": "Decision accuracy", "value": 78 }, "ramPeakMb": 3576.1953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 552.9883355847172, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.16777726693333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-kev-05b-prototype-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-clef-flash-scifact", "suite": "decision", "cohort": "decision-7f5dc5b310209fde6751c4b7", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:57:52Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 22.00050919423169, "accuracy": { "name": "Decision accuracy", "value": 86 }, "ramPeakMb": 9788.79296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27508.454220626736, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2223544085546882 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-clef-flash-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-omni-600m-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:54:13Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 282.9636530573584, "accuracy": { "name": "Decision accuracy", "value": 54.666666666666664 }, "ramPeakMb": 796.03125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 390.3210376078884, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6375925688932222 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-omni-600m-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-omni-600m-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:52:09Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 275.0700004511962, "accuracy": { "name": "Decision accuracy", "value": 35.333333333333336 }, "ramPeakMb": 616.17578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 535.6454711830398, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0331672272391865 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-omni-600m-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-omni-600m-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:49:23Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 258.95658001260665, "accuracy": { "name": "Decision accuracy", "value": 76.33333333333333 }, "ramPeakMb": 641.17578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 675.4800360411173, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.18405751607680199 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-omni-600m-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-laya-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:45:38Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 183.07060711449037, "accuracy": { "name": "Decision accuracy", "value": 53.666666666666664 }, "ramPeakMb": 905.20703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2518.0448531080037, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6196434088331283 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-laya-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-laya-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:32:32Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 198.20825294291487, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 781.88671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 600.6140758475522, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.672411245098413 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-laya-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-laya-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:29:25Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 211.6981845105475, "accuracy": { "name": "Decision accuracy", "value": 36.333333333333336 }, "ramPeakMb": 596.19140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 670.986702106505, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7945735184813871 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-laya-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-laya-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:25:56Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 207.8293166338378, "accuracy": { "name": "Decision accuracy", "value": 72.66666666666667 }, "ramPeakMb": 643.9921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 821.9565431552473, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19527640042563918 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-laya-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-scifact", "suite": "decision", "cohort": "decision-feccb634e8c60ac1bf84fc4f", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:21:25Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 67 }, "ramPeakMb": 3436.78125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1556.656252189229, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.45377041479125924 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-xnli-de", "suite": "decision", "cohort": "decision-37877df59e90aa7929e0ec71", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:12:58Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 3611.2734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 456.5616366455409, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7736378116513077 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-anli-r3", "suite": "decision", "cohort": "decision-2d06a19195baa03d52512041", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:10:20Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 3428.0546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 504.18460525242455, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8565027013940122 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-boolq", "suite": "decision", "cohort": "decision-14b07dadcc5fc43f34da5f19", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:07:28Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 76.66666666666667 }, "ramPeakMb": 3518.94921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 536.3697747098437, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1916598466458067 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decisiontune-1-0-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T05:02:49Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 18.413505332076483, "accuracy": { "name": "Decision accuracy", "value": 85.33333333333333 }, "ramPeakMb": 5775.91796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 29651.243665280057, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.22612249433108603 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-clef-flash-xnli-de", "suite": "decision", "cohort": "decision-aab06f52e9a337c27af79895", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T03:35:35Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 21.41255736549551, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": 9174.1171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 10239.007400705208, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2920167119667471 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-clef-flash-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-clef-flash-anli-r3", "suite": "decision", "cohort": "decision-7f15df28dca9554e29dfecf2", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T02:42:49Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 21.410496985990427, "accuracy": { "name": "Decision accuracy", "value": 62.666666666666664 }, "ramPeakMb": 9006.8984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 12074.451152122152, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5580002597133525 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-clef-flash-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T02:30:32Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 25.663565170344892, "accuracy": { "name": "Decision accuracy", "value": 73.66666666666667 }, "ramPeakMb": 5302.89453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6117.6223551908815, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.35785697693770857 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T01:58:56Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 20.48714525961128, "accuracy": { "name": "Decision accuracy", "value": 60 }, "ramPeakMb": 5138.03515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 9584.221268759575, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5812950472210842 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-clef-flash-boolq", "suite": "decision", "cohort": "decision-37e68a5b93b2990f89699cca", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T01:40:48Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 21.523697937427475, "accuracy": { "name": "Decision accuracy", "value": 92 }, "ramPeakMb": 9035.97265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 13250.201436687688, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.06566419943664363 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-clef-flash-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T01:09:16Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 29.558427937251512, "accuracy": { "name": "Decision accuracy", "value": 91.33333333333333 }, "ramPeakMb": 5178.75390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 7440.179175525166, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07088764752990828 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-h2o-lightning-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-clef-german-2b-scifact", "suite": "decision", "cohort": "decision-7b9a92431f5ed927f8d0cc82", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-03T00:28:34Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 346.0861645376935, "accuracy": { "name": "Decision accuracy", "value": 79.33333333333333 }, "ramPeakMb": 610.4140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1748.6974690491722, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.29834323754082176 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-clef-german-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-3b-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-03T00:27:58Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 50.51706141540568, "accuracy": { "name": "Decision accuracy", "value": 83.33333333333333 }, "ramPeakMb": 3644.4453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 9436.613822011082, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.260923228238569 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-3b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-clef-german-2b-xnli-de", "suite": "decision", "cohort": "decision-f0a8ac0aca3a20bef9af08e5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-03T00:19:27Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 330.616908106696, "accuracy": { "name": "Decision accuracy", "value": 62.333333333333336 }, "ramPeakMb": 687.59375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 663.1340622863112, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5035090136859346 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-clef-german-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-clef-german-2b-anli-r3", "suite": "decision", "cohort": "decision-f64412c86afc224b8f2ed7de", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-03T00:15:58Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 338.96607159696543, "accuracy": { "name": "Decision accuracy", "value": 49.666666666666664 }, "ramPeakMb": 526.43359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 762.6722013269318, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6860513593103339 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-clef-german-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-clef-german-2b-boolq", "suite": "decision", "cohort": "decision-722ef262c6a65e6cfe85e365", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-03T00:11:59Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 344.92206575633463, "accuracy": { "name": "Decision accuracy", "value": 88.33333333333333 }, "ramPeakMb": 575.03515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 826.834121812329, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.09495435788857548 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-clef-german-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-3b-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T23:38:49Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 48.238637621935105, "accuracy": { "name": "Decision accuracy", "value": 73.33333333333333 }, "ramPeakMb": 3257.03515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2310.3886323133097, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3371580556011027 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-3b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-3b-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T23:26:53Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 50.00599235790938, "accuracy": { "name": "Decision accuracy", "value": 56.333333333333336 }, "ramPeakMb": 3084.59765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2989.3084065491953, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5743876251326651 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-3b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-d1-3b-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T23:11:31Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 43.53227500702197, "accuracy": { "name": "Decision accuracy", "value": 89 }, "ramPeakMb": 3126.12109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3814.6409755339846, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07607964193949958 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-d1-3b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-scifact", "suite": "decision", "cohort": "decision-be97072dd0114a1a78fdfd37", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T22:50:28Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "openjev-flash authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 21.97168786031192, "accuracy": { "name": "Decision accuracy", "value": 86.66666666666667 }, "ramPeakMb": 9725.0390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 23812.462807878816, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.20492452273333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-open-jev-2b-xnli-de", "suite": "decision", "cohort": "decision-efa8b5defee2ea9914179683", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T21:59:48Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "zefan PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cpu" }, "metrics": { "tokensPerSecond": 15.030078741626165, "accuracy": { "name": "Decision accuracy", "value": 63.666666666666664 }, "ramPeakMb": 3711.37890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 20540.810551109415, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4626109740907701 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-open-jev-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-xnli-de", "suite": "decision", "cohort": "decision-c65b7dbf6a39edcc048840d9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T20:47:31Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "openjev-flash authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 23.16584397304735, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 9826.0546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5794.882046582061, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.31826246539999997 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-anli-r3", "suite": "decision", "cohort": "decision-8d705f959a666994be5c7992", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T20:17:35Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "openjev-flash authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 23.333688244664394, "accuracy": { "name": "Decision accuracy", "value": 67.66666666666667 }, "ramPeakMb": 9634.1171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 7432.4583772701835, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5156941204333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-open-jev-2b-anli-r3", "suite": "decision", "cohort": "decision-1169477c5ca12c4560ad3664", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T20:14:05Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "zefan PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cpu" }, "metrics": { "tokensPerSecond": 15.753432892238584, "accuracy": { "name": "Decision accuracy", "value": 48.666666666666664 }, "ramPeakMb": 3562.3671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27059.498898809543, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6569166488223114 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-open-jev-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-boolq", "suite": "decision", "cohort": "decision-629d425b3b6b98921e3d9966", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T19:39:14Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "openjev-flash authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 23.412257423025622, "accuracy": { "name": "Decision accuracy", "value": 89.33333333333333 }, "ramPeakMb": 10215.8671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 8593.504235753984, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.0817543337 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-openjev-flash-9b-q4-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-lev-4b-scifact", "suite": "decision", "cohort": "decision-8a4b6bbcc76c1a401875baf9", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T18:52:22Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 133.83122461846625, "accuracy": { "name": "Decision accuracy", "value": 48.333333333333336 }, "ramPeakMb": 512.90234375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 8491.291948046613, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6377836185875767 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-lev-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-lev-4b-xnli-de", "suite": "decision", "cohort": "decision-a16c72d7093e88ed785631de", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T18:08:23Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 128.23590668473554, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 690.09765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2795.5248723588497, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4726188416736957 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-lev-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-open-jev-2b-boolq", "suite": "decision", "cohort": "decision-c5b7c7857950a5ea26622087", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T17:55:20Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "zefan PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cpu" }, "metrics": { "tokensPerSecond": 15.961046957172423, "accuracy": { "name": "Decision accuracy", "value": 81 }, "ramPeakMb": 3653.0078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 11352.221055393262, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11480729668919273 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-open-jev-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-lev-4b-anli-r3", "suite": "decision", "cohort": "decision-4afa325bfdecf1ddbb01b2ba", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T17:53:56Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 124.27502358066519, "accuracy": { "name": "Decision accuracy", "value": 59.666666666666664 }, "ramPeakMb": 500.29296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3515.214246165706, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6043838642395672 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-lev-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-lev-4b-boolq", "suite": "decision", "cohort": "decision-4130e8eb3d4a355fa3e052db", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T17:35:47Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 130.92497258629413, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 653.0390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1826.9496537467542, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.10982897870820459 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-lev-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-scifact", "suite": "decision", "cohort": "decision-3a7ed22caf0a7dd01a3f6497", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T17:24:47Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 78.93125549539616, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 10816.01953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6704.5683826841805, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.29373389796666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-scifact", "suite": "decision", "cohort": "decision-c39cdaf696cf0fa1a0585094", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T16:54:48Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 112.83685358907331, "accuracy": { "name": "Decision accuracy", "value": 50.333333333333336 }, "ramPeakMb": 512.00390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 4838.69689704113, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.9102981352898517 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-xnli-de", "suite": "decision", "cohort": "decision-03806d30438ab3c643651367", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T16:49:54Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 79.38293138902738, "accuracy": { "name": "Decision accuracy", "value": 67.33333333333333 }, "ramPeakMb": 10887.63671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1766.6686135090033, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4220233688 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-anli-r3", "suite": "decision", "cohort": "decision-2345c5d73c81d8b4ab0ac81c", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T16:40:29Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 68.6070750883605, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 10832.73046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2615.2793488948373, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6774475157333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-xnli-de", "suite": "decision", "cohort": "decision-f4248a6d7799a1b3299ce673", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T16:29:40Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 101.22485747218327, "accuracy": { "name": "Decision accuracy", "value": 73 }, "ramPeakMb": 689.15234375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1551.0024308322074, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3592015402352225 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-boolq", "suite": "decision", "cohort": "decision-6baa03abf69d37641b8bf953", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T16:26:44Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 66.88183881081183, "accuracy": { "name": "Decision accuracy", "value": 91.66666666666667 }, "ramPeakMb": 10624.14453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3262.3704313892326, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.073823739 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-strands-hobson-2b-v21-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-anli-r3", "suite": "decision", "cohort": "decision-10be9dec41e4e80ce1053cf4", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T16:21:35Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 106.55062575666336, "accuracy": { "name": "Decision accuracy", "value": 57.666666666666664 }, "ramPeakMb": 499.6484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1842.817270559802, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5921201992833629 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-boolq", "suite": "decision", "cohort": "decision-a6aa94d3a03388c02ef8aac6", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T16:12:02Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 108.14055066778401, "accuracy": { "name": "Decision accuracy", "value": 91 }, "ramPeakMb": 504.9453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2033.6497145794176, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07184718123205808 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-h2o-lightning-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-d1-3b-scifact", "suite": "decision", "cohort": "decision-9b59c552591b80bfd63de757", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T16:07:38Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 227.46422482153187, "accuracy": { "name": "Decision accuracy", "value": 83.33333333333333 }, "ramPeakMb": 211.86328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2095.7581367972307, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2618458034646269 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-d1-3b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-scifact", "suite": "decision", "cohort": "decision-21dcf3fa98c5df0c3fc42901", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T15:59:44Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 72.60858978670333, "accuracy": { "name": "Decision accuracy", "value": 84 }, "ramPeakMb": 10837.93359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 7288.393860211169, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2488061851 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-d1-3b-xnli-de", "suite": "decision", "cohort": "decision-493e706481bf0ffd47458b7c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T15:56:42Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 192.19553136362592, "accuracy": { "name": "Decision accuracy", "value": 73.33333333333333 }, "ramPeakMb": 387.1015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 579.8782063727655, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.33627332390575543 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-d1-3b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-d1-3b-anli-r3", "suite": "decision", "cohort": "decision-d1daa55fe20ae9d2a8848c31", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T15:53:37Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 199.53506604476738, "accuracy": { "name": "Decision accuracy", "value": 56 }, "ramPeakMb": 195.6015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 749.1582121199463, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5749838254045747 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-d1-3b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-d1-3b-boolq", "suite": "decision", "cohort": "decision-36c05b72ba7b7ffdc2fd246a", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T15:49:41Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 209.11542989363616, "accuracy": { "name": "Decision accuracy", "value": 89 }, "ramPeakMb": 200.4140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 794.1068723836603, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.0759964236609799 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-d1-3b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-scifact", "suite": "decision", "cohort": "decision-12221f2330a8d2fa39eb6947", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T15:43:55Z", "model": { "name": "vllm-sr/Decision-2.0-Sol-2B", "revision": "df30237f9c1ced5765fdbb499dd1f2c7dca478ad", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 82.65194983499552, "accuracy": { "name": "Decision accuracy", "value": 82.33333333333333 }, "ramPeakMb": 9461.25390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6681.028107653838, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2834454589921576 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-xnli-de", "suite": "decision", "cohort": "decision-701ea28f7873553b816cad9a", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T15:21:42Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 69.7718594178618, "accuracy": { "name": "Decision accuracy", "value": 74.33333333333333 }, "ramPeakMb": 10739.56640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2010.0271728953037, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.36254789696666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-anli-r3", "suite": "decision", "cohort": "decision-04c25254e59e41d0e2cfb53a", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T15:11:06Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 72.22425844439366, "accuracy": { "name": "Decision accuracy", "value": 48.666666666666664 }, "ramPeakMb": 10799.86328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2484.299188821848, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6909279648666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-xnli-de", "suite": "decision", "cohort": "decision-9c8375440cdaae00395c92a5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T15:09:10Z", "model": { "name": "vllm-sr/Decision-2.0-Sol-2B", "revision": "df30237f9c1ced5765fdbb499dd1f2c7dca478ad", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 84.72180567290253, "accuracy": { "name": "Decision accuracy", "value": 69.33333333333333 }, "ramPeakMb": 8977.3203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1962.2260410167898, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.42004096236227645 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-anli-r3", "suite": "decision", "cohort": "decision-e8c5efe4382a9d41410894e2", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:58:52Z", "model": { "name": "vllm-sr/Decision-2.0-Sol-2B", "revision": "df30237f9c1ced5765fdbb499dd1f2c7dca478ad", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 85.01826479297915, "accuracy": { "name": "Decision accuracy", "value": 42.666666666666664 }, "ramPeakMb": 8852.3515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2416.265106878903, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7656230080554629 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-boolq", "suite": "decision", "cohort": "decision-5eb56b1f4ca1d0fb3ef8aeea", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:58:06Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 74.09988660460222, "accuracy": { "name": "Decision accuracy", "value": 90.33333333333333 }, "ramPeakMb": 10556.578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2944.5839033143907, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07580988073333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-strands-qwen35-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-boolq", "suite": "decision", "cohort": "decision-e360242eecb1883bcfaf4b96", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:46:17Z", "model": { "name": "vllm-sr/Decision-2.0-Sol-2B", "revision": "df30237f9c1ced5765fdbb499dd1f2c7dca478ad", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 84.62359157213572, "accuracy": { "name": "Decision accuracy", "value": 82.33333333333333 }, "ramPeakMb": 8826.8828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2542.947295611957, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.13554489993486465 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-sol-2b-native-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-d1-omni-600m-scifact", "suite": "decision", "cohort": "decision-7b9a92431f5ed927f8d0cc82", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:40:39Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1060.459005413012, "accuracy": { "name": "Decision accuracy", "value": 64.33333333333333 }, "ramPeakMb": 174.0703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 454.3944312859094, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4944235798497427 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-d1-omni-600m-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-d1-omni-600m-xnli-de", "suite": "decision", "cohort": "decision-f0a8ac0aca3a20bef9af08e5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:38:16Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1097.9091508581876, "accuracy": { "name": "Decision accuracy", "value": 55.333333333333336 }, "ramPeakMb": 349.94921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 100.59727308068734, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6387099993079756 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-d1-omni-600m-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-d1-omni-600m-anli-r3", "suite": "decision", "cohort": "decision-f64412c86afc224b8f2ed7de", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:37:43Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1141.2657817929535, "accuracy": { "name": "Decision accuracy", "value": 34.666666666666664 }, "ramPeakMb": 158.390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 129.10226728126872, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0300921593241008 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-d1-omni-600m-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu2-d-d1-omni-600m-boolq", "suite": "decision", "cohort": "decision-722ef262c6a65e6cfe85e365", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:37:02Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1166.2691467067962, "accuracy": { "name": "Decision accuracy", "value": 76.33333333333333 }, "ramPeakMb": 166.74609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 149.98253232877082, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.184505528188136 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu2-d-d1-omni-600m-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-scifact", "suite": "decision", "cohort": "decision-f23d244ce6f2d495cda1f5c5", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:36:00Z", "model": { "name": "vllm-sr/Decision-2.0-Eos-0.8B", "revision": "52cad37ef21f598302fe964cbfa164739cb965d5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 164.12237925872608, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 4379.74609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3364.5624837640207, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4275615223340887 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-laya-scifact", "suite": "decision", "cohort": "decision-9b59c552591b80bfd63de757", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:31:12Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 855.7679884271729, "accuracy": { "name": "Decision accuracy", "value": 53.333333333333336 }, "ramPeakMb": 176.20703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 538.6740404338343, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6190891432564043 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-laya-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-laya-xnli-de", "suite": "decision", "cohort": "decision-493e706481bf0ffd47458b7c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:28:23Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 809.0555434326213, "accuracy": { "name": "Decision accuracy", "value": 44 }, "ramPeakMb": 351.75390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 147.14276125169513, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6721993852728109 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-laya-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-laya-anli-r3", "suite": "decision", "cohort": "decision-d1daa55fe20ae9d2a8848c31", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:27:35Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 831.6180601097412, "accuracy": { "name": "Decision accuracy", "value": 36 }, "ramPeakMb": 160.02734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 170.80757799791172, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7943077990921543 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-laya-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-laya-boolq", "suite": "decision", "cohort": "decision-36c05b72ba7b7ffdc2fd246a", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:26:41Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 857.4796869480297, "accuracy": { "name": "Decision accuracy", "value": 72.33333333333333 }, "ramPeakMb": 200.4921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 199.21949087176472, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19669982894822452 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-laya-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-scifact", "suite": "decision", "cohort": "decision-feccb634e8c60ac1bf84fc4f", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:25:23Z", "model": { "name": "vllm-sr/Decision-2.0-Kai-0.6B", "revision": "cc7f30cda9e7c7b28c5f71ef33f01e09385f1c4b", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 232.92481469351105, "accuracy": { "name": "Decision accuracy", "value": 51 }, "ramPeakMb": 3701.84375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2328.490994173335, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5615117633744756 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-xnli-de", "suite": "decision", "cohort": "decision-6e01b97a49a457de7c9b4ce1", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:18:26Z", "model": { "name": "vllm-sr/Decision-2.0-Eos-0.8B", "revision": "52cad37ef21f598302fe964cbfa164739cb965d5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 167.61827609996232, "accuracy": { "name": "Decision accuracy", "value": 69.66666666666667 }, "ramPeakMb": 3956.23828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 991.7971786930381, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4262441866244841 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-xnli-de", "suite": "decision", "cohort": "decision-37877df59e90aa7929e0ec71", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:13:15Z", "model": { "name": "vllm-sr/Decision-2.0-Kai-0.6B", "revision": "cc7f30cda9e7c7b28c5f71ef33f01e09385f1c4b", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 241.04342211667975, "accuracy": { "name": "Decision accuracy", "value": 64 }, "ramPeakMb": 3478.2578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 676.4064827058852, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4820156500533235 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-anli-r3", "suite": "decision", "cohort": "decision-44f54ddfbb891f96a0126084", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:13:11Z", "model": { "name": "vllm-sr/Decision-2.0-Eos-0.8B", "revision": "52cad37ef21f598302fe964cbfa164739cb965d5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 163.41725776342432, "accuracy": { "name": "Decision accuracy", "value": 33.333333333333336 }, "ramPeakMb": 3774.01953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1257.0683750186192, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.9587574732159684 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-anli-r3", "suite": "decision", "cohort": "decision-2d06a19195baa03d52512041", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:09:40Z", "model": { "name": "vllm-sr/Decision-2.0-Kai-0.6B", "revision": "cc7f30cda9e7c7b28c5f71ef33f01e09385f1c4b", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 255.4279079055762, "accuracy": { "name": "Decision accuracy", "value": 33.333333333333336 }, "ramPeakMb": 3342.421875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 764.6384516143007, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7813754045623912 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-boolq", "suite": "decision", "cohort": "decision-2047e0bc1d51130fb3fd3be0", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:06:36Z", "model": { "name": "vllm-sr/Decision-2.0-Eos-0.8B", "revision": "52cad37ef21f598302fe964cbfa164739cb965d5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 16384, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 170.15847948116007, "accuracy": { "name": "Decision accuracy", "value": 78.66666666666667 }, "ramPeakMb": 3909.18359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1264.6641765340853, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.16151230404022943 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu2-d-decision2-eos-08b-native-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-boolq", "suite": "decision", "cohort": "decision-14b07dadcc5fc43f34da5f19", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T14:05:39Z", "model": { "name": "vllm-sr/Decision-2.0-Kai-0.6B", "revision": "cc7f30cda9e7c7b28c5f71ef33f01e09385f1c4b", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 255.19176837677313, "accuracy": { "name": "Decision accuracy", "value": 79 }, "ramPeakMb": 3473.28515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 809.7048008810574, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1502288853143308 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-cpu1-d-decision2-kai-06b-native-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-julia-1-scifact", "suite": "decision", "cohort": "decision-9b59c552591b80bfd63de757", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T14:00:44Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 3657.0449334564005, "accuracy": { "name": "Decision accuracy", "value": 37 }, "ramPeakMb": 254.6796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 126.37525882494326, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.1685922699076685 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-julia-1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-julia-1-xnli-de", "suite": "decision", "cohort": "decision-493e706481bf0ffd47458b7c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T13:59:59Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 2806.0012177453036, "accuracy": { "name": "Decision accuracy", "value": 35.333333333333336 }, "ramPeakMb": 436.33203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 33.44973601808306, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.2085696722876889 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-julia-1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-julia-1-anli-r3", "suite": "decision", "cohort": "decision-d1daa55fe20ae9d2a8848c31", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T13:59:42Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 3350.286330332668, "accuracy": { "name": "Decision accuracy", "value": 32.666666666666664 }, "ramPeakMb": 242.1796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 40.349785064463504, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.2684857729029668 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-julia-1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261102-next-01a1930a-igpu1-d-julia-1-boolq", "suite": "decision", "cohort": "decision-36c05b72ba7b7ffdc2fd246a", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-11-02T13:59:25Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 3432.1535872300797, "accuracy": { "name": "Decision accuracy", "value": 57.666666666666664 }, "ramPeakMb": 269.12890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 46.18965788414547, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3548582402077574 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261102-next-01a1930a-igpu1-d-julia-1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-lite-scifact", "suite": "decision", "cohort": "decision-e8f6224ee8a6e351199f1850", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T10:25:38Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 44.13142504511524, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 2441.921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 12761.881118143016, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3281874952386315 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-lite-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-lite-xnli-de", "suite": "decision", "cohort": "decision-ab989d6afd17eee20d0ed909", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T09:19:33Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 40.766137280704484, "accuracy": { "name": "Decision accuracy", "value": 69.33333333333333 }, "ramPeakMb": 2157.70703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 4274.217400916386, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3871131601519058 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-lite-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-d3-edge-scifact", "suite": "decision", "cohort": "decision-bda5f640abc1707c5d5d8f89", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T09:16:52Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 100.18502780862097, "accuracy": { "name": "Decision accuracy", "value": 70.66666666666667 }, "ramPeakMb": 1992.8125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5621.598479523866, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.42885753831645973 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-d3-edge-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-lite-anli-r3", "suite": "decision", "cohort": "decision-6f0170bb965203ea415353a8", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:57:28Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 42.07339305970785, "accuracy": { "name": "Decision accuracy", "value": 42 }, "ramPeakMb": 1953.09765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5072.722952574452, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.834480851990302 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-lite-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-d3-edge-xnli-de", "suite": "decision", "cohort": "decision-3b7a9b7247c28a1ec61fe2e9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:47:44Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 95.0500319375955, "accuracy": { "name": "Decision accuracy", "value": 63.666666666666664 }, "ramPeakMb": 1653.71484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1833.1749056931585, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5016537496986987 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-d3-edge-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-d3-edge-anli-r3", "suite": "decision", "cohort": "decision-619c2f55e987b57b7cfd7adb", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:38:13Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 97.407645712978, "accuracy": { "name": "Decision accuracy", "value": 40.333333333333336 }, "ramPeakMb": 1410.05078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2191.06688293803, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8804516675176645 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-d3-edge-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-lite-boolq", "suite": "decision", "cohort": "decision-378cabdf482c315aa031298e", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:31:21Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 41.8824771036854, "accuracy": { "name": "Decision accuracy", "value": 85.66666666666667 }, "ramPeakMb": 2043.23828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5567.801846009097, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.10635553110316007 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-lite-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-d3-edge-boolq", "suite": "decision", "cohort": "decision-a1fcc25de6e03994471e2501", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:26:52Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 97.8038056803119, "accuracy": { "name": "Decision accuracy", "value": 81.66666666666667 }, "ramPeakMb": 1524.484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2384.297131499819, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.13140140460044575 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-d3-edge-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-scifact", "suite": "decision", "cohort": "decision-2f9eada1e9d8caad5f4913f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:14:01Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "laya PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cpu" }, "metrics": { "tokensPerSecond": 706.068681459581, "accuracy": { "name": "Decision accuracy", "value": 39.666666666666664 }, "ramPeakMb": 3309.88671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 670.1331080453625, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8054371234666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-xnli-de", "suite": "decision", "cohort": "decision-b7a8a315bc86b7cbe8797b01", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:10:26Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "laya PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cpu" }, "metrics": { "tokensPerSecond": 705.2366546048806, "accuracy": { "name": "Decision accuracy", "value": 47.666666666666664 }, "ramPeakMb": 2552.68359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 143.01582219447786, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6762827914333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-anli-r3", "suite": "decision", "cohort": "decision-0ac8fff8c69aeb7b884608af", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:09:35Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "laya PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cpu" }, "metrics": { "tokensPerSecond": 795.5716648218033, "accuracy": { "name": "Decision accuracy", "value": 39.333333333333336 }, "ramPeakMb": 2360.6875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 178.71844815536556, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8638180312333332 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-boolq", "suite": "decision", "cohort": "decision-4624f6b7394fe28ca7e4b5e1", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:08:34Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "laya PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cpu" }, "metrics": { "tokensPerSecond": 816.3960141494164, "accuracy": { "name": "Decision accuracy", "value": 68 }, "ramPeakMb": 2406.2578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 211.33126204658765, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.24909894159999998 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-laya-multilingual-native-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-scifact", "suite": "decision", "cohort": "decision-2f3dcb6c6bed973542c8ca61", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:06:54Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 83.95411542136677, "accuracy": { "name": "Decision accuracy", "value": 76.66666666666667 }, "ramPeakMb": 10617.2109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6089.04039348495, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.39057931899431486 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-nano-scifact", "suite": "decision", "cohort": "decision-e8f6224ee8a6e351199f1850", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T08:01:59Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 17.850737975072793, "accuracy": { "name": "Decision accuracy", "value": 82.33333333333333 }, "ramPeakMb": 4194.7890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 31550.516330835526, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2684919154452215 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-nano-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-xnli-de", "suite": "decision", "cohort": "decision-e701e5dfd5375b91cef575d5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T07:35:14Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 72.97489575213798, "accuracy": { "name": "Decision accuracy", "value": 69.66666666666667 }, "ramPeakMb": 10733.2109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1675.1422811008524, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.43141803020274494 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-anli-r3", "suite": "decision", "cohort": "decision-84111a9b2849e7bc91b6b38f", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T07:26:25Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.06348037249707, "accuracy": { "name": "Decision accuracy", "value": 40.333333333333336 }, "ramPeakMb": 10698.71484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2094.7232837965325, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7192459976288936 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-boolq", "suite": "decision", "cohort": "decision-995c67b753a488f9521bfd3c", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T07:15:27Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.46340577789222, "accuracy": { "name": "Decision accuracy", "value": 87 }, "ramPeakMb": 9934.55078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2274.5363641578783, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.09952410217534358 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-jeetu-decider-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-scifact", "suite": "decision", "cohort": "decision-cb3b05e3ff12df08513c855f", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T07:02:06Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "evalengine authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 167.830917062776, "accuracy": { "name": "Decision accuracy", "value": 73.33333333333333 }, "ramPeakMb": 1678.85546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3522.2552773886127, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.40443702368571616 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-xnli-de", "suite": "decision", "cohort": "decision-a64dfc3010542889a6ab909f", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:43:51Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "evalengine authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 195.09767473924683, "accuracy": { "name": "Decision accuracy", "value": 68.33333333333333 }, "ramPeakMb": 1768.51953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1037.0702791328465, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.47531107548281076 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-anli-r3", "suite": "decision", "cohort": "decision-22e710115edd030315a1944a", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:38:25Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "evalengine authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 185.25098021113254, "accuracy": { "name": "Decision accuracy", "value": 34.333333333333336 }, "ramPeakMb": 1583.625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1306.9476504652994, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.945195354798373 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-boolq", "suite": "decision", "cohort": "decision-0c58882894983c6ecea20518", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:31:38Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "evalengine authored all-option llama.cpp readout / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cpu" }, "metrics": { "tokensPerSecond": 179.95587807422785, "accuracy": { "name": "Decision accuracy", "value": 84.33333333333333 }, "ramPeakMb": 1725.82421875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1413.0685961539468, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11718892453483196 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-evalengine-08b-q8-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-decima-small-scifact", "suite": "decision", "cohort": "decision-374f45f770c5083b8c8823d7", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:23:48Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 40.666666666666664 }, "ramPeakMb": 1217.828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 157.49759691845006, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6755408331666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-decima-small-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-decima-small-xnli-de", "suite": "decision", "cohort": "decision-b97b577929d864462064ad25", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:22:54Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 61.666666666666664 }, "ramPeakMb": 1311.73046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 65.33840550575405, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5106919745666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-decima-small-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-decima-small-anli-r3", "suite": "decision", "cohort": "decision-c0fb57d876d2c04e0a619f92", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:22:28Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 1150.28515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 73.1489523112153, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8130318033333332 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-decima-small-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-decima-small-boolq", "suite": "decision", "cohort": "decision-0cbe539d4c5829ae19646fab", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:22:00Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 68.33333333333333 }, "ramPeakMb": 1232.34765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 71.09853800133958, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19824781573333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-decima-small-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-lev-4b-scifact", "suite": "decision", "cohort": "decision-182830ed1829a5f421ea0b4c", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T06:21:09Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 34.01197099763184, "accuracy": { "name": "Decision accuracy", "value": 82.66666666666667 }, "ramPeakMb": 5965.6484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 33411.765524530296, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2795640576640399 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-lev-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-nano-xnli-de", "suite": "decision", "cohort": "decision-ab989d6afd17eee20d0ed909", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T05:18:44Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 17.267846168840045, "accuracy": { "name": "Decision accuracy", "value": 75 }, "ramPeakMb": 3865.96875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 10090.623441373755, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3581080635752925 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-nano-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-nano-anli-r3", "suite": "decision", "cohort": "decision-6f0170bb965203ea415353a8", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T04:26:46Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 17.382228537949494, "accuracy": { "name": "Decision accuracy", "value": 52 }, "ramPeakMb": 3684.234375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 12278.440949082336, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7291139714123692 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-nano-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-lev-4b-xnli-de", "suite": "decision", "cohort": "decision-d6254bae4f3b2f2de945cb9f", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T03:24:31Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 23.583092536505706, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 5485.609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 15201.003265866992, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.47618941055085123 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-lev-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-d3-nano-boolq", "suite": "decision", "cohort": "decision-378cabdf482c315aa031298e", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T03:23:36Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": 17.513133534037536, "accuracy": { "name": "Decision accuracy", "value": 90 }, "ramPeakMb": 3802.078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 13315.3403347329, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.06686678981256254 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-d3-nano-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-scifact", "suite": "decision", "cohort": "decision-3d03a979203b077135ed7b47", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T02:13:21Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "Mapika authored llama.cpp binding / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cpu" }, "metrics": { "tokensPerSecond": 38.03370060275939, "accuracy": { "name": "Decision accuracy", "value": 82.66666666666667 }, "ramPeakMb": 5538.03515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 13256.6642742994, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.26415025713333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-lev-4b-anli-r3", "suite": "decision", "cohort": "decision-03b99001dfb624b27e5a242e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T02:06:56Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 28.539312133620015, "accuracy": { "name": "Decision accuracy", "value": 59.666666666666664 }, "ramPeakMb": 5324.76171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 15307.07296966381, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6040847648512803 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-lev-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-xnli-de", "suite": "decision", "cohort": "decision-64808b35b10cf6d68c7769fd", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T01:04:27Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "Mapika authored llama.cpp binding / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cpu" }, "metrics": { "tokensPerSecond": 39.012278877205084, "accuracy": { "name": "Decision accuracy", "value": 77.33333333333333 }, "ramPeakMb": 5459.01953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2954.027210152805, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3278265340666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-anli-r3", "suite": "decision", "cohort": "decision-8df9b3c96bee3c0488dbf6fa", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T00:49:08Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "Mapika authored llama.cpp binding / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cpu" }, "metrics": { "tokensPerSecond": 39.37134597840385, "accuracy": { "name": "Decision accuracy", "value": 60.666666666666664 }, "ramPeakMb": 5290.8828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3922.311082566836, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5693915204 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-lev-4b-boolq", "suite": "decision", "cohort": "decision-2d5914a0d872dfe07a8986e2", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T00:48:55Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 40.79640953767895, "accuracy": { "name": "Decision accuracy", "value": 89.33333333333333 }, "ramPeakMb": 5365.375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5863.09766089631, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.10341898014261433 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-lev-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-boolq", "suite": "decision", "cohort": "decision-3a3c6c9b89afc28f6c855639", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T00:28:56Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "cpu", "backend": "Mapika authored llama.cpp binding / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cpu" }, "metrics": { "tokensPerSecond": 39.17408554500472, "accuracy": { "name": "Decision accuracy", "value": 91.33333333333333 }, "ramPeakMb": 5486.890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 4344.538767543369, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.08666666666666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-4b-v3-q4-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T00:17:23Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 85.03105426851933, "accuracy": { "name": "Decision accuracy", "value": 79.33333333333333 }, "ramPeakMb": 2931.45703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 7117.39969833657, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2983041819637792 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-scifact", "suite": "decision", "cohort": "decision-f8bf47180a0fab26c1eecf7e", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-02T00:05:12Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 140.7956303227518, "accuracy": { "name": "Decision accuracy", "value": 81 }, "ramPeakMb": 9665.5859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3936.2017040556143, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.30798077666305723 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-xnli-de", "suite": "decision", "cohort": "decision-26bd4938531e4a607e064489", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:44:39Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 149.04680530552983, "accuracy": { "name": "Decision accuracy", "value": 59.333333333333336 }, "ramPeakMb": 6233.59765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1108.6673947462502, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.560037672110274 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:40:37Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 73.67651807155059, "accuracy": { "name": "Decision accuracy", "value": 62.666666666666664 }, "ramPeakMb": 2715.6953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2975.755899870516, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5024873400720852 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-anli-r3", "suite": "decision", "cohort": "decision-2fb4df8f91c9469cf01c6f3e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:38:50Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 144.43417839998736, "accuracy": { "name": "Decision accuracy", "value": 42.333333333333336 }, "ramPeakMb": 6170.53125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1415.3621319500962, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8127484141778214 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-boolq", "suite": "decision", "cohort": "decision-34f7a7a5d85d0b482ca07dd6", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:31:27Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": 142.8229716167536, "accuracy": { "name": "Decision accuracy", "value": 78 }, "ramPeakMb": 6547.3828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1569.7288103970156, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.14538920505415193 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-jpt-08b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:25:15Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 67.82318274480905, "accuracy": { "name": "Decision accuracy", "value": 48.666666666666664 }, "ramPeakMb": 2518.359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3811.676030785893, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6857608441549283 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-scifact", "suite": "decision", "cohort": "decision-3fab242b12f63b8974a4c57c", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:22:34Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 177.11758723457936, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 4899.91796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2846.6964115326605, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.29473913243333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-xnli-de", "suite": "decision", "cohort": "decision-eb0253c2a84431186130edf9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:07:45Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 165.75552903921795, "accuracy": { "name": "Decision accuracy", "value": 66.33333333333333 }, "ramPeakMb": 5081.9296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 695.2608700375034, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.46252762606666664 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:05:40Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 71.48459318705734, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 2543.83984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3989.57762251026, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.09422696236762607 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-clef-german-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-anli-r3", "suite": "decision", "cohort": "decision-9a5295523023fc670ea36d33", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T23:04:05Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 92.9961751339465, "accuracy": { "name": "Decision accuracy", "value": 35 }, "ramPeakMb": 4891.22265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1660.5700873636913, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8615389018333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-boolq", "suite": "decision", "cohort": "decision-79506be127f36d2ddeea18b7", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:55:33Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 171.70601381029505, "accuracy": { "name": "Decision accuracy", "value": 83 }, "ramPeakMb": 4973.88671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 991.1902883108511, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1191328792 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-mapika-08b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-scifact", "suite": "decision", "cohort": "decision-319a2931bdcf2dfc6505b008", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:49:42Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 57.666666666666664 }, "ramPeakMb": 1689.5703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 645.1121454060194, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5474528195666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-xnli-de", "suite": "decision", "cohort": "decision-6b02f8db5aca086be8c96be1", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:46:14Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 69 }, "ramPeakMb": 1724.07421875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 266.42663965467364, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4722376521 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-anli-r3", "suite": "decision", "cohort": "decision-253537b767d235e513e5bde5", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:44:43Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 27 }, "ramPeakMb": 1543.2734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 280.7080154927098, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0208896278333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-julia-1-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:44:15Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 1393.510291609778, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 600.5390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 331.6516589670209, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.1650211715349386 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-julia-1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-boolq", "suite": "decision", "cohort": "decision-4d4df3145114a1f5258307cf", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:43:10Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "author PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 79.33333333333333 }, "ramPeakMb": 1623.203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 274.98534732022864, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.15398289296666665 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu2-d-decima-base-clean-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-julia-1-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:42:28Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 1444.507701067231, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 593.95703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 64.97715445245073, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.1976182101607697 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-julia-1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-julia-1-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:42:05Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 1614.0292124317991, "accuracy": { "name": "Decision accuracy", "value": 33.333333333333336 }, "ramPeakMb": 404.171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 83.75519618362887, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.2516362751963943 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-julia-1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-cpuall-01a1930a-cpu1-d-julia-1-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T22:41:36Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 1586.6330408484732, "accuracy": { "name": "Decision accuracy", "value": 58 }, "ramPeakMb": 419.15234375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 99.91598303992457, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3542917581800582 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-cpuall-01a1930a-cpu1-d-julia-1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-scifact", "suite": "decision", "cohort": "decision-fb86e809076a36f59feb155c", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:26:26Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4917.4929073735275, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": 2033.73046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 112.69970499988024, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.30887533237043496 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-xnli-de", "suite": "decision", "cohort": "decision-e908c47cd3b9da9b98e1ce64", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:25:38Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2233.8572841720256, "accuracy": { "name": "Decision accuracy", "value": 59 }, "ramPeakMb": 2212.0625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 73.9721980021568, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5606167417813972 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-anli-r3", "suite": "decision", "cohort": "decision-9b0ae51cfdcf5c5ba9cb0df5", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:25:03Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2559.761870755384, "accuracy": { "name": "Decision accuracy", "value": 42 }, "ramPeakMb": 2022.58203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 79.86159533126435, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.81179486268072 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-boolq", "suite": "decision", "cohort": "decision-c0d253edffa0540a3151b5ae", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:24:26Z", "model": { "name": "kirp/jpt-0.8b", "revision": "1431c0509bbc10772cd59964cc7af5835c8720d6", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2779.38816186014, "accuracy": { "name": "Decision accuracy", "value": 77.66666666666667 }, "ramPeakMb": 2055.08984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 80.66283666664579, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1456136836265135 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-jptretry2-01a1930a-gpu-d-jpt-08b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-scifact", "suite": "decision", "cohort": "decision-99f8500ecfd1e2792ccddf08", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:22:22Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "Mapika authored llama.cpp binding / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cuda:0" }, "metrics": { "tokensPerSecond": 2429.8134076615565, "accuracy": { "name": "Decision accuracy", "value": 82 }, "ramPeakMb": 3878.72265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 207.50564566405956, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.26856287623333336 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-xnli-de", "suite": "decision", "cohort": "decision-83ce76d3e150b8ed046cd601", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:21:07Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "Mapika authored llama.cpp binding / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cuda:0" }, "metrics": { "tokensPerSecond": 1688.748041719495, "accuracy": { "name": "Decision accuracy", "value": 77.66666666666667 }, "ramPeakMb": 4041.671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 68.24187533386673, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3295593015666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-anli-r3", "suite": "decision", "cohort": "decision-e0b6a319eec420559bb661af", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:20:35Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "Mapika authored llama.cpp binding / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cuda:0" }, "metrics": { "tokensPerSecond": 1844.4533710305343, "accuracy": { "name": "Decision accuracy", "value": 62.333333333333336 }, "ramPeakMb": 3854.87109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 83.72489600015494, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5726201141666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-boolq", "suite": "decision", "cohort": "decision-a947291b8dd3f3f809ebaee5", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:19:59Z", "model": { "name": "Mapika/decider-4b-GGUF", "revision": "d3a78581751d518bbfa9e871a6d84059554d53c5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "Mapika authored llama.cpp binding / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Mapika authored llama.cpp binding / cuda:0" }, "metrics": { "tokensPerSecond": 1933.946664931716, "accuracy": { "name": "Decision accuracy", "value": 91.33333333333333 }, "ramPeakMb": 4026.625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 88.0031163317229, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.08666666666666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-mapika-4b-v3-q4-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-scifact", "suite": "decision", "cohort": "decision-cc3a27c01231f994e38e8951", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:18:23Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "zefan PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2531.1926701477055, "accuracy": { "name": "Decision accuracy", "value": 73.66666666666667 }, "ramPeakMb": 5182.73046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 586.1268553347327, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.38567660534106696 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-xnli-de", "suite": "decision", "cohort": "decision-eba7a30de322362e40181e3c", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:15:06Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "zefan PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 732.401994769472, "accuracy": { "name": "Decision accuracy", "value": 63.666666666666664 }, "ramPeakMb": 5367.68359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 421.53080167016014, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.46278282744048854 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-anli-r3", "suite": "decision", "cohort": "decision-65425ddecb45f64273d17e80", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:12:42Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "zefan PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 986.3826087559343, "accuracy": { "name": "Decision accuracy", "value": 49 }, "ramPeakMb": 5177.828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 432.16495933321613, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.656990627910133 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-boolq", "suite": "decision", "cohort": "decision-3d27e46cb645390095b92f53", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:10:14Z", "model": { "name": "ZefanCai/Open-Jev-2B", "revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "zefan PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "zefan PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1162.0354298691516, "accuracy": { "name": "Decision accuracy", "value": 81 }, "ramPeakMb": 5218.828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 155.92754633456934, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11541590398024844 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-zefanretry-01a1930a-gpu-d-open-jev-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-scifact", "suite": "decision", "cohort": "decision-043172d6162c5c61a3f08a0e", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:06:08Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5305.300713766576, "accuracy": { "name": "Decision accuracy", "value": 49 }, "ramPeakMb": 3393.578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 92.7933576675908, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7088672945 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-xnli-de", "suite": "decision", "cohort": "decision-59ca417c665a80b389f92ab3", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:05:30Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2616.899812398426, "accuracy": { "name": "Decision accuracy", "value": 54.666666666666664 }, "ramPeakMb": 3571.35546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 43.16940200185248, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6042852969000001 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-anli-r3", "suite": "decision", "cohort": "decision-db103b83f33c02263dc61d85", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:05:07Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3266.6877320209624, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 3386.29296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 44.34461199950116, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8733027420333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-boolq", "suite": "decision", "cohort": "decision-03340e5386d940eab51acc5d", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:04:44Z", "model": { "name": "jaredpalmer/kev-0.5b", "revision": "9ce2fd39db3a397c89733f94af948e3d1fdfffcd", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3422.3144436293433, "accuracy": { "name": "Decision accuracy", "value": 78 }, "ramPeakMb": 3415.05859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 46.34777700276269, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1677773858 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-kev-05b-prototype-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-scifact", "suite": "decision", "cohort": "decision-68ccff1aac28257136969ad1", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:03:51Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "openjev-flash authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 1448.0119965033473, "accuracy": { "name": "Decision accuracy", "value": 87.66666666666667 }, "ramPeakMb": 6368.97265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 361.32297333407524, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.20227943096666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-xnli-de", "suite": "decision", "cohort": "decision-b75d418c5642000dd41b4e1c", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:01:42Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "openjev-flash authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 1167.3521057078112, "accuracy": { "name": "Decision accuracy", "value": 78.66666666666667 }, "ramPeakMb": 6423.265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 114.9981506667488, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.318060006 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-anli-r3", "suite": "decision", "cohort": "decision-94f4713a9260fee6366dd893", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T13:00:52Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "openjev-flash authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 1266.9038233740046, "accuracy": { "name": "Decision accuracy", "value": 66 }, "ramPeakMb": 6246.3046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 136.89015966878893, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5235024197666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-boolq", "suite": "decision", "cohort": "decision-808110eae57b4c4784ce5c65", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:59:55Z", "model": { "name": "openjev/OpenJev-Flash-9B-GGUF", "revision": "a3acd96966b06c2f161209f888b66235029f9ac1", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "openjev-flash authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "openjev-flash authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 1335.501918679348, "accuracy": { "name": "Decision accuracy", "value": 88.33333333333333 }, "ramPeakMb": 6306.03515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 150.64997700062426, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.08177393013333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-openjev-flash-9b-q4-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-scifact", "suite": "decision", "cohort": "decision-fbe64a0f4e8ea754062c595c", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:56:58Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "evalengine authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 4843.733924258719, "accuracy": { "name": "Decision accuracy", "value": 73 }, "ramPeakMb": 2238.09765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 122.04289966728538, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.40523440673836636 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-xnli-de", "suite": "decision", "cohort": "decision-b0a142fa6fdda8130e6f944c", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:56:10Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "evalengine authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 4716.315257114656, "accuracy": { "name": "Decision accuracy", "value": 68.33333333333333 }, "ramPeakMb": 2296.5078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 42.90001600185254, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4731062803984089 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-anli-r3", "suite": "decision", "cohort": "decision-0db2f7346c983f0f82e968d1", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:55:48Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "evalengine authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 5219.534554525752, "accuracy": { "name": "Decision accuracy", "value": 35.333333333333336 }, "ramPeakMb": 2107.87890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 46.38600066808673, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.9458742387132476 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-boolq", "suite": "decision", "cohort": "decision-56afaae059915a4bd733e626", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:55:26Z", "model": { "name": "evalengine/decision-0.8b-gguf", "revision": "ce993792ad283053004bf0230b1709ff7351bb24", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "evalengine authored all-option llama.cpp readout / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "evalengine authored all-option llama.cpp readout / cuda:0" }, "metrics": { "tokensPerSecond": 5298.586390193733, "accuracy": { "name": "Decision accuracy", "value": 84 }, "ramPeakMb": 2496.0234375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 47.99204566535385, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11775782018761516 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-final5050-01a1930a-gpu-d-evalengine-08b-q8-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-extra-01a1930a-gpu-d-mapika-08b-scifact", "suite": "decision", "cohort": "decision-bfa845f12622855279db576b", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:54:04Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4968.46044232489, "accuracy": { "name": "Decision accuracy", "value": 78 }, "ramPeakMb": 2407.33203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 101.4801276678918, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2953449858 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-extra-01a1930a-gpu-d-mapika-08b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-extra-01a1930a-gpu-d-mapika-08b-xnli-de", "suite": "decision", "cohort": "decision-51599477ba81849427257e2b", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:53:20Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1672.1646541530788, "accuracy": { "name": "Decision accuracy", "value": 66.33333333333333 }, "ramPeakMb": 2592.015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 68.918651669325, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.46296716613333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-extra-01a1930a-gpu-d-mapika-08b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-extra-01a1930a-gpu-d-mapika-08b-anli-r3", "suite": "decision", "cohort": "decision-305d64a5903265404452b2f3", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:52:48Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2165.625007308168, "accuracy": { "name": "Decision accuracy", "value": 35 }, "ramPeakMb": 2402.1015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 71.30812866749086, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8642108260333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-extra-01a1930a-gpu-d-mapika-08b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-extra-01a1930a-gpu-d-mapika-08b-boolq", "suite": "decision", "cohort": "decision-e975fc168238392d5ce38faf", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:52:16Z", "model": { "name": "Mapika/decider-0.8b", "revision": "a0a01d6f8135298f400a8c856b355793012ae971", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2356.3924458472557, "accuracy": { "name": "Decision accuracy", "value": 82.66666666666667 }, "ramPeakMb": 2425.59765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 72.22622599782578, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11940864386666668 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-extra-01a1930a-gpu-d-mapika-08b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-base-clean-scifact", "suite": "decision", "cohort": "decision-071527abf90fa7b943130d96", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:34:02Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 57.666666666666664 }, "ramPeakMb": 2189.06640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 55.88123499939684, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.547453006 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-base-clean-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-base-clean-xnli-de", "suite": "decision", "cohort": "decision-b47642d6a176f8cc2fa82631", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:33:36Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 69 }, "ramPeakMb": 2371.26953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 44.39807300145427, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4722376521 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-base-clean-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-base-clean-anli-r3", "suite": "decision", "cohort": "decision-892a0182996bdf7b2e5f57b8", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:33:11Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 27 }, "ramPeakMb": 2184.984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 44.34722399940559, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0208894864666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-base-clean-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-base-clean-boolq", "suite": "decision", "cohort": "decision-9acae2c71d9b404fb714a0cc", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:32:48Z", "model": { "name": "amyrmahdy/decima-base-clean", "revision": "a37e006fd4015c3c96176c341f3d6620f8a101f3", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 79.33333333333333 }, "ramPeakMb": 2214.5390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 44.18456000217702, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1539827812 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-base-clean-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-small-scifact", "suite": "decision", "cohort": "decision-071527abf90fa7b943130d96", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:32:02Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 40.666666666666664 }, "ramPeakMb": 2059.48828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 29.686511000036262, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6755415298999999 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-small-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-small-xnli-de", "suite": "decision", "cohort": "decision-b47642d6a176f8cc2fa82631", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:31:44Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 61.666666666666664 }, "ramPeakMb": 2240.20703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 28.66708666784689, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5106924803666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-small-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-small-anli-r3", "suite": "decision", "cohort": "decision-892a0182996bdf7b2e5f57b8", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:31:26Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 37.333333333333336 }, "ramPeakMb": 2055.26953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 29.308930667854536, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8130320050000001 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-small-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-decima-small-boolq", "suite": "decision", "cohort": "decision-9acae2c71d9b404fb714a0cc", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:31:08Z", "model": { "name": "amyrmahdy/decima-small", "revision": "480c4ea9e54f2946b947fb3ceb3c7cc7ece77522", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 68.33333333333333 }, "ramPeakMb": 2062.65625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.87364266696386, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19824763353333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-decima-small-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-scifact", "suite": "decision", "cohort": "decision-7e1c1412b75fcc3284920800", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:30:41Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3199.0745268331293, "accuracy": { "name": "Decision accuracy", "value": 83.33333333333333 }, "ramPeakMb": 2902.55859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 165.42284200045592, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2491209776 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-xnli-de", "suite": "decision", "cohort": "decision-a98a54f598810807573bdb5b", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:29:36Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1485.6381625164547, "accuracy": { "name": "Decision accuracy", "value": 74.33333333333333 }, "ramPeakMb": 3084.9296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 94.39938800157204, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3633547932 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-anli-r3", "suite": "decision", "cohort": "decision-9a263adee48c81f724450707", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:28:53Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1896.3477953550066, "accuracy": { "name": "Decision accuracy", "value": 48.666666666666664 }, "ramPeakMb": 2898.58203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 94.61696167030216, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6907065194666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-boolq", "suite": "decision", "cohort": "decision-cec2adbc61f7211366dd0c0d", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:28:11Z", "model": { "name": "StrandsAgents/strands-decider-2B-qwen3.5-v1-2610", "revision": "216d1ccb46a9e9b036eb9187b248db98480f77d4", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2156.7797457650836, "accuracy": { "name": "Decision accuracy", "value": 90.33333333333333 }, "ramPeakMb": 2928.69921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 101.16625666657153, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07585192136666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-qwen35-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-scifact", "suite": "decision", "cohort": "decision-7e1c1412b75fcc3284920800", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:26:15Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3240.1374611784295, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 2902.43359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 163.3264040000116, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.29383000243333335 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-xnli-de", "suite": "decision", "cohort": "decision-a98a54f598810807573bdb5b", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:25:12Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1507.1202521348348, "accuracy": { "name": "Decision accuracy", "value": 67.66666666666667 }, "ramPeakMb": 3084.61328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 93.05384433304425, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.42126499013333335 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-anli-r3", "suite": "decision", "cohort": "decision-9a263adee48c81f724450707", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:24:30Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1810.8236550686343, "accuracy": { "name": "Decision accuracy", "value": 46 }, "ramPeakMb": 2899.48828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 99.0856653349086, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6771577462333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-boolq", "suite": "decision", "cohort": "decision-cec2adbc61f7211366dd0c0d", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:23:47Z", "model": { "name": "StrandsAgents/strands-decider-2B-hobson-v21", "revision": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 4096, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2101.8988339652906, "accuracy": { "name": "Decision accuracy", "value": 92.33333333333333 }, "ramPeakMb": 3076.4296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 103.80772366751141, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07353292656666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-strands-hobson-2b-v21-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-scifact", "suite": "decision", "cohort": "decision-864d1b87d367dbb863f4fbf2", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:21:51Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3671.375539369297, "accuracy": { "name": "Decision accuracy", "value": 76.33333333333333 }, "ramPeakMb": 4559.7109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 139.23936533275992, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3902762662474881 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-xnli-de", "suite": "decision", "cohort": "decision-9618ad590dfabc1e79d33e7f", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:20:53Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1515.2968082993052, "accuracy": { "name": "Decision accuracy", "value": 70 }, "ramPeakMb": 4744.08203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 80.67286399852794, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.43143824856834706 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-anli-r3", "suite": "decision", "cohort": "decision-2fd124b6532fd04317594d1f", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:20:16Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1971.1859729149053, "accuracy": { "name": "Decision accuracy", "value": 40.666666666666664 }, "ramPeakMb": 4555.203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 81.89316933295534, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7193899229329451 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-boolq", "suite": "decision", "cohort": "decision-a0d92c11b28d255c8023c24a", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:19:38Z", "model": { "name": "Jeetu456/decider-2b", "revision": "cd0216309bf590f387500d78d1970af15d04baf5", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "author PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 12288, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "author PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2086.9660195010492, "accuracy": { "name": "Decision accuracy", "value": 87 }, "ramPeakMb": 4606.109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 84.42558800044935, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.09941423955790916 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-author-01a1930a-gpu-d-jeetu-decider-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-lev-4b-scifact", "suite": "decision", "cohort": "decision-8f50c5a35362e79c685b1853", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:16:37Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3447.329822364491, "accuracy": { "name": "Decision accuracy", "value": 82.66666666666667 }, "ramPeakMb": 4156.3984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 329.6464389997224, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.28107639099708914 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-lev-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-lev-4b-xnli-de", "suite": "decision", "cohort": "decision-c5ecaa22487c0dae03b35b5e", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:14:49Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2854.6562769112834, "accuracy": { "name": "Decision accuracy", "value": 69.66666666666667 }, "ramPeakMb": 4334.23828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 125.57962566846982, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4674351398744262 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-lev-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-lev-4b-anli-r3", "suite": "decision", "cohort": "decision-a7ff885182fdfc0a4665be9f", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:14:05Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3060.6484196253673, "accuracy": { "name": "Decision accuracy", "value": 59 }, "ramPeakMb": 4148.55078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 142.73228200016698, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6036240239526622 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-lev-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-lev-4b-boolq", "suite": "decision", "cohort": "decision-17d357458a3f43b899679373", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:13:15Z", "model": { "name": "ggml-org/lev-GGUF", "revision": "2ef3b3c3ab3defcab1058ab34cac52c467fa9304", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2737.7218898112146, "accuracy": { "name": "Decision accuracy", "value": 89.33333333333333 }, "ramPeakMb": 4151.91796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 87.36947833288771, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1043561197581728 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-lev-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-clef-german-2b-scifact", "suite": "decision", "cohort": "decision-297fa9cb39c5aacb5d02fde4", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:11:50Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 6871.181945117854, "accuracy": { "name": "Decision accuracy", "value": 79.66666666666667 }, "ramPeakMb": 3331.515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 88.07800533211169, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2992783824636735 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-clef-german-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-clef-german-2b-xnli-de", "suite": "decision", "cohort": "decision-eca36a6d9cfe61ff5234f656", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:11:19Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 4997.571194090929, "accuracy": { "name": "Decision accuracy", "value": 62 }, "ramPeakMb": 3503.61328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 43.86997699853964, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.504818493876892 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-clef-german-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-clef-german-2b-anli-r3", "suite": "decision", "cohort": "decision-ee5eccb0f8550bfc997b90dd", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:11:00Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5186.3827720003555, "accuracy": { "name": "Decision accuracy", "value": 48.666666666666664 }, "ramPeakMb": 3343.09765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 49.845916000582896, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6843880416283687 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-clef-german-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-clef-german-2b-boolq", "suite": "decision", "cohort": "decision-f717c11b67e5f928dacd8fd7", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:10:41Z", "model": { "name": "johannhartmann/Qwen3.5-2B_qlora_decisions-de-GGUF", "revision": "4336446b0e16e3fe1502dfe31b29eede7dd62618", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5286.008210763717, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 3347.09765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 53.952495335252024, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.09531185071668835 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-clef-german-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-julia-1-scifact", "suite": "decision", "cohort": "decision-297fa9cb39c5aacb5d02fde4", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:09:28Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 16781.50472944787, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 1079.01953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.53984266911478, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.1689718580772102 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-julia-1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-julia-1-xnli-de", "suite": "decision", "cohort": "decision-eca36a6d9cfe61ff5234f656", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:09:15Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3748.3511604595865, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 1252.43359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 25.040343335519236, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.197748536926705 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-julia-1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-julia-1-anli-r3", "suite": "decision", "cohort": "decision-ee5eccb0f8550bfc997b90dd", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:09:05Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5775.777331299427, "accuracy": { "name": "Decision accuracy", "value": 33.333333333333336 }, "ramPeakMb": 1065.48046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 23.40521899983287, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.25010687231988 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-julia-1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-wave2-01a1930a-gpu-d-julia-1-boolq", "suite": "decision", "cohort": "decision-f717c11b67e5f928dacd8fd7", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T12:08:56Z", "model": { "name": "ggml-org/Julia-1-GGUF", "revision": "1e6b48a6983d7467f8eedc2cb0d21267b5fc052d", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 6401.8127901008365, "accuracy": { "name": "Decision accuracy", "value": 58 }, "ramPeakMb": 1073.4609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 24.76329833404937, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.344241081274071 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-wave2-01a1930a-gpu-d-julia-1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-scifact", "suite": "decision", "cohort": "decision-52030ee7ddd73d8cd87d4bd5", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T11:52:51Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "laya PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 9603.250410488341, "accuracy": { "name": "Decision accuracy", "value": 39.666666666666664 }, "ramPeakMb": 2828.7265625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 49.27081766848763, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8054371057333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-xnli-de", "suite": "decision", "cohort": "decision-254ce0618e75a9ea5d4a19db", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T11:52:24Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "laya PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3702.5121883048787, "accuracy": { "name": "Decision accuracy", "value": 47.666666666666664 }, "ramPeakMb": 2978.54296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.240963667475928, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6762832470333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-anli-r3", "suite": "decision", "cohort": "decision-71c74ec4edbd4682bc7af751", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T11:52:06Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "laya PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5108.703705088687, "accuracy": { "name": "Decision accuracy", "value": 39.333333333333336 }, "ramPeakMb": 2792.0078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.83158733431871, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8638181489333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-boolq", "suite": "decision", "cohort": "decision-0336a81cba2cdc7ca40919eb", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T11:51:47Z", "model": { "name": "convaiinnovations/laya-multilingual", "revision": "1720e3e3357cfe1e281542e223f8273b0890ca34", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "laya PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "laya PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5939.425297729112, "accuracy": { "name": "Decision accuracy", "value": 68 }, "ramPeakMb": 2849.48046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 29.048265000650037, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.24909923223333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-layanative-01a1930a-gpu-d-laya-multilingual-native-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-openai-01a1930a-api-d-openai-luna-scifact", "suite": "decision", "cohort": "decision-6c7dc76fdbf37ce07fa87c0e", "cohortLabel": "SciFact claims dev (all 300, v1) / Hosted API / kEmbed01 client / api / memory guard", "measuredAt": "2026-11-01T02:02:17Z", "model": { "name": "gpt-6-luna", "revision": null, "precision": "API" }, "engine": { "device": "api", "backend": "OpenAI Decisions API", "hardware": "Hosted API / kEmbed01 client", "threads": null }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "Hosted API called from the stated client host: latency includes network and provider queueing. Only the successful attempt is timed; retries are counted. Provider hardware and memory are unknown." ], "version": "decision-questions-v1", "backendCohort": "OpenAI Decisions API" }, "metrics": { "tokensPerSecond": 1657.162325297267, "accuracy": { "name": "Decision accuracy", "value": 86 }, "ramPeakMb": null, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 335.8150203542512, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.24517733333333336 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-openai-01a1930a-api-d-openai-luna-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-openai-01a1930a-api-d-openai-luna-xnli-de", "suite": "decision", "cohort": "decision-0201654bcaf195204a705f27", "cohortLabel": "XNLI German test (300-sample v1) / Hosted API / kEmbed01 client / api / memory guard", "measuredAt": "2026-11-01T02:00:31Z", "model": { "name": "gpt-6-luna", "revision": null, "precision": "API" }, "engine": { "device": "api", "backend": "OpenAI Decisions API", "hardware": "Hosted API / kEmbed01 client", "threads": null }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "Hosted API called from the stated client host: latency includes network and provider queueing. Only the successful attempt is timed; retries are counted. Provider hardware and memory are unknown." ], "version": "decision-questions-v1", "backendCohort": "OpenAI Decisions API" }, "metrics": { "tokensPerSecond": 679.7481224865537, "accuracy": { "name": "Decision accuracy", "value": 71.66666666666667 }, "ramPeakMb": null, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 281.0060477027825, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.415242 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-openai-01a1930a-api-d-openai-luna-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-openai-01a1930a-api-d-openai-luna-anli-r3", "suite": "decision", "cohort": "decision-7405522b41e1e613aa5a5499", "cohortLabel": "ANLI round 3 test (300-sample v1) / Hosted API / kEmbed01 client / api / memory guard", "measuredAt": "2026-11-01T01:59:02Z", "model": { "name": "gpt-6-luna", "revision": null, "precision": "API" }, "engine": { "device": "api", "backend": "OpenAI Decisions API", "hardware": "Hosted API / kEmbed01 client", "threads": null }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "Hosted API called from the stated client host: latency includes network and provider queueing. Only the successful attempt is timed; retries are counted. Provider hardware and memory are unknown." ], "version": "decision-questions-v1", "backendCohort": "OpenAI Decisions API" }, "metrics": { "tokensPerSecond": 810.7652467188334, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": null, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 282.9795689054299, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.46786533333333336 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-openai-01a1930a-api-d-openai-luna-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-openai-01a1930a-api-d-openai-luna-boolq", "suite": "decision", "cohort": "decision-4e652cf0b5dfd917e4be1ad9", "cohortLabel": "BoolQ validation (300-sample v1) / Hosted API / kEmbed01 client / api / memory guard", "measuredAt": "2026-11-01T01:57:31Z", "model": { "name": "gpt-6-luna", "revision": null, "precision": "API" }, "engine": { "device": "api", "backend": "OpenAI Decisions API", "hardware": "Hosted API / kEmbed01 client", "threads": null }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": null, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "Hosted API called from the stated client host: latency includes network and provider queueing. Only the successful attempt is timed; retries are counted. Provider hardware and memory are unknown." ], "version": "decision-questions-v1", "backendCohort": "OpenAI Decisions API" }, "metrics": { "tokensPerSecond": 982.2668114036189, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": null, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 297.06117517741706, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.15417999999999998 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-openai-01a1930a-api-d-openai-luna-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-scifact", "suite": "decision", "cohort": "decision-c39cdaf696cf0fa1a0585094", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-01T01:02:07Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 112.76184925374164, "accuracy": { "name": "Decision accuracy", "value": 50.333333333333336 }, "ramPeakMb": 515.8203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 4841.9153902375065, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.9102981352898517 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-decisiontune-1-0-scifact", "suite": "decision", "cohort": "decision-bd25ea0026e0d338edbb8d58", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:40:18Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 67 }, "ramPeakMb": 3804.578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 83.13399766630027, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4537703461123732 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-decisiontune-1-0-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-decisiontune-1-0-xnli-de", "suite": "decision", "cohort": "decision-ba1e3ce2b2f9e0c32f815829", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:39:33Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 3947.765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 38.84803600027226, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7736379738803444 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-decisiontune-1-0-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-decisiontune-1-0-anli-r3", "suite": "decision", "cohort": "decision-8ba6735b425d15f32cb1ad62", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:39:00Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 3801.609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 41.53448099920448, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8565021242790248 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-decisiontune-1-0-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-decisiontune-1-0-boolq", "suite": "decision", "cohort": "decision-b53f38923c8450cf1da32a50", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:38:28Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 76.66666666666667 }, "ramPeakMb": 3792.671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 41.60719933298727, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19165989559486032 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-decisiontune-1-0-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-nano-scifact", "suite": "decision", "cohort": "decision-e8f5f63d6b658c906602fffd", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:37:04Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4188.108704114515, "accuracy": { "name": "Decision accuracy", "value": 82.33333333333333 }, "ramPeakMb": 2896.3203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 134.47597466768153, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.268740069649124 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-nano-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-xnli-de", "suite": "decision", "cohort": "decision-f4248a6d7799a1b3299ce673", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-01T00:36:54Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 100.84656492211008, "accuracy": { "name": "Decision accuracy", "value": 73 }, "ramPeakMb": 693.3515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1556.8205037153286, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3592015402352225 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-nano-xnli-de", "suite": "decision", "cohort": "decision-1c61782a3369e4654a88d4ea", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:36:07Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2519.765988630489, "accuracy": { "name": "Decision accuracy", "value": 75 }, "ramPeakMb": 3079.59375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 69.15060133343407, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.35831281913647817 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-nano-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-nano-anli-r3", "suite": "decision", "cohort": "decision-8d774e752ec0f46d22cba308", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:35:31Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2834.6504952656755, "accuracy": { "name": "Decision accuracy", "value": 51.666666666666664 }, "ramPeakMb": 2893.109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 75.2920570007215, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7287469209414734 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-nano-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-nano-boolq", "suite": "decision", "cohort": "decision-e518e14e79ff062829738b2f", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:34:53Z", "model": { "name": "vllm-sr/d3-nano", "revision": "462efc366229ea268cf4eed3738b3fc4415e4817", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2972.2306863755157, "accuracy": { "name": "Decision accuracy", "value": 90.33333333333333 }, "ramPeakMb": 3061.90625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 78.45734666635205, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.06672131410850927 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-nano-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-lite-scifact", "suite": "decision", "cohort": "decision-e8f5f63d6b658c906602fffd", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:33:09Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5645.265281975731, "accuracy": { "name": "Decision accuracy", "value": 78.66666666666667 }, "ramPeakMb": 2107.4140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 99.76501933366914, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3284677969408283 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-lite-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-lite-xnli-de", "suite": "decision", "cohort": "decision-1c61782a3369e4654a88d4ea", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:32:28Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2715.546835205056, "accuracy": { "name": "Decision accuracy", "value": 69.33333333333333 }, "ramPeakMb": 2284.734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 64.16509966773447, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.38820940780414326 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-lite-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-lite-anli-r3", "suite": "decision", "cohort": "decision-8d774e752ec0f46d22cba308", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:31:57Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3100.5863419560874, "accuracy": { "name": "Decision accuracy", "value": 42.333333333333336 }, "ramPeakMb": 2076.26953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 68.83429233324325, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8341443637607698 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-lite-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-lite-boolq", "suite": "decision", "cohort": "decision-e518e14e79ff062829738b2f", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:31:25Z", "model": { "name": "vllm-sr/d3-lite", "revision": "03560c26f5199c3c04129a2ccb38960c913635e0", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3359.898873513316, "accuracy": { "name": "Decision accuracy", "value": 85.33333333333333 }, "ramPeakMb": 2128.640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 69.40486666775541, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.10626866724141233 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-lite-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-edge-scifact", "suite": "decision", "cohort": "decision-e8f5f63d6b658c906602fffd", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:30:27Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 11005.954321678119, "accuracy": { "name": "Decision accuracy", "value": 70.66666666666667 }, "ramPeakMb": 2110.9609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 51.17230033298256, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.42851001884511514 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-edge-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-edge-xnli-de", "suite": "decision", "cohort": "decision-1c61782a3369e4654a88d4ea", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:29:57Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4819.4404867127005, "accuracy": { "name": "Decision accuracy", "value": 63.666666666666664 }, "ramPeakMb": 2291.42578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 36.15426600115219, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5016319372095805 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-edge-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-edge-anli-r3", "suite": "decision", "cohort": "decision-8d774e752ec0f46d22cba308", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:29:35Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5781.930832116249, "accuracy": { "name": "Decision accuracy", "value": 40 }, "ramPeakMb": 2078.609375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 36.91269800066948, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8812367739396894 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-edge-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-gpu-d-d3-edge-boolq", "suite": "decision", "cohort": "decision-e518e14e79ff062829738b2f", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:29:14Z", "model": { "name": "vllm-sr/d3-edge", "revision": "c454ee39b862248d70abd7f84195d2f54ffe6d43", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "small-s1 PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5973.130016614436, "accuracy": { "name": "Decision accuracy", "value": 82 }, "ramPeakMb": 2154.89453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 39.04039133330419, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1311806053733443 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-gpu-d-d3-edge-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-scifact", "suite": "decision", "cohort": "decision-feccb634e8c60ac1bf84fc4f", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T00:29:09Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 67 }, "ramPeakMb": 3435.3359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1329.5069348953741, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.45377041479125924 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-anli-r3", "suite": "decision", "cohort": "decision-10be9dec41e4e80ce1053cf4", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-01T00:28:46Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 106.7480090462866, "accuracy": { "name": "Decision accuracy", "value": 57.666666666666664 }, "ramPeakMb": 500.9296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1839.4097940336603, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5921201992833629 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-xnli-de", "suite": "decision", "cohort": "decision-37877df59e90aa7929e0ec71", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T00:21:59Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 3618.3828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 346.6318326363883, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7736378116513077 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-clef-flash-scifact", "suite": "decision", "cohort": "decision-8f50c5a35362e79c685b1853", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:20:13Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 1789.694095634284, "accuracy": { "name": "Decision accuracy", "value": 85.66666666666667 }, "ramPeakMb": 7000.8359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 338.15834866768756, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.22228627013730276 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-clef-flash-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-anli-r3", "suite": "decision", "cohort": "decision-2d06a19195baa03d52512041", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T00:19:53Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 3424.68359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 399.97022892086534, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8565027013940122 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-boolq", "suite": "decision", "cohort": "decision-a6aa94d3a03388c02ef8aac6", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-11-01T00:19:14Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q5_K_M" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 108.11795799324148, "accuracy": { "name": "Decision accuracy", "value": 91 }, "ramPeakMb": 632.953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2034.0746725326364, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07184718123205808 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-q5-01a1930a-igpu2-d-h2o-lightning-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-clef-flash-xnli-de", "suite": "decision", "cohort": "decision-c5ecaa22487c0dae03b35b5e", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:18:14Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 1594.574587796573, "accuracy": { "name": "Decision accuracy", "value": 80.33333333333333 }, "ramPeakMb": 7141.68359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 137.49330699938582, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2925596194493927 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-clef-flash-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-boolq", "suite": "decision", "cohort": "decision-14b07dadcc5fc43f34da5f19", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-11-01T00:17:33Z", "model": { "name": "decision-tune/decisiontune-1.0", "revision": "fc8bed9a4f8e0f64f2769e761e036b2ae555f78d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "small-s1 PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "small-s1 PyTorch / cpu" }, "metrics": { "tokensPerSecond": null, "accuracy": { "name": "Decision accuracy", "value": 76.66666666666667 }, "ramPeakMb": 3553.4453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 442.7885132004546, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1916598466458067 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-small-01a1930a-cpu1-d-decisiontune-1-0-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-clef-flash-anli-r3", "suite": "decision", "cohort": "decision-a7ff885182fdfc0a4665be9f", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:17:21Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 1663.215088603562, "accuracy": { "name": "Decision accuracy", "value": 60.666666666666664 }, "ramPeakMb": 6960.96484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 155.43389533403874, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.565489248755419 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-clef-flash-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-clef-flash-boolq", "suite": "decision", "cohort": "decision-17d357458a3f43b899679373", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:16:23Z", "model": { "name": "bartowski/Cloudflare_clef-flash-GGUF", "revision": "5fcdd9ba6d3d5b962ad149e31457de5783e49de5", "precision": "Q4_K_M" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 1683.2902315753875, "accuracy": { "name": "Decision accuracy", "value": 92 }, "ramPeakMb": 6981.28515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 169.42612033482874, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.06539550500085135 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-clef-flash-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-scifact", "suite": "decision", "cohort": "decision-297fa9cb39c5aacb5d02fde4", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:10:51Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3375.1650486382327, "accuracy": { "name": "Decision accuracy", "value": 85.33333333333333 }, "ramPeakMb": 5531.37109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 161.76492866730163, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.22674631289399363 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-xnli-de", "suite": "decision", "cohort": "decision-eca36a6d9cfe61ff5234f656", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:09:47Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2266.8063077036886, "accuracy": { "name": "Decision accuracy", "value": 73.33333333333333 }, "ramPeakMb": 5713.16796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 69.26043900020886, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3572938235245368 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-anli-r3", "suite": "decision", "cohort": "decision-ee5eccb0f8550bfc997b90dd", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:09:17Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2548.3400281060876, "accuracy": { "name": "Decision accuracy", "value": 60.666666666666664 }, "ramPeakMb": 5523.1640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 77.0514653334006, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5815569136746794 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-boolq", "suite": "decision", "cohort": "decision-f717c11b67e5f928dacd8fd7", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 24 GiB commit cap", "measuredAt": "2026-11-01T00:08:45Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2711.760277423118, "accuracy": { "name": "Decision accuracy", "value": 91 }, "ramPeakMb": 5525.703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 81.09861400026905, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.0714146985421639 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261101-retry24g-01a1930a-gpu-d-h2o-lightning-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-scifact", "suite": "decision", "cohort": "decision-4dce3075f2f31048368c86f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T23:53:29Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.07631134258301, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 11003.12890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6372.904871079663, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2885460463333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-xnli-de", "suite": "decision", "cohort": "decision-687a2e91c91ad53438d451b9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T23:20:14Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 57.93856935445541, "accuracy": { "name": "Decision accuracy", "value": 72.66666666666667 }, "ramPeakMb": 11045, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1816.4641361694278, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.37016998879999996 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-anli-r3", "suite": "decision", "cohort": "decision-f4b9c93e3a04e6563e5a7c85", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T23:10:31Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 53.96657733997653, "accuracy": { "name": "Decision accuracy", "value": 45.666666666666664 }, "ramPeakMb": 10720.96875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2677.9537840533885, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7950413072666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-boolq", "suite": "decision", "cohort": "decision-5ae8ea48d1f9a9183341fa25", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:56:30Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 68.30027324426474, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 11097.73046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2316.1449554905994, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07455011256666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-malkuth-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-scifact", "suite": "decision", "cohort": "decision-4dce3075f2f31048368c86f0", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:40:10Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 159.79240658818262, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 6506.0078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 3073.9883733394286, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4187147084333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-xnli-de", "suite": "decision", "cohort": "decision-687a2e91c91ad53438d451b9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:24:10Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 125.69088929335962, "accuracy": { "name": "Decision accuracy", "value": 58.333333333333336 }, "ramPeakMb": 5429.84375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 837.3187104094541, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5373063677333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-anli-r3", "suite": "decision", "cohort": "decision-f4b9c93e3a04e6563e5a7c85", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:19:45Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 133.7518437889166, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 5241.11328125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1080.5084693118504, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8298923993999999 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-boolq", "suite": "decision", "cohort": "decision-5ae8ea48d1f9a9183341fa25", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:14:07Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "kev PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cpu" }, "metrics": { "tokensPerSecond": 136.40599015282152, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 5342.4296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1159.724240527139, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.13742714756666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-kev-08b-v1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-scifact", "suite": "decision", "cohort": "decision-382c42c4f094812c745686ca", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:05:25Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 31.412026746164017, "accuracy": { "name": "Decision accuracy", "value": 85.33333333333333 }, "ramPeakMb": 5776.6640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 17381.346888099408, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.22612249433108603 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-scifact", "suite": "decision", "cohort": "decision-3fab242b12f63b8974a4c57c", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T22:04:42Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 79.86630310977289, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 10630.296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6313.05044014618, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3453141796 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-xnli-de", "suite": "decision", "cohort": "decision-eb0253c2a84431186130edf9", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T21:31:50Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 70.9272782444977, "accuracy": { "name": "Decision accuracy", "value": 71 }, "ramPeakMb": 10961.515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1624.8097514199133, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4056900391666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-anli-r3", "suite": "decision", "cohort": "decision-9a5295523023fc670ea36d33", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T21:23:14Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 72.33915358133542, "accuracy": { "name": "Decision accuracy", "value": 47 }, "ramPeakMb": 10626.734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2134.759103768544, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6832482440000001 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-boolq", "suite": "decision", "cohort": "decision-79506be127f36d2ddeea18b7", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T21:11:58Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "mapika PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cpu" }, "metrics": { "tokensPerSecond": 73.1317979979326, "accuracy": { "name": "Decision accuracy", "value": 89 }, "ramPeakMb": 10824.4296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2327.2138521487554, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.11028465659999999 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-morecpu-01a1930a-cpu2-d-mapika-2b-v12-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-scifact", "suite": "decision", "cohort": "decision-573a5975072011052bb9c400", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T20:48:21Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 82.73689266455605, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": 10664.37890625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6118.189645486321, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.31952417356639806 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-xnli-de", "suite": "decision", "cohort": "decision-65898afe0bbca3ecffe0ab2c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T20:35:21Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 30.84525380268697, "accuracy": { "name": "Decision accuracy", "value": 73.66666666666667 }, "ramPeakMb": 5312.69921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5089.924077276471, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.35785697693770857 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-xnli-de", "suite": "decision", "cohort": "decision-29470aae630ccfa3fd9d4333", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T20:16:29Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 71.08284971691322, "accuracy": { "name": "Decision accuracy", "value": 69 }, "ramPeakMb": 10652.55859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1691.5941582561936, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4127984267902147 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-malkuth-2b-scifact", "suite": "decision", "cohort": "decision-8d5f8e21689e65f478d4ebd6", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:12:23Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2703.2372498957548, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 4540.83984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 181.70806133236815, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2896114005666666 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-malkuth-2b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-malkuth-2b-xnli-de", "suite": "decision", "cohort": "decision-f2661db21ba2fdfb33859716", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:11:15Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 829.8143820842474, "accuracy": { "name": "Decision accuracy", "value": 72.66666666666667 }, "ramPeakMb": 4722.5078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 126.82756000082009, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.37159808246666665 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-malkuth-2b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-malkuth-2b-anli-r3", "suite": "decision", "cohort": "decision-5ab16fb53a51409b71044b53", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:10:24Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1095.9054385944871, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 4536.6015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 131.87269166703723, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7955444024333334 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-malkuth-2b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-malkuth-2b-boolq", "suite": "decision", "cohort": "decision-ed1c5f67db1fb9a8fa477555", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:09:32Z", "model": { "name": "dhtocks/malkuth-2b", "revision": "37ae1672eebcf9ee5417418bd0be49f2e53d9087", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1206.8654107224277, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 4565.76953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 131.0778583327192, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.074489563 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-malkuth-2b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-anli-r3", "suite": "decision", "cohort": "decision-7678752a6da92f3d877b6f2e", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T20:09:07Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 33.60228718317425, "accuracy": { "name": "Decision accuracy", "value": 60 }, "ramPeakMb": 5148.0390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 5843.451437188293, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5812950472210842 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-anli-r3", "suite": "decision", "cohort": "decision-bf6f78b24fd1ba935c50eafd", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T20:07:34Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.05688124168695, "accuracy": { "name": "Decision accuracy", "value": 46.666666666666664 }, "ramPeakMb": 10449.4296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2068.9478226691913, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7503270239378498 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-kev-08b-v1-scifact", "suite": "decision", "cohort": "decision-8d5f8e21689e65f478d4ebd6", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:01:44Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3025.539731471527, "accuracy": { "name": "Decision accuracy", "value": 69 }, "ramPeakMb": 2382.95703125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 162.35119799966924, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4179575693 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-kev-08b-v1-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-kev-08b-v1-xnli-de", "suite": "decision", "cohort": "decision-f2661db21ba2fdfb33859716", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T20:00:45Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 831.2388429464753, "accuracy": { "name": "Decision accuracy", "value": 58.666666666666664 }, "ramPeakMb": 2564.5859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 126.61022066808073, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5358072506 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-kev-08b-v1-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-kev-08b-v1-anli-r3", "suite": "decision", "cohort": "decision-5ab16fb53a51409b71044b53", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:59:55Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1112.822390237268, "accuracy": { "name": "Decision accuracy", "value": 36.666666666666664 }, "ramPeakMb": 2378.484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 129.86798366735457, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.8296013231666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-kev-08b-v1-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-kev-08b-v1-boolq", "suite": "decision", "cohort": "decision-ed1c5f67db1fb9a8fa477555", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:59:05Z", "model": { "name": "jaredpalmer/kev-0.8b", "revision": "bf75a6a8848ea6960ff2ed108d9ed44c2941174f", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "kev PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 65536, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "kev PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1245.7487295651295, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 2516.76171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 126.98654999917683, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.13788171456666667 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-kev-08b-v1-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-boolq", "suite": "decision", "cohort": "decision-ca96e21b0797e1c83c4d0749", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:56:44Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 77.95301926857714, "accuracy": { "name": "Decision accuracy", "value": 90.66666666666667 }, "ramPeakMb": 10606.078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2388.5326710929317, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07518003965761152 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-2b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-mapika-2b-v12-scifact", "suite": "decision", "cohort": "decision-63b72105f6972ab860f6e9cb", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:52:15Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4056.3535908228746, "accuracy": { "name": "Decision accuracy", "value": 78.33333333333333 }, "ramPeakMb": 4556.53515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 124.29882866737898, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3448076691333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-mapika-2b-v12-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-mapika-2b-v12-xnli-de", "suite": "decision", "cohort": "decision-33d636db63623e0fc46f932c", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:51:25Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 1641.4098775509021, "accuracy": { "name": "Decision accuracy", "value": 71.33333333333333 }, "ramPeakMb": 4739.53515625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 70.20996699817867, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4057906674333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-mapika-2b-v12-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-mapika-2b-v12-anli-r3", "suite": "decision", "cohort": "decision-29f7ca49c133e111846a62a4", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:50:50Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2094.6098578249835, "accuracy": { "name": "Decision accuracy", "value": 46.666666666666664 }, "ramPeakMb": 4553.7734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 73.72574233323878, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6835370471333333 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-mapika-2b-v12-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-mapika-2b-v12-boolq", "suite": "decision", "cohort": "decision-05422722cc0ce67cb2384bf6", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:50:16Z", "model": { "name": "Mapika/decider-2b", "revision": "7614a6c7ea5a047aa7d7bcfb1a0015469381c3e3", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "mapika PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 32768, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "mapika PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2190.790512787772, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 4582.984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 77.68580899903706, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1129959677 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-mapika-2b-v12-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-scifact", "suite": "decision", "cohort": "decision-ccd01fb433ffcfb5a7acfd59", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:47:27Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "jev-style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4457.549419228681, "accuracy": { "name": "Decision accuracy", "value": 80 }, "ramPeakMb": 4460.33203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 113.56015433421514, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.31994559756065055 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-xnli-de", "suite": "decision", "cohort": "decision-0b255b49375a508072a2d9e6", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:46:21Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "jev-style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 2848.0206777032217, "accuracy": { "name": "Decision accuracy", "value": 68.66666666666667 }, "ramPeakMb": 4642.12109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 42.219965000501055, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.41250605547428765 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-anli-r3", "suite": "decision", "cohort": "decision-f33f5108944a4933864ba333", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:45:37Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "jev-style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3248.9620786503333, "accuracy": { "name": "Decision accuracy", "value": 46.666666666666664 }, "ramPeakMb": 4455.69140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 49.07002999951752, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7514286261142741 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-boolq", "suite": "decision", "cohort": "decision-673e4bc4e6b6b65a709e1d3f", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:44:50Z", "model": { "name": "chaoliangUNSW/Jev-Style-2B-Decision-v3", "revision": "5bad2d53ae04e832e2b30aa2a89bec76ec03deec", "precision": "BF16" }, "engine": { "device": "gpu", "backend": "jev-style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "jev-style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 3583.3037879054827, "accuracy": { "name": "Decision accuracy", "value": 90.33333333333333 }, "ramPeakMb": 4488.01171875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 51.96135866620656, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07531649717071083 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-more-01a1930a-gpu-d-jev-style-2b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-scifact", "suite": "decision", "cohort": "decision-573a5975072011052bb9c400", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:42:40Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 189.03977671609766, "accuracy": { "name": "Decision accuracy", "value": 72.33333333333333 }, "ramPeakMb": 4838.51953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2899.918787056619, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.39118776354021795 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-boolq", "suite": "decision", "cohort": "decision-895d52e541adcd8d3736ee67", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:38:58Z", "model": { "name": "h2oai/h2o-lightning-4b-gguf", "revision": "aba20a244eb1038b8546cd25c2d6046f0c3e4256", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 31.806212784695266, "accuracy": { "name": "Decision accuracy", "value": 91.33333333333333 }, "ramPeakMb": 5189.66015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 6914.372405438432, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07088764752990828 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-expand-01a1930a-cpu1-d-h2o-lightning-4b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-scifact", "suite": "decision", "cohort": "decision-473a55f0363c281523429d58", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:30:47Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP16" }, "engine": { "device": "gpu", "backend": "Jev-Style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 7079.5729056930695, "accuracy": { "name": "Decision accuracy", "value": 72.33333333333333 }, "ramPeakMb": 4209.8671875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 77.43404966691742, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.39119484978490077 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-xnli-de", "suite": "decision", "cohort": "decision-f530d1d7f160bc0275be8219", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:29:59Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP16" }, "engine": { "device": "gpu", "backend": "Jev-Style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4287.279745750256, "accuracy": { "name": "Decision accuracy", "value": 71.66666666666667 }, "ramPeakMb": 4388.55859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 39.475691666981824, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3853929772361377 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-anli-r3", "suite": "decision", "cohort": "decision-f165a1c1578802a1a0505f0c", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:29:24Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP16" }, "engine": { "device": "gpu", "backend": "Jev-Style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 5169.912846876636, "accuracy": { "name": "Decision accuracy", "value": 34 }, "ramPeakMb": 4200.1015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 40.31531533313682, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.9054135688010945 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-boolq", "suite": "decision", "cohort": "decision-cbfac18e5d1321237c3faebf", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T19:28:50Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP16" }, "engine": { "device": "gpu", "backend": "Jev-Style PyTorch / cuda:0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cuda:0" }, "metrics": { "tokensPerSecond": 4985.199182035977, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 4246.5625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 41.96288366714725, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.12444894732543385 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-r1-01a1930a-gpu-d-jev-style-08b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-xnli-de", "suite": "decision", "cohort": "decision-29470aae630ccfa3fd9d4333", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:27:33Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 197.10127549940026, "accuracy": { "name": "Decision accuracy", "value": 71.66666666666667 }, "ramPeakMb": 5021.578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 858.6617864573296, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.3853451190393599 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-anli-r3", "suite": "decision", "cohort": "decision-bf6f78b24fd1ba935c50eafd", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:23:01Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 195.68959326392593, "accuracy": { "name": "Decision accuracy", "value": 34 }, "ramPeakMb": 4828.4921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1065.0881489929936, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.905443198743802 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-boolq", "suite": "decision", "cohort": "decision-ca96e21b0797e1c83c4d0749", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T19:17:26Z", "model": { "name": "chaoliangUNSW/Jev-Style-0.8B-Decision-v3", "revision": "b023d1f9c7858fbf01504577a3bfc349ea5c7385", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "Jev-Style PyTorch / cpu", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 25600, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "Jev-Style PyTorch / cpu" }, "metrics": { "tokensPerSecond": 196.92111364997012, "accuracy": { "name": "Decision accuracy", "value": 80.66666666666667 }, "ramPeakMb": 4967.640625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1062.320486899019, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1244416410152801 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-native-01a1930a-cpu2-d-jev-style-08b-v3-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-d1-omni-600m-scifact", "suite": "decision", "cohort": "decision-7b9a92431f5ed927f8d0cc82", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:39:08Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1048.990918207862, "accuracy": { "name": "Decision accuracy", "value": 64.33333333333333 }, "ramPeakMb": 210.29296875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 459.36209580337163, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4944235798497427 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-d1-omni-600m-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-d1-omni-600m-xnli-de", "suite": "decision", "cohort": "decision-f0a8ac0aca3a20bef9af08e5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:36:44Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1091.0959658642644, "accuracy": { "name": "Decision accuracy", "value": 55.333333333333336 }, "ramPeakMb": 350.796875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 101.22543765358084, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6387099993079756 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-d1-omni-600m-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-d1-omni-600m-anli-r3", "suite": "decision", "cohort": "decision-f64412c86afc224b8f2ed7de", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:36:10Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1128.2777056550317, "accuracy": { "name": "Decision accuracy", "value": 34.666666666666664 }, "ramPeakMb": 159.7109375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 130.58841742730388, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0300921593241008 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-d1-omni-600m-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-d1-omni-600m-boolq", "suite": "decision", "cohort": "decision-722ef262c6a65e6cfe85e365", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:35:28Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 1143.4557081482874, "accuracy": { "name": "Decision accuracy", "value": 76.33333333333333 }, "ramPeakMb": 606.7578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 152.9748802279937, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.184505528188136 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-d1-omni-600m-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-d1-omni-600m-scifact", "suite": "decision", "cohort": "decision-1327b9c352250228116d5264", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T18:34:37Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 270.09832679410204, "accuracy": { "name": "Decision accuracy", "value": 64.66666666666667 }, "ramPeakMb": 1027.48046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 1784.0416576663847, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.4923046385949579 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-d1-omni-600m-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-d1-omni-600m-xnli-de", "suite": "decision", "cohort": "decision-f0003c56384949df6911b554", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T18:25:22Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 283.78556950140376, "accuracy": { "name": "Decision accuracy", "value": 54.666666666666664 }, "ramPeakMb": 806.26953125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 389.1905668801822, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6375925688932222 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-d1-omni-600m-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-d1-omni-600m-anli-r3", "suite": "decision", "cohort": "decision-2cc5b42c7579b732cf2ae7ca", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T18:23:20Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 290.3909558622093, "accuracy": { "name": "Decision accuracy", "value": 35.333333333333336 }, "ramPeakMb": 586.55859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 507.38494786288356, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0331672272391865 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-d1-omni-600m-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-d1-omni-600m-boolq", "suite": "decision", "cohort": "decision-a8558176b0af966ecc6734ec", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T18:20:42Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 282.1352243421633, "accuracy": { "name": "Decision accuracy", "value": 76.33333333333333 }, "ramPeakMb": 651.2578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 619.9863927230277, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.18405751607680199 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-d1-omni-600m-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-laya-scifact", "suite": "decision", "cohort": "decision-7b9a92431f5ed927f8d0cc82", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:17:18Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 859.2042049134972, "accuracy": { "name": "Decision accuracy", "value": 53.333333333333336 }, "ramPeakMb": 177.1015625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 536.5197206482602, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6190891432564043 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-laya-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-r1-igpu1-d-d1-3b-scifact", "suite": "decision", "cohort": "decision-9b59c552591b80bfd63de757", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:14:43Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 231.0535436757142, "accuracy": { "name": "Decision accuracy", "value": 83.33333333333333 }, "ramPeakMb": 206.7734375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2063.2014225631915, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2618458034646269 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-r1-igpu1-d-d1-3b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-laya-xnli-de", "suite": "decision", "cohort": "decision-f0a8ac0aca3a20bef9af08e5", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:14:30Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 811.8052759417164, "accuracy": { "name": "Decision accuracy", "value": 44 }, "ramPeakMb": 352.33203125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 146.6443618866227, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6721993852728109 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-laya-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-laya-anli-r3", "suite": "decision", "cohort": "decision-f64412c86afc224b8f2ed7de", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:13:42Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 837.6778567806545, "accuracy": { "name": "Decision accuracy", "value": 36 }, "ramPeakMb": 160.9140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 169.5719488307562, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7943077990921543 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-laya-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-igpu2-d-laya-boolq", "suite": "decision", "cohort": "decision-722ef262c6a65e6cfe85e365", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:12:49Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 865.3159071428533, "accuracy": { "name": "Decision accuracy", "value": 72.33333333333333 }, "ramPeakMb": 198.67578125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 197.41537773263795, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19669982894822452 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-igpu2-d-laya-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-laya-scifact", "suite": "decision", "cohort": "decision-1327b9c352250228116d5264", "cohortLabel": "SciFact claims dev (all 300, v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T18:11:46Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 189.92540495745624, "accuracy": { "name": "Decision accuracy", "value": 53.666666666666664 }, "ramPeakMb": 921.546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 2427.163444001926, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6196434088331283 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-laya-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-r1-igpu1-d-d1-3b-xnli-de", "suite": "decision", "cohort": "decision-493e706481bf0ffd47458b7c", "cohortLabel": "XNLI German test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:03:58Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 194.87245347706306, "accuracy": { "name": "Decision accuracy", "value": 73.33333333333333 }, "ramPeakMb": 383.05859375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 571.9125408000158, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.33627332390575543 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-r1-igpu1-d-d1-3b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-r1-igpu1-d-d1-3b-anli-r3", "suite": "decision", "cohort": "decision-d1daa55fe20ae9d2a8848c31", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T18:00:55Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 204.39414651695824, "accuracy": { "name": "Decision accuracy", "value": 56 }, "ramPeakMb": 191.359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 731.3484064032673, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5749838254045747 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-r1-igpu1-d-d1-3b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-laya-xnli-de", "suite": "decision", "cohort": "decision-f0003c56384949df6911b554", "cohortLabel": "XNLI German test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T17:59:11Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 222.14335583811211, "accuracy": { "name": "Decision accuracy", "value": 45 }, "ramPeakMb": 791.94140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 535.9001902961367, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.672411245098413 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-laya-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-r1-igpu1-d-d1-3b-boolq", "suite": "decision", "cohort": "decision-36c05b72ba7b7ffdc2fd246a", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-31T17:57:05Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan0", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / Vulkan0" }, "metrics": { "tokensPerSecond": 211.7817717493657, "accuracy": { "name": "Decision accuracy", "value": 89 }, "ramPeakMb": 647.48828125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 784.1090318033821, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.0759964236609799 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-r1-igpu1-d-d1-3b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-laya-anli-r3", "suite": "decision", "cohort": "decision-2cc5b42c7579b732cf2ae7ca", "cohortLabel": "ANLI round 3 test (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T17:56:23Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 221.96163601602436, "accuracy": { "name": "Decision accuracy", "value": 36.333333333333336 }, "ramPeakMb": 607.35546875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 639.9604418865056, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7945735184813871 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-laya-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-3b-scifact", "suite": "decision", "cohort": "decision-ace8c39b7daa6cfc0c287161", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:56:15Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5329.891902662669, "accuracy": { "name": "Decision accuracy", "value": 83.33333333333333 }, "ramPeakMb": 3636.00390625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 89.4408383332969, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.2615395449021141 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-3b-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-3b-xnli-de", "suite": "decision", "cohort": "decision-68883e30e28efa6796280af3", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:55:42Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 2801.7082149233925, "accuracy": { "name": "Decision accuracy", "value": 74 }, "ramPeakMb": 3815.078125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 39.779303000347376, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.33761084142190956 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-3b-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-3b-anli-r3", "suite": "decision", "cohort": "decision-9d9869a353ecd9d812310e1d", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:55:24Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3419.946689492307, "accuracy": { "name": "Decision accuracy", "value": 56 }, "ramPeakMb": 3629.69140625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 43.70925833218886, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.5748015159770271 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-3b-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-3b-boolq", "suite": "decision", "cohort": "decision-bfa343af8f71b88a3d6daf1a", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:55:05Z", "model": { "name": "LiquidAI/d1-3B-GGUF", "revision": "bb1e436ea78eb96a3f1acb6da865f70c2fbeb563", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 3613.790709353203, "accuracy": { "name": "Decision accuracy", "value": 88.66666666666667 }, "ramPeakMb": 3634.98046875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 45.951747999739986, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.07665394497918686 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-3b-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-omni-600m-scifact", "suite": "decision", "cohort": "decision-ace8c39b7daa6cfc0c287161", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:53:29Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 14069.866855607224, "accuracy": { "name": "Decision accuracy", "value": 63.666666666666664 }, "ramPeakMb": 1220.89453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 34.24813266620428, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.49545020382322963 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-omni-600m-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-omni-600m-xnli-de", "suite": "decision", "cohort": "decision-68883e30e28efa6796280af3", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:53:16Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 4234.83311405553, "accuracy": { "name": "Decision accuracy", "value": 54.333333333333336 }, "ramPeakMb": 1383.34765625, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 26.08052399989295, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.6380014847856319 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-omni-600m-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-omni-600m-anli-r3", "suite": "decision", "cohort": "decision-9d9869a353ecd9d812310e1d", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:53:06Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5752.125102400811, "accuracy": { "name": "Decision accuracy", "value": 34.333333333333336 }, "ramPeakMb": 1204.33984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 25.614881000850193, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 1.0321210283384596 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-omni-600m-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-cpu2-d-laya-boolq", "suite": "decision", "cohort": "decision-a8558176b0af966ecc6734ec", "cohortLabel": "BoolQ validation (300-sample v1) / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-31T17:53:05Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "cpu", "backend": "llama.cpp / CPU", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CPU" }, "metrics": { "tokensPerSecond": 227.6436959358707, "accuracy": { "name": "Decision accuracy", "value": 72.66666666666667 }, "ramPeakMb": 638.359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 750.4124634964197, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.19527640042563918 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-cpu2-d-laya-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-d1-omni-600m-boolq", "suite": "decision", "cohort": "decision-bfa343af8f71b88a3d6daf1a", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:52:56Z", "model": { "name": "LiquidAI/d1-omni-600M-GGUF", "revision": "8eb2d2a1c5404bba8a34a88bd8e159c242197d51", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 6948.890733945902, "accuracy": { "name": "Decision accuracy", "value": 76 }, "ramPeakMb": 1214.8984375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 25.172362999680143, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.18531829070508446 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-d1-omni-600m-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-laya-scifact", "suite": "decision", "cohort": "decision-ace8c39b7daa6cfc0c287161", "cohortLabel": "SciFact claims dev (all 300, v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:52:35Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "SciFact claims dev (all 300, v1)", "revision": "de83983728d6ec97f0f2ac4d03caa6b7774a2393ee639d33cbbd81aea478a0e3", "samples": 300, "languages": [ "en" ], "public": true, "license": "Claims CC BY 4.0; abstracts ODC-By 1.0", "attribution": "Wadden et al., SciFact (EMNLP 2020); abstracts from S2ORC" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 9661.23339937062, "accuracy": { "name": "Decision accuracy", "value": 53.333333333333336 }, "ramPeakMb": 1392.9453125, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 47.71440466700975, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.619866225931372 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-laya-scifact.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-laya-xnli-de", "suite": "decision", "cohort": "decision-68883e30e28efa6796280af3", "cohortLabel": "XNLI German test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:52:19Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "XNLI German test (300-sample v1)", "revision": "c05b8d223e4d90898b949e1467d7177148d63466f76098a47d66bee6e7c42ef4", "samples": 300, "languages": [ "de" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Conneau et al., XNLI (EMNLP 2018)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 4283.004903793416, "accuracy": { "name": "Decision accuracy", "value": 44.333333333333336 }, "ramPeakMb": 1559.93359375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.79512733250158, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.671676681132561 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-laya-xnli-de.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-laya-anli-r3", "suite": "decision", "cohort": "decision-9d9869a353ecd9d812310e1d", "cohortLabel": "ANLI round 3 test (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:52:08Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "ANLI round 3 test (300-sample v1)", "revision": "fc5d3fe408579f2d50b5fa16f28d7ebbd9d1738ebf06bcf49bc53da50e10fcc5", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-NC 4.0", "attribution": "Nie et al., Adversarial NLI (ACL 2020)" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 5121.249493575272, "accuracy": { "name": "Decision accuracy", "value": 35.666666666666664 }, "ramPeakMb": 1384.484375, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 27.736720666483354, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.7951775964516866 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-laya-anli-r3.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "r20261031-01a1930a-gpu-d-laya-boolq", "suite": "decision", "cohort": "decision-bfa343af8f71b88a3d6daf1a", "cohortLabel": "BoolQ validation (300-sample v1) / NVIDIA GeForce RTX 5050 GPU / cuda:0 / 12 GiB commit cap", "measuredAt": "2026-10-31T17:51:57Z", "model": { "name": "ggml-org/Laya-GGUF", "revision": "845f314dd047fc2444a5289a4d50c1e1d59e27e1", "precision": "Q8_0" }, "engine": { "device": "gpu", "backend": "llama.cpp / CUDA0", "hardware": "NVIDIA GeForce RTX 5050 GPU", "threads": 4 }, "dataset": { "name": "BoolQ validation (300-sample v1)", "revision": "28a463bf7890c8a6cba77e117b8457122af6b2983e71df9ecab47945b3e7c2f6", "samples": 300, "languages": [ "en" ], "public": true, "license": "CC BY-SA 3.0", "attribution": "Clark et al., BoolQ (NAACL 2019), SuperGLUE v2 distribution" }, "protocol": { "warmups": 8, "repeats": 1, "batchSize": 1, "maxTokens": 8192, "timing": "Wall time of one successful HTTP request including JSON serialization, server-side inference and response parsing; excludes loading, warmups, retries and scoring.", "notes": [ "One typed question per request, sent one request at a time. Question wording, options and sampling are pinned by decision-questions-v1.", "Decision accuracy: argmax option (noul: probability >= 0.5) equals the gold label, over all requests; rejected or refused requests count as wrong. Brier score is over answered requests only; choice Brier sums over all options.", "Latency is the mean measured request; median and p95 are in the evidence.", "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown." ], "version": "decision-questions-v1", "backendCohort": "llama.cpp / CUDA0" }, "metrics": { "tokensPerSecond": 6333.007575426198, "accuracy": { "name": "Decision accuracy", "value": 71.66666666666667 }, "ramPeakMb": 1394.19921875, "vramPeakMb": null, "vectorsPerSecond": null, "dimensions": null, "latencyMs": 26.974018999996286, "pagesPerSecond": null, "cerPercent": null, "werPercent": null, "brierScore": 0.1967864144936428 }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "decision/evidence/r20261031-01a1930a-gpu-d-laya-boolq.json" }, "retrievalMetricsPercent": null }, { "schemaVersion": 1, "id": "rd130878c3d3446df-igpu1-e-jina-syn-512-pfx", "suite": "embedding", "cohort": "embedding-82d86d3e47570d232c0bdb0c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-30T00:55:40Z", "model": { "name": "jinaai/jina-code-embeddings-0.5b-GGUF", "revision": "941797c2653f7f0425a06a860d5035f77a685731", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 543.0061712502709, "accuracy": { "name": "nDCG@10", "value": 93.05423705548237 }, "ramPeakMb": 196.3125, "vramPeakMb": null, "vectorsPerSecond": 13.110812987934105, "dimensions": 896, "latencyMs": 305.0916830009858, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rd130878c3d3446df-igpu1-e-jina-syn-512-pfx.json" }, "retrievalMetricsPercent": { "nDCG@10": 93.05423705548237, "R@10": 100, "RR@10": 95.83333333333334, "P@1": 91.66666666666666 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu1-e-bge-syn-512", "suite": "embedding", "cohort": "embedding-82d86d3e47570d232c0bdb0c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-29T18:23:03Z", "model": { "name": "BAAI/bge-code-v1", "revision": "bd67852057c5d7ddcc7b8234d9d6c410117ed851", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server.", "Locally converted GGUF, not an upstream-published artifact; converter 412a363c655b479279d3050d0d1c67a5733697b4; artifact SHA-256 e58a59517c2957b3333240645a7796e81aa58ce93eb21d75ea426460c30b0789.", "BGE-Code retains its upstream custom tokenizer's required EOS token 151643; the adapter appends it when the derived GGUF tokenizer omits automatic EOS." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 187.97502747414265, "accuracy": { "name": "nDCG@10", "value": 97.07329892423125 }, "ramPeakMb": 199.15234375, "vramPeakMb": null, "vectorsPerSecond": 4.280266280246132, "dimensions": 1536, "latencyMs": 934.5212979997086, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu1-e-bge-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.07329892423125, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu1-e-qwen-syn-512-pfx", "suite": "embedding", "cohort": "embedding-82d86d3e47570d232c0bdb0c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-29T16:40:04Z", "model": { "name": "Qwen/Qwen3-Embedding-0.6B-GGUF", "revision": "370f27d7550e0def9b39c1f16d3fbaa13aa67728", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 507.5802796371459, "accuracy": { "name": "nDCG@10", "value": 100 }, "ramPeakMb": 202.0234375, "vramPeakMb": null, "vectorsPerSecond": 11.186342250956384, "dimensions": 1024, "latencyMs": 357.57890383320046, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu1-e-qwen-syn-512-pfx.json" }, "retrievalMetricsPercent": { "nDCG@10": 100, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu2-e-m3-syn-512", "suite": "embedding", "cohort": "embedding-e6543bac240b1aa86cbc852a", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node2 / openvino:gpu.0 / memory guard", "measuredAt": "2026-10-29T15:26:55Z", "model": { "name": "BAAI/bge-m3", "revision": "5617a9f61b028005a4858fdac845db406aefb181", "precision": "FP32" }, "engine": { "device": "igpu", "backend": "OpenVINO / explicit Intel GPU / upstream ONNX dense", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm tokenizer, synchronous GPU inference, dense pooling and normalization; excludes download/load, formatting, count-only pass, scoring and warmups.", "notes": [ "Separate OpenVINO ONNX cohort, not Torch or GGUF latency peers.", "CPU attention-mask mean pooling for E5; dense CLS for BGE-M3. BGE sparse and ColBERT outputs are not implemented.", "Counts are actual pinned HF tokenizer attention-mask sums after truncation including prompts and special tokens, excluding batch padding.", "RAM is process-lifetime peak RSS including load; shared GPU memory unknown.", "Precision is the verified compiled-model hint, not proof of every operator dtype. Parity against the pinned reference remains a separate host preflight gate." ], "backendCohort": "OpenVINO / explicit Intel GPU / upstream ONNX dense", "dimensions": "native", "pooling": "upstream-model-native-dense", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 829.1786946703628, "accuracy": { "name": "nDCG@10", "value": 96.70454521417923 }, "ramPeakMb": 3869.05078125, "vramPeakMb": null, "vectorsPerSecond": 25.06333585905379, "dimensions": 1024, "latencyMs": 159.59567483332648, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu2-e-m3-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 96.70454521417923, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu1-e-gemma-syn-512-pfx", "suite": "embedding", "cohort": "embedding-82d86d3e47570d232c0bdb0c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0 / memory guard", "measuredAt": "2026-10-29T14:38:58Z", "model": { "name": "ggml-org/embeddinggemma-2-GGUF", "revision": "58faf90940886df2d62be555c8765d00dac362c5", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 714.4388298367276, "accuracy": { "name": "nDCG@10", "value": 97.68686855867337 }, "ramPeakMb": 256.578125, "vramPeakMb": null, "vectorsPerSecond": 18.536791260628608, "dimensions": 768, "latencyMs": 215.78707683329412, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu1-e-gemma-syn-512-pfx.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.68686855867337, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu2-e-voyage-syn-512-pfx", "suite": "embedding", "cohort": "embedding-3efa2504288addab61e4f5cd", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-29T14:34:19Z", "model": { "name": "MongoDB/voyage-4-nano-GGUF", "revision": "03547551689253732502486d62de4cd0970d6564", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 980.2710147853423, "accuracy": { "name": "nDCG@10", "value": 99.01767665550587 }, "ramPeakMb": 190.375, "vramPeakMb": null, "vectorsPerSecond": 23.432773261801014, "dimensions": 2048, "latencyMs": 170.70109266668018, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu2-e-voyage-syn-512-pfx.json" }, "retrievalMetricsPercent": { "nDCG@10": 99.01767665550587, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "rffb2d0bb736243b1-igpu1-e-e5-syn-512", "suite": "embedding", "cohort": "embedding-659516d3c9c4c59753e99237", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / openvino:gpu.0 / memory guard", "measuredAt": "2026-10-29T14:34:12Z", "model": { "name": "intfloat/multilingual-e5-small", "revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", "precision": "FP16" }, "engine": { "device": "igpu", "backend": "OpenVINO / explicit Intel GPU / upstream ONNX dense", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm tokenizer, synchronous GPU inference, dense pooling and normalization; excludes download/load, formatting, count-only pass, scoring and warmups.", "notes": [ "Separate OpenVINO ONNX cohort, not Torch or GGUF latency peers.", "CPU attention-mask mean pooling for E5; dense CLS for BGE-M3. BGE sparse and ColBERT outputs are not implemented.", "Counts are actual pinned HF tokenizer attention-mask sums after truncation including prompts and special tokens, excluding batch padding.", "RAM is process-lifetime peak RSS including load; shared GPU memory unknown.", "Precision is the verified compiled-model hint, not proof of every operator dtype. Parity against the pinned reference remains a separate host preflight gate." ], "backendCohort": "OpenVINO / explicit Intel GPU / upstream ONNX dense", "dimensions": "native", "pooling": "upstream-model-native-dense", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 12225.971981551773, "accuracy": { "name": "nDCG@10", "value": 97.2650999438097 }, "ramPeakMb": 1584.53125, "vramPeakMb": null, "vectorsPerSecond": 343.5870346103543, "dimensions": 384, "latencyMs": 11.64188283337353, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rffb2d0bb736243b1-igpu1-e-e5-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.2650999438097, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r8f42bbbe50e94347-cpu1-r7664b33260f94780-cpu1-e-bge-syn-512", "suite": "embedding", "cohort": "embedding-25158caa50d81ed261757e94", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-28T13:47:28Z", "model": { "name": "BAAI/bge-code-v1", "revision": "bd67852057c5d7ddcc7b8234d9d6c410117ed851", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 1, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 85.87900755213113, "accuracy": { "name": "nDCG@10", "value": 97.07329892423125 }, "ramPeakMb": 6311.328125, "vramPeakMb": null, "vectorsPerSecond": 1.9554992231984318, "dimensions": 1536, "latencyMs": 511.3783672920059, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r8f42bbbe50e94347-cpu1-r7664b33260f94780-cpu1-e-bge-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.07329892423125, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r7664b33260f94780-cpu2-e-jina-syn-512", "suite": "embedding", "cohort": "embedding-7265b2535afd8cb94748beaf", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-28T04:07:20Z", "model": { "name": "jinaai/jina-code-embeddings-0.5b", "revision": "4db235132dafbe56a8b9c5f59b59795ecf58a4a7", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 340.80198798460367, "accuracy": { "name": "nDCG@10", "value": 92.70575230314401 }, "ramPeakMb": 3267.40234375, "vramPeakMb": null, "vectorsPerSecond": 8.228619428199686, "dimensions": 896, "latencyMs": 486.10827550146496, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r7664b33260f94780-cpu2-e-jina-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 92.70575230314401, "R@10": 100, "RR@10": 95.83333333333334, "P@1": 91.66666666666666 } }, { "schemaVersion": 1, "id": "rc71de2e680b24561-cpu1-e-gemma-syn-512", "suite": "embedding", "cohort": "embedding-ed2b45b4ed504080ea2f838c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-27T23:33:20Z", "model": { "name": "google/embeddinggemma-2", "revision": "914f7f89142e33e77833254d9c9b90c3cef7303b", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 918.3572000257338, "accuracy": { "name": "nDCG@10", "value": 97.68686855867337 }, "ramPeakMb": 1993.18359375, "vramPeakMb": null, "vectorsPerSecond": 23.82764627093796, "dimensions": 768, "latencyMs": 167.87222516722977, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/rc71de2e680b24561-cpu1-e-gemma-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.68686855867337, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r7664b33260f94780-cpu2-e-m3-syn-512", "suite": "embedding", "cohort": "embedding-7265b2535afd8cb94748beaf", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-27T20:14:15Z", "model": { "name": "BAAI/bge-m3", "revision": "5617a9f61b028005a4858fdac845db406aefb181", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 284.648979379663, "accuracy": { "name": "nDCG@10", "value": 96.70454521417923 }, "ramPeakMb": 2016.33984375, "vramPeakMb": null, "vectorsPerSecond": 8.603999376715254, "dimensions": 1024, "latencyMs": 464.9000801679601, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r7664b33260f94780-cpu2-e-m3-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 96.70454521417923, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r7664b33260f94780-cpu2-e-voyage-syn-512", "suite": "embedding", "cohort": "embedding-7265b2535afd8cb94748beaf", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-27T15:32:21Z", "model": { "name": "voyageai/voyage-4-nano", "revision": "433ece5e27a1611e9360dd7134ecaaf79fb3681d", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 536.1146222293681, "accuracy": { "name": "nDCG@10", "value": 99.01767665550587 }, "ramPeakMb": 2424.45703125, "vramPeakMb": null, "vectorsPerSecond": 12.815488977594457, "dimensions": 2048, "latencyMs": 312.1223081689095, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r7664b33260f94780-cpu2-e-voyage-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 99.01767665550587, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r7664b33260f94780-cpu1-e-qwen3-syn-512", "suite": "embedding", "cohort": "embedding-ed2b45b4ed504080ea2f838c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-27T15:10:17Z", "model": { "name": "Qwen/Qwen3-Embedding-0.6B", "revision": "97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed01 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 306.6052960409048, "accuracy": { "name": "nDCG@10", "value": 100 }, "ramPeakMb": 3851.72265625, "vramPeakMb": null, "vectorsPerSecond": 6.757141510543356, "dimensions": 1024, "latencyMs": 591.9662913317248, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r7664b33260f94780-cpu1-e-qwen3-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 100, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r7664b33260f94780-cpu2-e-e5-syn-512", "suite": "embedding", "cohort": "embedding-7265b2535afd8cb94748beaf", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU) / cpu / memory guard", "measuredAt": "2026-10-27T15:10:16Z", "model": { "name": "intfloat/multilingual-e5-small", "revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", "precision": "FP32" }, "engine": { "device": "cpu", "backend": "SentenceTransformers / torch", "hardware": "Intel Core i5-12600H CPU / kEmbed02 VM (4 vCPU)", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [ "de", "en", "pl" ], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batched encode including tokenization, inference, normalization and CPU transfer; excludes dataset preparation/formatting, load, token counting, scoring and warmups; CUDA synchronized", "notes": [ "local-batch-warmup-v2: fixed evenly spaced ID-sorted warmup batches; full corpus/query repeats; native inference dimensions; not comparable to full-pass warmups or historical single-index/single-query timings.", "Synthetic 12-document/12-query smoke-quality corpus, not a representative leaderboard; same-language grade 2, translations 1.", "nDCG@10: query macro mean then repeat mean.", "latencyMs is mean batch latency; TPS counts truncated input tokens including prompts and special tokens, excluding padding.", "RAM is process-lifetime peak RSS including loading; VRAM is torch peak allocated after warmups, not total device use." ], "version": "local-batch-warmup-v2", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 2062.067512638194, "accuracy": { "name": "nDCG@10", "value": 97.2650999438097 }, "ramPeakMb": 929.7421875, "vramPeakMb": null, "vectorsPerSecond": 57.950375062431675, "dimensions": 384, "latencyMs": 69.02457483132214, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r7664b33260f94780-cpu2-e-e5-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.2650999438097, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r9974465f7ebb4a68-igpu2-e-jina-syn-512", "suite": "embedding", "cohort": "embedding-8224a895776504b6cf401ab7", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node2 / vulkan:0 / memory guard", "measuredAt": "2026-10-27T09:12:17Z", "model": { "name": "jinaai/jina-code-embeddings-0.5b-GGUF", "revision": "941797c2653f7f0425a06a860d5035f77a685731", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "version": "gguf-native-context-headroom-v1", "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 541.0902251231367, "accuracy": { "name": "nDCG@10", "value": 93.05423705548237 }, "ramPeakMb": 194.3203125, "vramPeakMb": null, "vectorsPerSecond": 13.064552719270909, "dimensions": 896, "latencyMs": 306.1719819997961, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r9974465f7ebb4a68-igpu2-e-jina-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 93.05423705548237, "R@10": 100, "RR@10": 95.83333333333334, "P@1": 91.66666666666666 } }, { "schemaVersion": 1, "id": "r20261026-01a17710-igpu2-e-qwen-syn-512", "suite": "embedding", "cohort": "embedding-5a5db12f8ce026b271ff86f6", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node2 / vulkan:0", "measuredAt": "2026-10-26T12:40:28Z", "model": { "name": "Qwen/Qwen3-Embedding-0.6B-GGUF", "revision": "370f27d7550e0def9b39c1f16d3fbaa13aa67728", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 560.0881329157485, "accuracy": { "name": "nDCG@10", "value": 100 }, "ramPeakMb": 202.0625, "vramPeakMb": null, "vectorsPerSecond": 12.343540119355337, "dimensions": 1024, "latencyMs": 324.05614283440326, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r20261026-01a17710-igpu2-e-qwen-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 100, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r20261026-01a17710-igpu1-e-voyage-syn-512", "suite": "embedding", "cohort": "embedding-a7902dbb6f74be839c57fc4c", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node1 / vulkan:0", "measuredAt": "2026-10-26T11:43:24Z", "model": { "name": "MongoDB/voyage-4-nano-GGUF", "revision": "03547551689253732502486d62de4cd0970d6564", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node1", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 966.0022000752373, "accuracy": { "name": "nDCG@10", "value": 99.01767665550587 }, "ramPeakMb": 330.26953125, "vramPeakMb": null, "vectorsPerSecond": 23.091686057575394, "dimensions": 2048, "latencyMs": 173.222517837227, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r20261026-01a17710-igpu1-e-voyage-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 99.01767665550587, "R@10": 100, "RR@10": 100, "P@1": 100 } }, { "schemaVersion": 1, "id": "r20261026-01a17710-igpu2-e-gemma-syn-512", "suite": "embedding", "cohort": "embedding-5a5db12f8ce026b271ff86f6", "cohortLabel": "Synthetic EN-DE-PL retrieval v1 / Intel Iris Xe ADL GT2 node2 / vulkan:0", "measuredAt": "2026-10-26T09:28:03Z", "model": { "name": "ggml-org/embeddinggemma-2-GGUF", "revision": "58faf90940886df2d62be555c8765d00dac362c5", "precision": "Q8_0" }, "engine": { "device": "igpu", "backend": "llama.cpp / Vulkan / GGUF native", "hardware": "Intel Iris Xe ADL GT2 node2", "threads": 4 }, "dataset": { "name": "Synthetic EN-DE-PL retrieval v1", "revision": "ad19a1bb59a9753f15edd7ef868c4704da7b488b9c6d1877e5a9c3cf427ff882", "samples": 24, "languages": [], "public": true }, "protocol": { "warmups": 1, "repeats": 1, "batchSize": 4, "maxTokens": 512, "timing": "Warm batch HTTP including exact /tokenize, client token truncation, server inference and normalization, JSON serialization and CPU vector transfer; excludes load, dataset formatting, count-only pass, scoring and warmups.", "notes": [ "RAM is the sum of independent client and owned-server process-lifetime peak RSS (not a simultaneous peak; shared pages may be counted twice). Server Linux VmHWM is sampled from startup at 10 ms; Windows uses peak_wset. GPU shared-memory peak unknown.", "Native GGUF quality/latency cohort; never Torch latency peers.", "Counts are exact submitted token IDs, checked against server usage; automatic BOS/EOS retained, no estimated tokens or chat template.", "Full native output dimensions, L2 normalized by llama-server." ], "backendCohort": "llama.cpp / Vulkan / GGUF native", "dimensions": "native", "pooling": "upstream-model-native", "warmupBatches": 2 }, "metrics": { "tokensPerSecond": 734.5671484712693, "accuracy": { "name": "nDCG@10", "value": 97.68686855867337 }, "ramPeakMb": 255.46484375, "vramPeakMb": null, "vectorsPerSecond": 19.059039527903206, "dimensions": 768, "latencyMs": 209.87416465262262, "pagesPerSecond": null, "cerPercent": null, "werPercent": null }, "source": { "kind": "measured", "label": "Sanitized runner evidence", "path": "embedding/evidence/r20261026-01a17710-igpu2-e-gemma-syn-512.json" }, "retrievalMetricsPercent": { "nDCG@10": 97.68686855867337, "R@10": 100, "RR@10": 100, "P@1": 100 } } ] }