piq36v2_numeric_grid_offsets

Prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Gold / expected16Evaluatornormalized_exactTagslogic_grid, hardMetadata

—

Response contract

{
  "instruction": "Return only the final answer. Do not include explanation.",
  "mode": "short_answer"
}

Provenance

—

ModalityNot specified

Wrong-answer distribution · literal raw responses

20 — 2 models29 — 2 models19 — 1 model23 — 1 model

Every model response

gpt-5.6 gpt-5.6 ✓ PASS 16

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "16",
  "expected_normalized": "16"
}

Latency

6025.408850007807 ms

Usage

{
  "completion_tokens": 4,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 67,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cache_write_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 71
}

Provider metadata

{
  "id": "chatcmpl-EH04N3IfDrq2TCNHkOZSXrwdr0Wrq",
  "model": "gpt-5.6-sol",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {},
  "id": "gpt-5.6",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-5.6",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 32B AWQ qwen32b-awq ✗ FAIL 29

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "29",
  "expected_normalized": "16"
}

Latency

159.68161799537484 ms

Usage

{
  "input_tokens": 90,
  "output_tokens": 3
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 159.1551379970042,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "model_load_ms": 3622.3182269895915,
  "peak_cuda_memory_allocated_bytes": 19440437760,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen32b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

GPT-4o-mini gpt-4o-mini ✗ FAIL 20

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "20",
  "expected_normalized": "16"
}

Latency

559.9637359991902 ms

Usage

{
  "completion_tokens": 1,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 68,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 69
}

Provider metadata

{
  "id": "chatcmpl-EGccwnhhDbhlcONA6nr84Cw6fgiiJ",
  "model": "gpt-4o-mini-2024-07-18",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "temperature": 0
  },
  "id": "gpt-4o-mini",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-4o-mini",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B qwen2.5-14b ✗ FAIL 20

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "20",
  "expected_normalized": "16"
}

Latency

82.26492299581878 ms

Usage

{
  "input_tokens": 90,
  "output_tokens": 3
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 81.9016909954371,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "model_load_ms": 71984.59202000231,
  "peak_cuda_memory_allocated_bytes": 29578535424,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Gemma 2 9B IT gemma2-9b-it ✗ FAIL 19

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "19",
  "expected_normalized": "16"
}

Latency

92.46312400500756 ms

Usage

{
  "input_tokens": 75,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 92.08970000327099,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "model_load_ms": 3646.1853710061405,
  "peak_cuda_memory_allocated_bytes": 18542295552,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "gemma2-9b-it",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B AWQ qwen2.5-14b-awq ✗ FAIL 29

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "29",
  "expected_normalized": "16"
}

Latency

80.30792300996836 ms

Usage

{
  "input_tokens": 90,
  "output_tokens": 3
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 79.8314970015781,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "model_load_ms": 2744.904780993238,
  "peak_cuda_memory_allocated_bytes": 10057558528,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Phi-3 Medium phi3-medium-4k ✗ FAIL 23

Effective candidate prompt

A 3×3 grid starts with 2 at top-left. Moving one cell right adds 3; moving one cell down adds 4. What is the bottom-right value?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "23",
  "expected_normalized": "16"
}

Latency

106.29417400923558 ms

Usage

{
  "input_tokens": 71,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 106.00264598906506,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "model_load_ms": 112327.89194400539,
  "peak_cuda_memory_allocated_bytes": 27995448832,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "phi3-medium-4k",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—