piq45_advanced_letter_number_code

Prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Gold / expectedO19Evaluatornormalized_exactTagsmixed, hardMetadata

—

Response contract

{
  "instruction": "Return only the final answer. Do not include explanation.",
  "mode": "short_answer"
}

Provenance

—

ModalityNot specified

Wrong-answer distribution · literal raw responses

M18 — 1 modelM21 — 1 modelO18 — 1 model

Every model response

gpt-5.6 gpt-5.6 ✓ PASS O19

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "o19",
  "expected_normalized": "o19"
}

Latency

6116.5818459994625 ms

Usage

{
  "completion_tokens": 5,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 77,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cache_write_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 82
}

Provider metadata

{
  "id": "chatcmpl-EH054FDFtJfO3kJHqAMzAFg83tIFv",
  "model": "gpt-5.6-sol",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {},
  "id": "gpt-5.6",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-5.6",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 32B AWQ qwen32b-awq ✓ PASS O19

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "o19",
  "expected_normalized": "o19"
}

Latency

222.63904100691434 ms

Usage

{
  "input_tokens": 101,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 222.0511159976013,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "model_load_ms": 3622.3182269895915,
  "peak_cuda_memory_allocated_bytes": 19445602816,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen32b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

GPT-4o-mini gpt-4o-mini ✓ PASS O19

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "o19",
  "expected_normalized": "o19"
}

Latency

2651.835991004191 ms

Usage

{
  "completion_tokens": 2,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 78,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 80
}

Provider metadata

{
  "id": "chatcmpl-EGcdQC6uxWQpB4DG6HDrXH8OP9Tfy",
  "model": "gpt-4o-mini-2024-07-18",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "temperature": 0
  },
  "id": "gpt-4o-mini",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-4o-mini",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B qwen2.5-14b ✗ FAIL O18

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "o18",
  "expected_normalized": "o19"
}

Latency

118.975784003851 ms

Usage

{
  "input_tokens": 101,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 118.43930500617716,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "model_load_ms": 71984.59202000231,
  "peak_cuda_memory_allocated_bytes": 29582067200,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Gemma 2 9B IT gemma2-9b-it ✓ PASS O19

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "o19",
  "expected_normalized": "o19"
}

Latency

114.13638900557999 ms

Usage

{
  "input_tokens": 84,
  "output_tokens": 5
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 113.81650700059254,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "model_load_ms": 3646.1853710061405,
  "peak_cuda_memory_allocated_bytes": 18543342080,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "gemma2-9b-it",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B AWQ qwen2.5-14b-awq ✗ FAIL M18

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "m18",
  "expected_normalized": "o19"
}

Latency

105.09662299591582 ms

Usage

{
  "input_tokens": 101,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 104.71596599381883,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "model_load_ms": 2744.904780993238,
  "peak_cuda_memory_allocated_bytes": 10060957184,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Phi-3 Medium phi3-medium-4k ✗ FAIL M21

Effective candidate prompt

Code: A1, C4, F8, J13, ?; letter increments: +2, +3, +4...; number increments: +3, +4, +5.... What is the next code?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule conclusively not satisfied

Evaluation evidence

{
  "actual_normalized": "m21",
  "expected_normalized": "o19"
}

Latency

106.56251899490599 ms

Usage

{
  "input_tokens": 81,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 106.2746870011324,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "model_load_ms": 112327.89194400539,
  "peak_cuda_memory_allocated_bytes": 28002637312,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "phi3-medium-4k",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—