piq43_advanced_fraction_sequence

Prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Gold / expected1/5Evaluatornormalized_exactTagsnumeric_sequence, hardMetadata

—

Response contract

{
  "instruction": "Return only the final answer. Do not include explanation.",
  "mode": "short_answer"
}

Provenance

—

ModalityNot specified

Wrong-answer distribution · literal raw responses

No failed responses

Every model response

gpt-5.6 gpt-5.6 ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

6036.380237987032 ms

Usage

{
  "completion_tokens": 6,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 58,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cache_write_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 64
}

Provider metadata

{
  "id": "chatcmpl-EH04vOKuuyUKnpH2V0E4Glh6JhK2Z",
  "model": "gpt-5.6-sol",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {},
  "id": "gpt-5.6",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-5.6",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 32B AWQ qwen32b-awq ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

197.64781100093387 ms

Usage

{
  "input_tokens": 81,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 197.10161301190965,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "model_load_ms": 3622.3182269895915,
  "peak_cuda_memory_allocated_bytes": 19436212736,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen32b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

GPT-4o-mini gpt-4o-mini ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

1109.744323999621 ms

Usage

{
  "completion_tokens": 3,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 59,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 62
}

Provider metadata

{
  "id": "chatcmpl-EGcdLieU56gSb2K3i0zT5mQfXaAMJ",
  "model": "gpt-4o-mini-2024-07-18",
  "provider": "openai_compatible"
}

Model config

{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "temperature": 0
  },
  "id": "gpt-4o-mini",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-4o-mini",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B qwen2.5-14b ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

110.98797198792454 ms

Usage

{
  "input_tokens": 81,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 110.60143599752337,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "model_load_ms": 71984.59202000231,
  "peak_cuda_memory_allocated_bytes": 29575646720,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Gemma 2 9B IT gemma2-9b-it ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

113.61445600050502 ms

Usage

{
  "input_tokens": 64,
  "output_tokens": 5
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 113.3360219973838,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "model_load_ms": 3646.1853710061405,
  "peak_cuda_memory_allocated_bytes": 18541014016,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "gemma2-9b-it",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Qwen 2.5 14B AWQ qwen2.5-14b-awq ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

96.09080500376876 ms

Usage

{
  "input_tokens": 81,
  "output_tokens": 4
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 95.69475999160204,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "model_load_ms": 2744.904780993238,
  "peak_cuda_memory_allocated_bytes": 10054918656,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—

Phi-3 Medium phi3-medium-4k ✓ PASS 1/5

Effective candidate prompt

Sequence: 1, 1/2, 1/3, 1/4, ?; what is the next term?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.

Evaluation reason

deterministic rule satisfied

Evaluation evidence

{
  "actual_normalized": "1/5",
  "expected_normalized": "1/5"
}

Latency

124.4251419993816 ms

Usage

{
  "input_tokens": 60,
  "output_tokens": 5
}

Provider metadata

{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 124.1409669892164,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "model_load_ms": 112327.89194400539,
  "peak_cuda_memory_allocated_bytes": 27983098880,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}

Model config

{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "phi3-medium-4k",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}

Execution error

—

Evaluation error

—