PEGASUS EVAL / RESULTS EXPLORER

Observed benchmark evidence · read-only

← Back to matrix

piq34_mixed_two_sequences_interleaved

Prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?
Gold / expected20Evaluatornormalized_exactTagsmixed, hardMetadata
—
Response contract
{
  "instruction": "Return only the final answer. Do not include explanation.",
  "mode": "short_answer"
}
Provenance
—
ModalityNot specified

Wrong-answer distribution · literal raw responses

18 — 1 modelIK — 1 model

Every model response

gpt-5.6 gpt-5.6   ✓ PASS   20
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "20",
  "expected_normalized": "20"
}
Latency
961.5516219928395 ms
Usage
{
  "completion_tokens": 4,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 83,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cache_write_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 87
}
Provider metadata
{
  "id": "chatcmpl-EH04GA8tyImuTyYzFlvEQ6ZtAJFj1",
  "model": "gpt-5.6-sol",
  "provider": "openai_compatible"
}
Model config
{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {},
  "id": "gpt-5.6",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-5.6",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}
Execution error
—
Evaluation error
—
Qwen 32B AWQ qwen32b-awq   ✓ PASS   20
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "20",
  "expected_normalized": "20"
}
Latency
180.45232200529426 ms
Usage
{
  "input_tokens": 107,
  "output_tokens": 3
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 179.96174900326878,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "model_load_ms": 3622.3182269895915,
  "peak_cuda_memory_allocated_bytes": 19450947072,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen32b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
GPT-4o-mini gpt-4o-mini   ✓ PASS   20
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "20",
  "expected_normalized": "20"
}
Latency
506.118958001025 ms
Usage
{
  "completion_tokens": 1,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 84,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 85
}
Provider metadata
{
  "id": "chatcmpl-EGccvCVx7Yt2kvKaUwVdppsMq6YZC",
  "model": "gpt-4o-mini-2024-07-18",
  "provider": "openai_compatible"
}
Model config
{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "temperature": 0
  },
  "id": "gpt-4o-mini",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-4o-mini",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}
Execution error
—
Evaluation error
—
Qwen 2.5 14B qwen2.5-14b   ✓ PASS   20
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "20",
  "expected_normalized": "20"
}
Latency
93.41639500053134 ms
Usage
{
  "input_tokens": 107,
  "output_tokens": 3
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 93.06845201354008,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "model_load_ms": 71984.59202000231,
  "peak_cuda_memory_allocated_bytes": 29584994816,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Gemma 2 9B IT gemma2-9b-it   ✓ PASS   20
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "20",
  "expected_normalized": "20"
}
Latency
135.23475200054236 ms
Usage
{
  "input_tokens": 90,
  "output_tokens": 6
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 134.93039000604767,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "model_load_ms": 3646.1853710061405,
  "peak_cuda_memory_allocated_bytes": 18574511104,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "gemma2-9b-it",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Qwen 2.5 14B AWQ qwen2.5-14b-awq   ✗ FAIL   18
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule conclusively not satisfied
Evaluation evidence
{
  "actual_normalized": "18",
  "expected_normalized": "20"
}
Latency
88.75577100843657 ms
Usage
{
  "input_tokens": 107,
  "output_tokens": 3
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 88.19022699026391,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "model_load_ms": 2744.904780993238,
  "peak_cuda_memory_allocated_bytes": 10063790592,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Phi-3 Medium phi3-medium-4k   ✗ FAIL   IK
Effective candidate prompt
Sequence: A, 2, C, 6, E, 12, G, ?. Letters advance by 2. The numbers are 1×2, 2×3, 3×4, 4×5. What is missing?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule conclusively not satisfied
Evaluation evidence
{
  "actual_normalized": "ik",
  "expected_normalized": "20"
}
Latency
106.86450600042008 ms
Usage
{
  "input_tokens": 88,
  "output_tokens": 4
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 106.58198500459548,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "model_load_ms": 112327.89194400539,
  "peak_cuda_memory_allocated_bytes": 28010509824,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "phi3-medium-4k",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—