PEGASUS EVAL / RESULTS EXPLORER

Observed benchmark evidence · read-only

← Back to matrix

piq08_letters_mirror

Prompt
Find the missing letter: A, Z, B, Y, C, X, ?
Gold / expectedDEvaluatornormalized_exactTagsletter_sequence, easyMetadata
—
Response contract
{
  "instruction": "Return only the final answer. Do not include explanation.",
  "mode": "short_answer"
}
Provenance
—
ModalityNot specified

Wrong-answer distribution · literal raw responses

W — 1 modelW — 1 model

Every model response

gpt-5.6 gpt-5.6   ✓ PASS   D
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "d",
  "expected_normalized": "d"
}
Latency
1605.908505996922 ms
Usage
{
  "completion_tokens": 4,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 48,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cache_write_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 52
}
Provider metadata
{
  "id": "chatcmpl-EH0330A2PqH8vE4OdUuqKh6tIiL3M",
  "model": "gpt-5.6-sol",
  "provider": "openai_compatible"
}
Model config
{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {},
  "id": "gpt-5.6",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-5.6",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}
Execution error
—
Evaluation error
—
Qwen 32B AWQ qwen32b-awq   ✓ PASS   D
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "d",
  "expected_normalized": "d"
}
Latency
121.1956400074996 ms
Usage
{
  "input_tokens": 71,
  "output_tokens": 2
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 120.72269999771379,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "model_load_ms": 3622.3182269895915,
  "peak_cuda_memory_allocated_bytes": 19432191488,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen32b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen32b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
GPT-4o-mini gpt-4o-mini   ✓ PASS   D
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "d",
  "expected_normalized": "d"
}
Latency
5602.787158997671 ms
Usage
{
  "completion_tokens": 1,
  "completion_tokens_details": {
    "accepted_prediction_tokens": 0,
    "audio_tokens": 0,
    "reasoning_tokens": 0,
    "rejected_prediction_tokens": 0
  },
  "prompt_tokens": 49,
  "prompt_tokens_details": {
    "audio_tokens": 0,
    "cached_tokens": 0
  },
  "total_tokens": 50
}
Provider metadata
{
  "id": "chatcmpl-EGcbupGGd1sreqk7oW9DOivC9NYav",
  "model": "gpt-4o-mini-2024-07-18",
  "provider": "openai_compatible"
}
Model config
{
  "adapter": "openai_compatible",
  "base_url": "https://api.openai.com/v1",
  "device": null,
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "temperature": 0
  },
  "id": "gpt-4o-mini",
  "local_files_only": false,
  "max_new_tokens": 512,
  "model": "gpt-4o-mini",
  "quantization_method": null,
  "secret_env": "OPENAI_API_KEY",
  "supports_json_mode": false,
  "timeout": 120
}
Execution error
—
Evaluation error
—
Qwen 2.5 14B qwen2.5-14b   ✓ PASS   D
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "d",
  "expected_normalized": "d"
}
Latency
57.84973700065166 ms
Usage
{
  "input_tokens": 71,
  "output_tokens": 2
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 57.52815199957695,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "model_load_ms": 71984.59202000231,
  "peak_cuda_memory_allocated_bytes": 29572570624,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Gemma 2 9B IT gemma2-9b-it   ✗ FAIL   W
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule conclusively not satisfied
Evaluation evidence
{
  "actual_normalized": "w",
  "expected_normalized": "d"
}
Latency
118.29649799619801 ms
Usage
{
  "input_tokens": 54,
  "output_tokens": 5
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 117.8465800039703,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "model_load_ms": 3646.1853710061405,
  "peak_cuda_memory_allocated_bytes": 18533996032,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "gemma2-9b-it",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/gemma2-9b-it",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Qwen 2.5 14B AWQ qwen2.5-14b-awq   ✗ FAIL   W
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule conclusively not satisfied
Evaluation evidence
{
  "actual_normalized": "w",
  "expected_normalized": "d"
}
Latency
57.515257998602465 ms
Usage
{
  "input_tokens": 71,
  "output_tokens": 2
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 57.133470007102005,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "model_load_ms": 2744.904780993238,
  "peak_cuda_memory_allocated_bytes": 10051984896,
  "provider": "transformers_local",
  "quantization_method": "awq",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "qwen2.5-14b-awq",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/qwen2.5-14b-awq",
  "quantization_method": "awq",
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—
Phi-3 Medium phi3-medium-4k   ✓ PASS   D
Effective candidate prompt
Find the missing letter: A, Z, B, Y, C, X, ?

Return only the final answer. Do not include explanation or reasoning.

Return only the final answer. Do not include explanation.
Evaluation reason
deterministic rule satisfied
Evaluation evidence
{
  "actual_normalized": "d",
  "expected_normalized": "d"
}
Latency
57.38753700279631 ms
Usage
{
  "input_tokens": 49,
  "output_tokens": 2
}
Provider metadata
{
  "adapter": "transformers_local",
  "cuda_version": "12.8",
  "device": "cuda",
  "dtype": "auto",
  "generation_latency_ms": 57.120413999655284,
  "gpu_compute_capability": [
    12,
    0
  ],
  "gpu_name": "NVIDIA GeForce RTX 5090",
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "model_load_ms": 112327.89194400539,
  "peak_cuda_memory_allocated_bytes": 27973260288,
  "provider": "transformers_local",
  "torch_version": "2.9.0+cu128",
  "transformers_version": "4.51.3"
}
Model config
{
  "adapter": "transformers_local",
  "base_url": null,
  "device": "cuda",
  "dtype": "auto",
  "extra_headers": {},
  "generation": {
    "do_sample": false
  },
  "id": "phi3-medium-4k",
  "local_files_only": true,
  "max_new_tokens": 32,
  "model": "[local-model-path-redacted]/phi3-medium-4k",
  "quantization_method": null,
  "secret_env": null,
  "supports_json_mode": false,
  "timeout": 60
}
Execution error
—
Evaluation error
—