Unified MemoryBenchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions
assistant-actions-030
hard multi
The prompt
Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to
user
What's the weather in Cedar Hollow today and tomorrow?
What we expected
{
"calls": [
{
"name": "get_weather",
"args": {
"location": {
"type": "contains_any",
"values": [
"cedar hollow"
]
},
"date": {
"type": "optional",
"inner": {
"type": "date",
"value": "2026-10-21"
}
}
},
"allow_extra_args": []
},
{
"name": "get_weather",
"args": {
"location": {
"type": "contains_any",
"values": [
"cedar hollow"
]
},
"date": {
"type": "date",
"value": "2026-10-22"
}
},
"allow_extra_args": []
}
]
}What each model answered
MiMo-V2.6 Distill 9B
✓ pass · score 100 · 1.65 s
get_weather({"location":"Cedar Hollow, IL"})
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-22"})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
},
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-22"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": 1,
"score": 1,
"args": {
"location": true,
"date": true
}
}
],
"extra_calls": 0,
"base": 1
}Ornith-1.5 9B
✓ pass · score 100 · 2.26 s
get_weather({"location":"Cedar Hollow, IL"})
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-22"})I'll check the weather for Cedar Hollow today and tomorrow.
Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
},
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-22"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": 1,
"score": 1,
"args": {
"location": true,
"date": true
}
}
],
"extra_calls": 0,
"base": 1
}Granite 4.2 8B
✗ fail · score 50 · 1.16 s
get_weather({"location":"Cedar Hollow, IL"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.5
}Spark-X2.5 4B
✗ fail · score 50 · 0.31 s
get_weather({"location":"Cedar Hollow, IL"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.5
}Mellum2.1 12B-A2.5B
✗ fail · score 50 · 0.56 s
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-21"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-21"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.5
}Ling-3.0-tiny (7.9B-A1.3B)
✓ pass · score 100 · 0.69 s
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-21"})
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-22"})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-21"
},
"error": null
},
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-22"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": 1,
"score": 1,
"args": {
"location": true,
"date": true
}
}
],
"extra_calls": 0,
"base": 1
}MiniCPM5 2B
✓ pass · score 100 · 0.60 s
get_weather({"location":"Cedar Hollow, IL"})
get_weather({"location":"Cedar Hollow, IL","date":"2026-10-22"})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
},
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL",
"date": "2026-10-22"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": 1,
"score": 1,
"args": {
"location": true,
"date": true
}
}
],
"extra_calls": 0,
"base": 1
}K2-Horizon 0.9B
✗ fail · score 50 · 0.28 s
get_weather({"location":"Cedar Hollow, IL"})I'll get the weather forecast for Cedar Hollow today and tomorrow for you.
Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "get_weather",
"args": {
"location": "Cedar Hollow, IL"
},
"error": null
}
],
"per_call": [
{
"expected": "get_weather",
"matched_index": 0,
"score": 1,
"args": {
"location": true,
"date": true
}
},
{
"expected": "get_weather",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.5
}← assistant-actions-029All Assistant actions itemsassistant-actions-031 →