Unified MemoryBenchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions
assistant-actions-034
hard multi
The prompt
Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to
user
Add the tent and a headlamp to the packing list, and remind me Thursday at 7 PM to charge the lantern.
What we expected
{
"calls": [
{
"name": "add_to_list",
"args": {
"list": {
"type": "norm",
"value": "packing"
},
"item": {
"type": "contains_all",
"values": [
"tent"
]
}
},
"allow_extra_args": []
},
{
"name": "add_to_list",
"args": {
"list": {
"type": "norm",
"value": "packing"
},
"item": {
"type": "contains_all",
"values": [
"headlamp"
]
}
},
"allow_extra_args": []
},
{
"name": "create_reminder",
"args": {
"text": {
"type": "contains_all",
"values": [
"lantern"
]
},
"when": {
"type": "datetime",
"value": "2026-10-22T19:00:00-05:00",
"tolerance_s": 60
}
},
"allow_extra_args": []
}
]
}What each model answered
MiMo-V2.6 Distill 9B
✓ pass · score 100 · 3.26 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Charge the lantern","when":"2026-10-22T19:00:00-05:00"})I'll add both items to your packing list and set the reminder.
Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Charge the lantern",
"when": "2026-10-22T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 1,
"args": {
"text": true,
"when": true
}
}
],
"extra_calls": 0,
"base": 1
}Ornith-1.5 9B
✓ pass · score 100 · 3.81 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Charge the lantern","when":"2026-10-22T19:00:00-05:00"})I'll add the tent and headlamp to your packing list, and set the reminder for Thursday at 7 PM.
Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Charge the lantern",
"when": "2026-10-22T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 1,
"args": {
"text": true,
"when": true
}
}
],
"extra_calls": 0,
"base": 1
}Granite 4.2 8B
✗ fail · score 33 · 1.41 s
add_to_list({"list":"packing","item":"tent"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": null,
"score": 0,
"args": "no call"
},
{
"expected": "create_reminder",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.3333333333333333
}Spark-X2.5 4B
✗ fail · score 83 · 1.32 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Charge the lantern","when":"2026-10-23T19:00:00-05:00"})Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Charge the lantern",
"when": "2026-10-23T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 0.5,
"args": {
"text": true,
"when": false
}
}
],
"extra_calls": 0,
"base": 0.8333333333333334
}Mellum2.1 12B-A2.5B
✗ fail · score 33 · 0.44 s
add_to_list({"list":"packing","item":"tent"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": null,
"score": 0,
"args": "no call"
},
{
"expected": "create_reminder",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.3333333333333333
}Ling-3.0-tiny (7.9B-A1.3B)
✓ pass · score 100 · 0.97 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Remember to charge the lantern before Thursday at 7 PM.","when":"2026-10-22T19:00:00-05:00"})Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Remember to charge the lantern before Thursday at 7 PM.",
"when": "2026-10-22T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 1,
"args": {
"text": true,
"when": true
}
}
],
"extra_calls": 0,
"base": 1
}MiniCPM5 2B
✓ pass · score 100 · 1.00 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Charge the lantern","when":"2026-10-22T19:00:00-05:00"})Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Charge the lantern",
"when": "2026-10-22T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 1,
"args": {
"text": true,
"when": true
}
}
],
"extra_calls": 0,
"base": 1
}K2-Horizon 0.9B
✗ fail · score 83 · 0.56 s
add_to_list({"list":"packing","item":"tent"})
add_to_list({"list":"packing","item":"headlamp"})
create_reminder({"text":"Charge the lantern","when":"2026-10-24T19:00:00-05:00"})Scoring detail
{
"n_calls": 3,
"calls": [
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "tent"
},
"error": null
},
{
"name": "add_to_list",
"args": {
"list": "packing",
"item": "headlamp"
},
"error": null
},
{
"name": "create_reminder",
"args": {
"text": "Charge the lantern",
"when": "2026-10-24T19:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "add_to_list",
"matched_index": 0,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "add_to_list",
"matched_index": 1,
"score": 1,
"args": {
"list": true,
"item": true
}
},
{
"expected": "create_reminder",
"matched_index": 2,
"score": 0.5,
"args": {
"text": true,
"when": false
}
}
],
"extra_calls": 0,
"base": 0.8333333333333334
}← assistant-actions-033All Assistant actions itemsassistant-actions-035 →