Unified MemoryBenchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions
assistant-actions-035
hard multi
The prompt
Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to
user
Tell Dev I'll be there at 3, and put a meeting with him called 'Budget review' on my calendar tomorrow at 3.
What we expected
{
"calls": [
{
"name": "send_message",
"args": {
"to": {
"type": "contains_all",
"values": [
"dev"
]
},
"text": {
"type": "contains_any",
"values": [
"3",
"three"
]
}
},
"allow_extra_args": []
},
{
"name": "add_calendar_event",
"args": {
"title": {
"type": "contains_all",
"values": [
"budget review"
]
},
"start": {
"type": "datetime",
"value": "2026-10-22T15:00:00-05:00",
"tolerance_s": 60
},
"end": {
"type": "datetime",
"value": "2026-10-22T16:00:00-05:00",
"tolerance_s": 60
},
"attendees": {
"type": "optional",
"inner": {
"type": "list_contains_all",
"values": [
"dev"
]
}
}
},
"allow_extra_args": [
"location"
]
}
]
}What each model answered
MiMo-V2.6 Distill 9B
✓ pass · score 100 · 3.90 s
send_message({"to":"Dev Patel","text":"I'll be there at 3."})
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00","attendees":["Dev Patel"]})I'll send Dev the message and add the calendar event.
Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel",
"text": "I'll be there at 3."
},
"error": null
},
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00",
"attendees": [
"Dev Patel"
]
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": 1,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 1
}Ornith-1.5 9B
✓ pass · score 100 · 4.02 s
send_message({"to":"Dev Patel","text":"I'll be there at 3."})
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00","attendees":["Dev Patel"]})I'll set that up for you now.
Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel",
"text": "I'll be there at 3."
},
"error": null
},
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00",
"attendees": [
"Dev Patel"
]
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": 1,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 1
}Granite 4.2 8B
✗ fail · score 50 · 3.13 s
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00","attendees":["Dev Patel"]})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00",
"attendees": [
"Dev Patel"
]
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": null,
"score": 0,
"args": "no call"
},
{
"expected": "add_calendar_event",
"matched_index": 0,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 0.5
}Spark-X2.5 4B
✓ pass · score 100 · 1.56 s
send_message({"to":"Dev Patel","text":"I'll be there at 3."})
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00"})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel",
"text": "I'll be there at 3."
},
"error": null
},
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": 1,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 1
}Mellum2.1 12B-A2.5B
✗ fail · score 50 · 0.49 s
send_message({"to":"Dev Patel","text":"I'll be there at 3"})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel",
"text": "I'll be there at 3"
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": null,
"score": 0,
"args": "no call"
}
],
"extra_calls": 0,
"base": 0.5
}Ling-3.0-tiny (7.9B-A1.3B)
✓ pass · score 100 · 1.14 s
send_message({"to":"Dev Patel","text":"I'll be there at 3."})
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00","attendees":["Dev Patel"]})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel",
"text": "I'll be there at 3."
},
"error": null
},
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00",
"attendees": [
"Dev Patel"
]
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": 1,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 1
}MiniCPM5 2B
✓ pass · score 100 · 1.11 s
send_message({"to":"Dev Patel (coworker)","text":"Tell Dev I'll be there at 3."})
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00"})Scoring detail
{
"n_calls": 2,
"calls": [
{
"name": "send_message",
"args": {
"to": "Dev Patel (coworker)",
"text": "Tell Dev I'll be there at 3."
},
"error": null
},
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00"
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": 0,
"score": 1,
"args": {
"to": true,
"text": true
}
},
{
"expected": "add_calendar_event",
"matched_index": 1,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 1
}K2-Horizon 0.9B
✗ fail · score 50 · 0.55 s
add_calendar_event({"title":"Budget review","start":"2026-10-22T15:00:00-05:00","end":"2026-10-22T16:00:00-05:00","location":"Dev Patel","attendees":["Dev Patel"]})Scoring detail
{
"n_calls": 1,
"calls": [
{
"name": "add_calendar_event",
"args": {
"title": "Budget review",
"start": "2026-10-22T15:00:00-05:00",
"end": "2026-10-22T16:00:00-05:00",
"location": "Dev Patel",
"attendees": [
"Dev Patel"
]
},
"error": null
}
],
"per_call": [
{
"expected": "send_message",
"matched_index": null,
"score": 0,
"args": "no call"
},
{
"expected": "add_calendar_event",
"matched_index": 0,
"score": 1,
"args": {
"title": true,
"start": true,
"end": true,
"attendees": true
}
}
],
"extra_calls": 0,
"base": 0.5
}← assistant-actions-034All Assistant actions itemsassistant-actions-036 →