Benchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions

assistant-actions-032

hard multi

The prompt

Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to

user

Put Jordan's soccer game on the calendar for Saturday at 9, and remind me Friday night at 8 to pack his cleats.

What we expected

{
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": {
          "type": "contains_all",
          "values": [
            "soccer"
          ]
        },
        "start": {
          "type": "datetime",
          "value": "2026-10-24T09:00:00-05:00",
          "tolerance_s": 60
        },
        "end": {
          "type": "datetime",
          "value": "2026-10-24T10:00:00-05:00",
          "tolerance_s": 60
        }
      },
      "allow_extra_args": [
        "location",
        "attendees"
      ]
    },
    {
      "name": "create_reminder",
      "args": {
        "text": {
          "type": "contains_all",
          "values": [
            "cleats"
          ]
        },
        "when": {
          "type": "datetime",
          "value": "2026-10-23T20:00:00-05:00",
          "tolerance_s": 60
        }
      },
      "allow_extra_args": []
    }
  ]
}

What each model answered

MiMo-V2.6 Distill 9B

✓ pass · score 100 · 4.30 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00"})
create_reminder({"text":"Pack Jordan's soccer cleats","when":"2026-10-23T20:00:00-05:00"})

I'll add the soccer game to the calendar and set the reminder.

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00"
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's soccer cleats",
        "when": "2026-10-23T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 1,
      "args": {
        "text": true,
        "when": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Ornith-1.5 9B

✓ pass · score 100 · 4.31 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00"})
create_reminder({"text":"Pack Jordan's cleats","when":"2026-10-23T20:00:00-05:00"})

I'll add the soccer game to the calendar and set the reminder for you.

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00"
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's cleats",
        "when": "2026-10-23T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 1,
      "args": {
        "text": true,
        "when": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Granite 4.2 8B

✗ fail · score 50 · 3.49 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00","location":"Fairmont Elementary","attendees":["Jordan"]})

Scoring detail

{
  "n_calls": 1,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00",
        "location": "Fairmont Elementary",
        "attendees": [
          "Jordan"
        ]
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Spark-X2.5 4B

✓ pass · score 100 · 2.04 s

add_calendar_event({"title":"Jordan's Soccer Game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00"})
create_reminder({"text":"Pack Jordan's soccer cleats for Friday night game","when":"2026-10-23T20:00:00-05:00"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's Soccer Game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00"
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's soccer cleats for Friday night game",
        "when": "2026-10-23T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 1,
      "args": {
        "text": true,
        "when": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Mellum2.1 12B-A2.5B

✗ fail · score 50 · 1.23 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00","location":"Cedar Hollow, IL","attendees":["Jordan Brennan"]})

Scoring detail

{
  "n_calls": 1,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00",
        "location": "Cedar Hollow, IL",
        "attendees": [
          "Jordan Brennan"
        ]
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Ling-3.0-tiny (7.9B-A1.3B)

✗ fail · score 58 · 1.29 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T11:00:00-05:00"})
create_reminder({"text":"Pack Jordan's cleats for his soccer game.","when":"2026-10-22T20:00:00-05:00"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T11:00:00-05:00"
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's cleats for his soccer game.",
        "when": "2026-10-22T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 0.6666666666666666,
      "args": {
        "title": true,
        "start": true,
        "end": false
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 0.5,
      "args": {
        "text": true,
        "when": false
      }
    }
  ],
  "extra_calls": 0,
  "base": 0.5833333333333333
}

MiniCPM5 2B

✗ fail · score 67 · 1.70 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T21:00:00-05:00","end":"2026-10-24T21:00:00-05:00","location":"Jordan's school","attendees":["Jordan Brennan","Mom","Elena Ruiz","Dev Patel","Priya Shah","Luis Ortega","Sam Rivera","Sam Okafor"]})
create_reminder({"text":"Pack Jordan's cleats","when":"2026-10-23T20:00:00-05:00"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T21:00:00-05:00",
        "end": "2026-10-24T21:00:00-05:00",
        "location": "Jordan's school",
        "attendees": [
          "Jordan Brennan",
          "Mom",
          "Elena Ruiz",
          "Dev Patel",
          "Priya Shah",
          "Luis Ortega",
          "Sam Rivera",
          "Sam Okafor"
        ]
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's cleats",
        "when": "2026-10-23T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 0.3333333333333333,
      "args": {
        "title": true,
        "start": false,
        "end": false
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 1,
      "args": {
        "text": true,
        "when": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 0.6666666666666666
}

K2-Horizon 0.9B

✓ pass · score 100 · 0.71 s

add_calendar_event({"title":"Jordan's soccer game","start":"2026-10-24T09:00:00-05:00","end":"2026-10-24T10:00:00-05:00","location":"Jordan's soccer game"})
create_reminder({"text":"Pack Jordan's cleats","when":"2026-10-23T20:00:00-05:00"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "add_calendar_event",
      "args": {
        "title": "Jordan's soccer game",
        "start": "2026-10-24T09:00:00-05:00",
        "end": "2026-10-24T10:00:00-05:00",
        "location": "Jordan's soccer game"
      },
      "error": null
    },
    {
      "name": "create_reminder",
      "args": {
        "text": "Pack Jordan's cleats",
        "when": "2026-10-23T20:00:00-05:00"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "add_calendar_event",
      "matched_index": 0,
      "score": 1,
      "args": {
        "title": true,
        "start": true,
        "end": true
      }
    },
    {
      "expected": "create_reminder",
      "matched_index": 1,
      "score": 1,
      "args": {
        "text": true,
        "when": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

← assistant-actions-031All Assistant actions itemsassistant-actions-033 →