Benchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions

assistant-actions-027

medium multi

The prompt

Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to

user

Set a timer for 25 minutes and play some lo-fi in the office.

What we expected

{
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": {
          "type": "number",
          "value": 1500,
          "tolerance": 0
        }
      },
      "allow_extra_args": [
        "label"
      ]
    },
    {
      "name": "play_music",
      "args": {
        "query": {
          "type": "contains_any",
          "values": [
            "lo fi",
            "lofi"
          ]
        },
        "room": {
          "type": "norm",
          "value": "office"
        }
      },
      "allow_extra_args": []
    }
  ]
}

What each model answered

MiMo-V2.6 Distill 9B

✓ pass · score 100 · 1.64 s

set_timer({"seconds":1500,"label":"25 minutes"})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500,
        "label": "25 minutes"
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Ornith-1.5 9B

✓ pass · score 100 · 1.69 s

set_timer({"seconds":1500})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Granite 4.2 8B

✗ fail · score 50 · 1.37 s

set_timer({"seconds":1500,"label":"focus timer"})

Scoring detail

{
  "n_calls": 1,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500,
        "label": "focus timer"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Spark-X2.5 4B

✓ pass · score 100 · 0.71 s

set_timer({"seconds":1500,"label":"25 min"})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500,
        "label": "25 min"
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Mellum2.1 12B-A2.5B

✗ fail · score 50 · 0.51 s

set_timer({"seconds":1500,"label":"lo-fi"})

Scoring detail

{
  "n_calls": 1,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500,
        "label": "lo-fi"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Ling-3.0-tiny (7.9B-A1.3B)

✓ pass · score 100 · 0.52 s

set_timer({"seconds":1500,"label":"lo-fi session"})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1500,
        "label": "lo-fi session"
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 1,
      "args": {
        "seconds": true
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

MiniCPM5 2B

✗ fail · score 50 · 0.57 s

set_timer({"seconds":25,"label":"lo-fi"})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 25,
        "label": "lo-fi"
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 0,
      "args": {
        "seconds": false
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

K2-Horizon 0.9B

✗ fail · score 50 · 0.34 s

set_timer({"seconds":1800,"label":"lo-fi"})
play_music({"query":"lo-fi","room":"office"})

Scoring detail

{
  "n_calls": 2,
  "calls": [
    {
      "name": "set_timer",
      "args": {
        "seconds": 1800,
        "label": "lo-fi"
      },
      "error": null
    },
    {
      "name": "play_music",
      "args": {
        "query": "lo-fi",
        "room": "office"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "set_timer",
      "matched_index": 0,
      "score": 0,
      "args": {
        "seconds": false
      }
    },
    {
      "expected": "play_music",
      "matched_index": 1,
      "score": 1,
      "args": {
        "query": true,
        "room": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

← assistant-actions-026All Assistant actions itemsassistant-actions-028 →