Unified Memory

Benchmarks / Which Small AI Model Should You Run on a Mac? We Ran 312 Tests / Assistant actions

assistant-actions-029

medium multi

The prompt

Tools offered: create_reminder, add_calendar_event, set_timer, set_alarm, control_light, set_thermostat, lock_door, send_message, play_music, get_weather, add_to_list, navigate_to

user

Movie night! Dim the living room lights to 20 and set the thermostat to 70.
What we expected
{
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": {
          "type": "norm",
          "value": "living_room"
        },
        "state": {
          "type": "norm",
          "value": "on"
        },
        "brightness": {
          "type": "number",
          "value": 20,
          "tolerance": 0
        }
      },
      "allow_extra_args": []
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": {
          "type": "number",
          "value": 70,
          "tolerance": 0
        }
      },
      "allow_extra_args": [
        "mode"
      ]
    }
  ]
}

What each model answered

MiMo-V2.6 Distill 9B

✓ pass · score 100 · 1.90 s

control_light({"room":"living_room","state":"on","brightness":20})
set_thermostat({"temp_f":70,"mode":"auto"})
Scoring detail
{
  "n_calls": 2,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 1,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Ornith-1.5 9B

✓ pass · score 100 · 2.49 s

control_light({"room":"living_room","state":"on","brightness":20})
set_thermostat({"temp_f":70})
I'll dim the living room lights and set the thermostat to 70 for your movie night.
Scoring detail
{
  "n_calls": 2,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 1,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

Granite 4.2 8B

✗ fail · score 50 · 1.75 s

control_light({"room":"living_room","state":"on","brightness":20})
Scoring detail
{
  "n_calls": 1,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Spark-X2.5 4B

✗ fail · score 83 · 0.74 s

control_light({"room":"living_room","state":"off","brightness":20})
set_thermostat({"temp_f":70})
Scoring detail
{
  "n_calls": 2,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "off",
        "brightness": 20
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 0.6666666666666666,
      "args": {
        "room": true,
        "state": false,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 1,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 0.8333333333333333
}

Mellum2.1 12B-A2.5B

✗ fail · score 50 · 0.57 s

control_light({"room":"living_room","state":"on","brightness":20})
Scoring detail
{
  "n_calls": 1,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": null,
      "score": 0,
      "args": "no call"
    }
  ],
  "extra_calls": 0,
  "base": 0.5
}

Ling-3.0-tiny (7.9B-A1.3B)

✓ pass · score 100 · 0.57 s

control_light({"room":"living_room","state":"on","brightness":20})
set_thermostat({"temp_f":70,"mode":"cool"})
Scoring detail
{
  "n_calls": 2,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "cool"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 1,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

MiniCPM5 2B

✓ pass · score 100 · 0.59 s

control_light({"room":"living_room","state":"on","brightness":20})
set_thermostat({"temp_f":70})
Scoring detail
{
  "n_calls": 2,
  "calls": [
    {
      "name": "control_light",
      "args": {
        "room": "living_room",
        "state": "on",
        "brightness": 20
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 0,
      "score": 1,
      "args": {
        "room": true,
        "state": true,
        "brightness": true
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 1,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 0,
  "base": 1
}

K2-Horizon 0.9B

✗ fail · score 3 · 5.36 s

set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
set_thermostat({"temp_f":70,"mode":"auto"})
Scoring detail
{
  "n_calls": 39,
  "calls": [
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    },
    {
      "name": "set_thermostat",
      "args": {
        "temp_f": 70,
        "mode": "auto"
      },
      "error": null
    }
  ],
  "per_call": [
    {
      "expected": "control_light",
      "matched_index": 26,
      "score": 0,
      "args": {
        "name": "mismatch"
      }
    },
    {
      "expected": "set_thermostat",
      "matched_index": 21,
      "score": 1,
      "args": {
        "temp_f": true
      }
    }
  ],
  "extra_calls": 37,
  "base": 0.5
}

← assistant-actions-028All Assistant actions itemsassistant-actions-030 →