From 50c9e43c7d511aea2932b34cd949c650c8c26c21 Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:56:22 +0200 Subject: [PATCH 1/4] Add AndroidKit semantic UI support Co-Authored-By: siGit Code --- crates/cli/src/mcp_xcrs.rs | 6 +- crates/xcrs/README.md | 6 + crates/xcrs/src/main.rs | 50 ++++++- crates/xcrs/src/mcp.rs | 228 ++++++++++++++++++++++--------- crates/xcrs/src/xcrs.rs | 269 +++++++++++++++++++++++++++++++++++++ docs/androidkit.md | 27 ++++ docs/mcp.md | 11 +- 7 files changed, 523 insertions(+), 74 deletions(-) create mode 100644 docs/androidkit.md diff --git a/crates/cli/src/mcp_xcrs.rs b/crates/cli/src/mcp_xcrs.rs index d816afd..cea3f94 100644 --- a/crates/cli/src/mcp_xcrs.rs +++ b/crates/cli/src/mcp_xcrs.rs @@ -39,6 +39,7 @@ xcrs::xcrs_mcp_tools!( "url_open", "ui_describe", "ui_element_list", + "ui_tap", "input_tap", "input_text", "input_swipe", @@ -89,7 +90,7 @@ pub async fn serve() -> Result<()> { mod tests { use super::*; - const CANONICAL_TOOL_NAMES: [&str; 18] = [ + const CANONICAL_TOOL_NAMES: [&str; 19] = [ "device_list", "device_select", "device_capabilities", @@ -100,6 +101,7 @@ mod tests { "url_open", "ui_describe", "ui_element_list", + "ui_tap", "input_tap", "input_text", "input_swipe", @@ -111,7 +113,7 @@ mod tests { ]; #[test] - fn automation_router_exposes_exactly_the_canonical_eighteen_tools() { + fn automation_router_exposes_exactly_the_canonical_nineteen_tools() { let tools = AutomationMcpServer::xcrs_tool_router().list_all(); let mut names: Vec<&str> = tools.iter().map(|tool| tool.name.as_ref()).collect(); names.sort_unstable(); diff --git a/crates/xcrs/README.md b/crates/xcrs/README.md index e2fb0dd..9616ed1 100644 --- a/crates/xcrs/README.md +++ b/crates/xcrs/README.md @@ -5,6 +5,8 @@ physical Apple devices, Android phones, and Android TV devices. XCRS combines Xcode, ControlKit, CoreDevice, and adb behind one target-aware automation workflow. +https://mobilenext.ai/docs/architecture#how-a-single-action-flows + MCP Registry name: `mcp-name: io.github.smbcloudXYZ/xcrs` ## Run the MCP server @@ -18,6 +20,10 @@ Android tools use `adb` from `ANDROID_SDK_ROOT`, `ANDROID_HOME`, the standard Android SDK locations, or `PATH`. Connect a device with USB debugging enabled and authorize the host before selecting it with `device_select`. +For Android accessibility-tree inspection and semantic `ui_tap`, install the +optional [XCRS AndroidKit](../../docs/androidkit.md) companion runner. Raw adb +actions remain available without it. + The same automation profile is available from the full smbCloud CLI: ```sh diff --git a/crates/xcrs/src/main.rs b/crates/xcrs/src/main.rs index 9f3b8ee..df2a233 100644 --- a/crates/xcrs/src/main.rs +++ b/crates/xcrs/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use std::path::PathBuf; -use xcrs::{IosAppTest, XcodeCommandLineTools}; +use xcrs::{AndroidDebugBridge, AndroidKit, IosAppTest, XcodeCommandLineTools}; #[derive(Debug, Parser)] #[command(name = "xcrs")] @@ -19,6 +19,8 @@ struct Cli { enum Commands { #[command(subcommand)] Simulator(SimulatorCommand), + #[command(subcommand)] + AndroidKit(AndroidKitCommand), IosAppTest { #[arg(long, conflicts_with = "simulator_udid")] simulator_name: Option, @@ -35,6 +37,30 @@ enum Commands { }, } +#[derive(Debug, Subcommand)] +enum AndroidKitCommand { + Install { + #[arg(long)] + serial: String, + #[arg(long)] + apk: PathBuf, + }, + Start { + #[arg(long)] + serial: String, + }, + Status { + #[arg(long)] + serial: String, + #[arg(long)] + json: bool, + }, + Stop { + #[arg(long)] + serial: String, + }, +} + #[derive(Debug, Subcommand)] enum SimulatorCommand { Find { @@ -63,6 +89,28 @@ async fn main() -> Result<()> { let tools = XcodeCommandLineTools::new(); match command { + Commands::AndroidKit(AndroidKitCommand::Install { serial, apk }) => { + AndroidDebugBridge::new().install_androidkit(&serial, &apk)?; + println!("Installed XCRS AndroidKit on {serial}"); + } + Commands::AndroidKit(AndroidKitCommand::Start { serial }) => { + AndroidDebugBridge::new().start_androidkit(&serial)?; + println!("Started XCRS AndroidKit on {serial}"); + } + Commands::AndroidKit(AndroidKitCommand::Status { serial, json }) => { + let info = AndroidKit::new() + .call(serial.clone(), "device.info", serde_json::json!({})) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&info)?); + } else { + println!("XCRS AndroidKit is ready on {serial}"); + } + } + Commands::AndroidKit(AndroidKitCommand::Stop { serial }) => { + AndroidDebugBridge::new().stop_androidkit(&serial)?; + println!("Stopped XCRS AndroidKit on {serial}"); + } Commands::Simulator(SimulatorCommand::Find { name, json }) => { let simulator = tools.simctl().find_simulator_by_name(&name)?; if json { diff --git a/crates/xcrs/src/mcp.rs b/crates/xcrs/src/mcp.rs index 39bc9e7..dcc2985 100644 --- a/crates/xcrs/src/mcp.rs +++ b/crates/xcrs/src/mcp.rs @@ -314,6 +314,7 @@ pub fn android_capabilities() -> serde_json::Value { "app_install_launch": false, "ui_describe": false, "ui_element_list": false, + "ui_tap": false, "orientation_get": false, "orientation_set": false, "input_click": false, @@ -602,11 +603,25 @@ pub struct UiTargetArgs { /// Local ControlKit JSON-RPC port. Defaults to 12004. #[serde(default)] pub controlkit_port: Option, - /// Bundle identifier of the app whose accessibility hierarchy should be read. + /// Android device serial from `adb devices -l`. + #[serde(default)] + pub android_serial: Option, + /// Target-neutral app identifier: an Apple bundle identifier or Android package name. + #[serde(default)] + pub app_id: Option, + /// Backward-compatible alias for `app_id` on Apple targets. #[serde(default)] pub bundle_id: Option, } +#[derive(Debug, Deserialize, JsonSchema)] +pub struct UiTapArgs { + #[serde(flatten)] + pub target: UiTargetArgs, + /// Exact accessible text, description, identifier, value, or hint to activate. + pub element: String, +} + #[derive(Debug, Deserialize, JsonSchema)] pub struct OrientationSetArgs { /// Exact Apple simulator name. Omit to use the target selected with @@ -672,6 +687,7 @@ macro_rules! xcrs_mcp_tools { $url_open_name:literal, $ui_describe_name:literal, $ui_element_list_name:literal, + $ui_tap_name:literal, $input_tap_name:literal, $input_text_name:literal, $input_swipe_name:literal, @@ -697,23 +713,29 @@ macro_rules! xcrs_mcp_tools { .and_then(|slot| slot.clone()) } - fn require_bundle_id( + fn require_app_id( tool_name: &str, + app_id: Option, bundle_id: Option, ) -> ::std::result::Result { - let bundle_id = bundle_id - .as_deref() - .map(str::trim) - .filter(|bundle_id| !bundle_id.is_empty()) - .ok_or_else(|| { + let app_id = app_id.as_deref().map(str::trim).filter(|value| !value.is_empty()); + let bundle_id = bundle_id.as_deref().map(str::trim).filter(|value| !value.is_empty()); + if let (Some(app_id), Some(bundle_id)) = (app_id, bundle_id) { + if app_id != bundle_id { + return Err(::rmcp::model::ErrorData::invalid_request( + format!("{tool_name} received conflicting app_id and bundle_id values."), + None, + )); + } + } + app_id.or(bundle_id).map(str::to_string).ok_or_else(|| { ::rmcp::model::ErrorData::invalid_request( format!( - "{tool_name} requires bundle_id. Pass the bundle identifier of the foreground Apple app." + "{tool_name} requires app_id. Pass the Apple bundle identifier or Android package name of the foreground app." ), None, ) - })?; - Ok(bundle_id.to_string()) + }) } /// Validate and tag a target from per-call fields, falling back to the @@ -947,7 +969,7 @@ macro_rules! xcrs_mcp_tools { name = $device_capabilities_name, title = "Get device capabilities", annotations(title = "Get device capabilities", read_only_hint = true, idempotent_hint = true), - description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, returns a static capability description reflecting exactly the adb actions this crate implements (screen capture, app launch/terminate, URL open, tap/text/swipe/button), since adb has no capability-discovery RPC. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out." + description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, starts from the adb actions this crate implements, then probes XCRS AndroidKit and enables semantic UI actions when its instrumentation runner is reachable. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device, and XCRS AndroidKit must be installed and started for semantic UI actions. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out." )] async fn device_capabilities( &self, @@ -968,11 +990,34 @@ macro_rules! xcrs_mcp_tools { args.android_serial, )?; match &target { - $crate::mcp::SelectedTarget::Android { .. } => { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let mut capabilities = $crate::mcp::android_capabilities(); + let androidkit = $crate::AndroidKit::new() + .call(device.serial.clone(), "device.info", ::serde_json::json!({})) + .await; + if let Some(object) = capabilities.as_object_mut() { + match androidkit { + Ok(info) => { + object.insert("ui_describe".to_string(), ::serde_json::json!(true)); + object.insert("ui_element_list".to_string(), ::serde_json::json!(true)); + object.insert("ui_tap".to_string(), ::serde_json::json!(true)); + object.insert("androidkit".to_string(), ::serde_json::json!({ "available": true, "info": info })); + } + Err(error) => { + object.insert("ui_tap".to_string(), ::serde_json::json!(false)); + object.insert("androidkit".to_string(), ::serde_json::json!({ + "available": false, + "setup": "Install and start XCRS AndroidKit with xcrs android-kit install/start.", + "error": error.to_string(), + })); + } + } + } Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "platform": "android", - "capabilities": $crate::mcp::android_capabilities(), + "capabilities": capabilities, }))?, ])) } @@ -1357,7 +1402,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_describe_name, title = "Describe UI", annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true), - description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." + description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground." )] async fn ui_describe( &self, @@ -1370,30 +1415,35 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { - let bundle_id = Self::require_bundle_id($ui_describe_name, args.bundle_id)?; - let target = Self::dispatch_apple_target( - $ui_describe_name, + let app_id = Self::require_app_id($ui_describe_name, args.app_id, args.bundle_id)?; + let target = Self::dispatch_target( args.simulator_name, args.simulator_udid, args.host, args.controlkit_port, + args.android_serial, )?; - let (simulator, controlkit) = Self::controlkit_for_target(&target)?; - let result = controlkit - .call( - "device.dump.ui", - ::serde_json::json!({ - "format": "json", - "bundleId": bundle_id, - }), - ) - .await - .map_err(|error| { - ::rmcp::model::ErrorData::internal_error(error.to_string(), None) - })?; + let (simulator, result) = match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let ui = $crate::AndroidKit::new() + .call(device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id })) + .await + .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (None, ui) + } + _ => { + let (simulator, controlkit) = Self::controlkit_for_target(&target)?; + let ui = controlkit.call("device.dump.ui", ::serde_json::json!({ + "format": "json", "bundleId": app_id, + })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (simulator, ui) + } + }; Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "simulator": simulator, + "target": target, "ui": result, }))?, ])) @@ -1403,7 +1453,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_element_list_name, title = "List UI elements", annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true), - description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." + description = "Purpose: list just the actionable accessibility elements of the foreground app with labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit; then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground." )] async fn ui_element_list( &self, @@ -1416,36 +1466,76 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { - let bundle_id = Self::require_bundle_id($ui_element_list_name, args.bundle_id)?; - let target = Self::dispatch_apple_target( - $ui_element_list_name, - args.simulator_name, - args.simulator_udid, - args.host, - args.controlkit_port, + let app_id = Self::require_app_id($ui_element_list_name, args.app_id, args.bundle_id)?; + let target = Self::dispatch_target( + args.simulator_name, args.simulator_udid, args.host, args.controlkit_port, args.android_serial, )?; - let (simulator, controlkit) = Self::controlkit_for_target(&target)?; - let ui = controlkit - .call( - "device.dump.ui", - ::serde_json::json!({ - "format": "json", - "bundleId": bundle_id, - }), - ) - .await - .map_err(|error| { - ::rmcp::model::ErrorData::internal_error(error.to_string(), None) - })?; - let elements = $crate::extract_controlkit_elements(&ui); + let (simulator, elements) = match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let ui = $crate::AndroidKit::new().call( + device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id }), + ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (None, $crate::extract_androidkit_elements(&ui)) + } + _ => { + let (simulator, controlkit) = Self::controlkit_for_target(&target)?; + let ui = controlkit.call("device.dump.ui", ::serde_json::json!({ + "format": "json", "bundleId": app_id, + })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (simulator, $crate::extract_controlkit_elements(&ui)) + } + }; Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "simulator": simulator, + "target": target, "elements": elements, }))?, ])) } + #[::rmcp::tool( + name = $ui_tap_name, + title = "Tap UI element", + annotations(title = "Tap UI element", read_only_hint = false, destructive_hint = false, idempotent_hint = false), + description = "Purpose: activate one uniquely identified accessible element in the foreground app. When to use vs siblings: use this for semantic interaction by label or identifier; use input_tap only when you deliberately need raw screen coordinates. Behavior: accepts app_id (or Apple-compatible bundle_id) and an exact element value. Apple targets use ControlKit; Android targets use XCRS AndroidKit over adb forwarding. Prerequisites: the app is foreground and its platform runner is installed and reachable. Failure modes: returns an error when the app is not foreground, AndroidKit/ControlKit is unavailable, or zero or multiple elements match." + )] + async fn ui_tap( + &self, + ::rmcp::handler::server::wrapper::Parameters(args): ::rmcp::handler::server::wrapper::Parameters<$crate::mcp::UiTapArgs>, + ) -> ::std::result::Result<::rmcp::model::CallToolResult, ::rmcp::model::ErrorData> { + let app_id = Self::require_app_id($ui_tap_name, args.target.app_id, args.target.bundle_id)?; + let element = args.element; + if element.trim().is_empty() { + return Err(::rmcp::model::ErrorData::invalid_request("ui_tap requires a non-empty element.", None)); + } + let target = Self::dispatch_target( + args.target.simulator_name, + args.target.simulator_udid, + args.target.host, + args.target.controlkit_port, + args.target.android_serial, + )?; + match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + $crate::AndroidKit::new().call( + device.serial.clone(), "device.ui.tap", ::serde_json::json!({ "appId": app_id, "element": element }), + ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + } + _ => { + let (_, controlkit) = Self::controlkit_for_target(&target)?; + controlkit.call("device.ui.tap", ::serde_json::json!({ "bundleId": app_id, "element": element })) + .await + .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + } + } + Ok(::rmcp::model::CallToolResult::success(vec![ + ::rmcp::model::ContentBlock::text(format!("Tapped '{}' on {}.", element, Self::target_label(&target, &None))), + ])) + } + #[::rmcp::tool( name = $input_tap_name, title = "Tap (iOS/tvOS/Android)", @@ -1900,6 +1990,7 @@ xcrs_mcp_tools!( "url_open", "ui_describe", "ui_element_list", + "ui_tap", "input_tap", "input_text", "input_swipe", @@ -1922,7 +2013,7 @@ impl ServerHandler for XcrsMcpServer { .with_instructions( "xcrs exposes a canonical cross-platform automation contract (device_list, \ device_select, device_capabilities, app_install_launch, screen_capture, \ - app_launch, app_terminate, url_open, ui_describe, ui_element_list, input_tap, \ + app_launch, app_terminate, url_open, ui_describe, ui_element_list, ui_tap, input_tap, \ input_text, input_swipe, input_button, input_click, input_spatial_tap, \ orientation_get, orientation_set) over Xcode/simctl/devicectl/ControlKit for \ Apple platforms and adb for Android.", @@ -1946,9 +2037,9 @@ pub async fn serve() -> Result<()> { mod tests { use super::*; - /// The 18 canonical tool names shared by the standalone and embedded + /// The 19 canonical tool names shared by the standalone and embedded /// automation servers. - const CANONICAL_TOOL_NAMES: [&str; 18] = [ + const CANONICAL_TOOL_NAMES: [&str; 19] = [ "device_list", "device_select", "device_capabilities", @@ -1959,6 +2050,7 @@ mod tests { "url_open", "ui_describe", "ui_element_list", + "ui_tap", "input_tap", "input_text", "input_swipe", @@ -1969,7 +2061,7 @@ mod tests { "orientation_set", ]; - /// Fully-qualified tool names this crate used before the 18-tool + /// Fully-qualified tool names this crate used before the canonical-tool /// consolidation (e.g. `xcrs_list_simulators`, `xcrs_use_target`, /// `xcrs_gesture`). None of these must reappear as a canonical name; a /// match here means a rename regressed back to pre-consolidation naming. @@ -2005,7 +2097,7 @@ mod tests { /// types it does not own, so they intentionally advertise no /// `output_schema`. Only `device_select` returns structured content built /// entirely from types this module owns, so it is the only tool with one. - const TOOLS_WITHOUT_OUTPUT_SCHEMA: [&str; 17] = [ + const TOOLS_WITHOUT_OUTPUT_SCHEMA: [&str; 18] = [ "device_list", "device_capabilities", "app_install_launch", @@ -2015,6 +2107,7 @@ mod tests { "url_open", "ui_describe", "ui_element_list", + "ui_tap", "input_tap", "input_text", "input_swipe", @@ -2026,7 +2119,7 @@ mod tests { ]; #[test] - fn tool_router_exposes_exactly_the_eighteen_canonical_tools() { + fn tool_router_exposes_exactly_the_nineteen_canonical_tools() { let tools = XcrsMcpServer::xcrs_tool_router().list_all(); assert_eq!( @@ -2192,9 +2285,9 @@ mod tests { } #[test] - fn ui_tools_keep_bundle_id_optional_in_the_input_schema() { + fn ui_tools_keep_app_identifiers_optional_in_the_input_schema() { let tools = XcrsMcpServer::xcrs_tool_router().list_all(); - for tool_name in ["ui_describe", "ui_element_list"] { + for tool_name in ["ui_describe", "ui_element_list", "ui_tap"] { let tool = tools .iter() .find(|tool| tool.name.as_ref() == tool_name) @@ -2207,7 +2300,9 @@ mod tests { .unwrap_or_default(); assert!(tool.input_schema["properties"].get("bundle_id").is_some()); + assert!(tool.input_schema["properties"].get("app_id").is_some()); assert!(!required.contains(&serde_json::json!("bundle_id"))); + assert!(!required.contains(&serde_json::json!("app_id"))); } } @@ -2237,20 +2332,21 @@ mod tests { } #[test] - fn ui_target_args_accept_omitted_bundle_id_and_validate_it_explicitly() { + fn ui_target_args_accept_omitted_app_id_and_validate_it_explicitly() { let args: UiTargetArgs = serde_json::from_value(serde_json::json!({})).expect("arguments should deserialize"); assert!(args.bundle_id.is_none()); - let error = XcrsMcpServer::require_bundle_id("ui_describe", args.bundle_id) - .expect_err("missing bundle_id should fail validation"); - assert!(error.to_string().contains("ui_describe requires bundle_id")); + let error = XcrsMcpServer::require_app_id("ui_describe", args.app_id, args.bundle_id) + .expect_err("missing app_id should fail validation"); + assert!(error.to_string().contains("ui_describe requires app_id")); assert_eq!( - XcrsMcpServer::require_bundle_id( + XcrsMcpServer::require_app_id( "ui_describe", + Some(" com.example.app ".to_string()), Some(" com.example.app ".to_string()) ) - .expect("non-empty bundle_id should pass validation"), + .expect("matching app identifiers should pass validation"), "com.example.app" ); } diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index 2f4f5de..3e40223 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -25,6 +25,82 @@ pub fn extract_controlkit_elements(root: &serde_json::Value) -> Vec Vec { + let mut elements = Vec::new(); + if let Some(hierarchy) = root.get("hierarchy").and_then(serde_json::Value::as_array) { + for element in hierarchy { + collect_androidkit_elements(element, &mut elements); + } + } else { + collect_androidkit_elements(root, &mut elements); + } + elements +} + +fn collect_androidkit_elements(element: &serde_json::Value, elements: &mut Vec) { + let Some(object) = element.as_object() else { + return; + }; + + let rect = object.get("rect").and_then(serde_json::Value::as_object); + let has_visible_rect = object + .get("visible") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + && rect + .and_then(|rect| { + Some(( + rect.get("x")?.as_f64()?, + rect.get("y")?.as_f64()?, + rect.get("width")?.as_f64()?, + rect.get("height")?.as_f64()?, + )) + }) + .is_some_and(|(_, _, width, height)| width > 0.0 && height > 0.0); + let text = object + .get("text") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let description = object + .get("contentDescription") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let resource_id = object + .get("resourceId") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let hint = object + .get("hint") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + + if has_visible_rect + && [text, description, resource_id, hint] + .iter() + .any(|value| !value.is_empty()) + { + elements.push(serde_json::json!({ + "type": object.get("role").cloned().unwrap_or(serde_json::Value::Null), + "label": if !text.is_empty() { text } else { description }, + "name": resource_id, + "value": text, + "placeholderValue": hint, + "rawIdentifier": resource_id, + "rect": object.get("rect").cloned().unwrap_or(serde_json::Value::Null), + "enabled": object.get("enabled").cloned().unwrap_or(serde_json::Value::Bool(false)), + "selected": object.get("selected").cloned().unwrap_or(serde_json::Value::Bool(false)), + "hittable": object.get("clickable").cloned().unwrap_or(serde_json::Value::Bool(false)), + })); + } + + if let Some(children) = object.get("children").and_then(serde_json::Value::as_array) { + for child in children { + collect_androidkit_elements(child, elements); + } + } +} + fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec) { let object = match element.as_object() { Some(object) => object, @@ -609,6 +685,92 @@ pub struct AndroidDebugBridge { adb_path: PathBuf, } +/// Client for an XCRS AndroidKit instrumentation runner forwarded over adb. +#[derive(Debug, Clone)] +pub struct AndroidKit { + client: reqwest::Client, +} + +impl Default for AndroidKit { + fn default() -> Self { + Self { + client: reqwest::Client::new(), + } + } +} + +impl AndroidKit { + pub const PACKAGE_NAME: &'static str = "xyz.smbcloud.xcrs.androidkit"; + + pub fn new() -> Self { + Self::default() + } + + pub async fn call( + &self, + serial: String, + method: &str, + params: serde_json::Value, + ) -> Result { + let bridge = AndroidDebugBridge::new(); + let local_port = tokio::task::spawn_blocking({ + let serial = serial.clone(); + move || bridge.forward_androidkit(&serial) + }) + .await + .context("AndroidKit forwarding task failed")??; + + let response = self + .client + .post(format!("http://127.0.0.1:{local_port}/rpc")) + .timeout(Duration::from_secs(10)) + .json(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": method, + "params": params, + })) + .send() + .await + .with_context(|| format!("failed to connect to XCRS AndroidKit for {serial}")); + + let cleanup_bridge = AndroidDebugBridge::new(); + let cleanup_serial = serial.clone(); + let cleanup = tokio::task::spawn_blocking(move || { + cleanup_bridge.remove_forward(&cleanup_serial, local_port) + }) + .await + .context("AndroidKit forwarding cleanup task failed")?; + cleanup.context("failed to remove AndroidKit adb forward")?; + + let response = response?; + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .context("failed to decode AndroidKit JSON-RPC response")?; + if !status.is_success() { + return Err(anyhow!("AndroidKit returned HTTP {status}: {body}")); + } + if let Some(error) = body.get("error") { + let code = error + .get("code") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + let message = error + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown JSON-RPC error"); + return Err(anyhow!( + "AndroidKit method '{method}' failed ({code}): {message}" + )); + } + body.get("result") + .cloned() + .ok_or_else(|| anyhow!("AndroidKit response for '{method}' did not contain a result")) + } +} + impl Default for AndroidDebugBridge { fn default() -> Self { Self { @@ -772,6 +934,86 @@ impl AndroidDebugBridge { Ok(()) } + pub fn install_androidkit(&self, serial: &str, apk_path: &Path) -> Result<()> { + if !apk_path.is_file() { + return Err(anyhow!( + "AndroidKit APK does not exist: {}", + apk_path.display() + )); + } + run_command( + &self.adb_path, + [ + "-s", + serial, + "install", + "-r", + apk_path + .to_str() + .ok_or_else(|| anyhow!("AndroidKit APK path is not valid UTF-8"))?, + ], + )?; + Ok(()) + } + + pub fn start_androidkit(&self, serial: &str) -> Result<()> { + Command::new(&self.adb_path) + .args([ + "-s", + serial, + "shell", + "am", + "instrument", + "-w", + "-r", + "xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .with_context(|| format!("failed to start AndroidKit on {serial}"))?; + Ok(()) + } + + pub fn stop_androidkit(&self, serial: &str) -> Result<()> { + self.run_shell( + serial, + &format!("am force-stop {}", AndroidKit::PACKAGE_NAME), + )?; + Ok(()) + } + + fn forward_androidkit(&self, serial: &str) -> Result { + let output = run_command( + &self.adb_path, + [ + "-s", + serial, + "forward", + "tcp:0", + "localabstract:xcrs-androidkit", + ], + )?; + output + .trim() + .parse::() + .with_context(|| format!("adb returned an invalid AndroidKit forward port: {output:?}")) + } + + fn remove_forward(&self, serial: &str, local_port: u16) -> Result<()> { + run_command( + &self.adb_path, + [ + "-s", + serial, + "forward", + "--remove", + &format!("tcp:{local_port}"), + ], + )?; + Ok(()) + } + fn run_shell(&self, serial: &str, command: &str) -> Result { run_command(&self.adb_path, ["-s", serial, "shell", command]) } @@ -1455,6 +1697,33 @@ mod tests { assert_eq!(elements[0]["hittable"], serde_json::json!(true)); } + #[test] + fn normalizes_visible_androidkit_elements() { + let hierarchy = serde_json::json!({ + "hierarchy": [{ + "role": "android.widget.Button", + "text": "Continue", + "hint": "", + "contentDescription": "", + "resourceId": "com.example:id/continue", + "enabled": true, + "clickable": true, + "selected": false, + "visible": true, + "rect": { "x": 10, "y": 20, "width": 100, "height": 40 } + }] + }); + + let elements = extract_androidkit_elements(&hierarchy); + + assert_eq!(elements.len(), 1); + assert_eq!(elements[0]["label"], serde_json::json!("Continue")); + assert_eq!( + elements[0]["rawIdentifier"], + serde_json::json!("com.example:id/continue") + ); + } + #[test] fn parses_device_tunnel_address() { let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n"; diff --git a/docs/androidkit.md b/docs/androidkit.md new file mode 100644 index 0000000..900c952 --- /dev/null +++ b/docs/androidkit.md @@ -0,0 +1,27 @@ +# XCRS AndroidKit + +XCRS AndroidKit is the optional Android companion runner for semantic E2E +automation. It is separate from adb: adb remains sufficient for screenshots, +coordinate input, app lifecycle, and system buttons. + +Build an AndroidKit APK from the public `xcrs-androidkit` repository, then +install and start it on the selected Android device: + +```sh +xcrs android-kit install --serial --apk +xcrs android-kit start --serial +xcrs android-kit status --serial +``` + +AndroidKit binds only to an Android local socket. xcrs creates and removes its +own `adb forward` connection for each RPC call; it does not expose a network +service on the device. + +When AndroidKit is available, `ui_describe`, `ui_element_list`, and `ui_tap` +accept `app_id` as the Android package name. `ui_tap` uses an exact accessible +text, content description, resource identifier, or hint and fails if it finds +zero or more than one visible enabled target. The app must be foreground. + +`bundle_id` remains accepted as a compatibility alias for Apple callers. New +cross-platform clients should send `app_id`; passing different `app_id` and +`bundle_id` values is invalid. diff --git a/docs/mcp.md b/docs/mcp.md index f681831..5d7e2f3 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -276,8 +276,8 @@ them by server. 2. Call `device_select` with the Apple simulator/runner details or Android adb serial. Later actions use that target by default. 3. Call `device_capabilities` before choosing an input or inspection action. -4. Use `screen_capture` or the Apple-only `ui_describe`/`ui_element_list` to - inspect the app, then drive it with the `input_*` tools. +4. Use `screen_capture` or `ui_describe`/`ui_element_list` to + inspect the app, then use semantic `ui_tap` or the raw `input_*` tools. | Tool | Purpose | | --- | --- | @@ -288,14 +288,15 @@ them by server. | `app_launch` / `app_terminate` | Start or stop an installed app on the selected target. | | `screen_capture` | Capture the selected target as a PNG image. | | `url_open` | Open an HTTP(S) URL or custom scheme where the target supports it. | -| `ui_describe` / `ui_element_list` | Inspect the Apple ControlKit accessibility hierarchy. | +| `ui_describe` / `ui_element_list` / `ui_tap` | Inspect or activate accessible elements through ControlKit (Apple) or AndroidKit (Android). | | `input_tap` / `input_text` / `input_swipe` / `input_button` | Drive shared touch, text, gesture, and button actions. | | `input_click` | Click a macOS target with pointer coordinates. | | `input_spatial_tap` | Perform a visionOS spatial tap. | | `orientation_get` / `orientation_set` | Read or update orientation where supported. | -See [ControlKit runners](./controlkit.md) for Apple runner setup. Android tools -require adb and an authorized device; Android TV app launches resolve the +See [ControlKit runners](./controlkit.md) for Apple runner setup and +[XCRS AndroidKit](./androidkit.md) for Android semantic UI automation. Android +tools require adb and an authorized device; Android TV app launches resolve the device's Leanback launcher activity. #### Automation migration in 0.5.0 From e69cb000b85e6270ea7dfbeebc25552dc891fbf6 Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:49:10 +0200 Subject: [PATCH 2/4] Keep AndroidKit instrumentation attached Co-Authored-By: siGit Code --- crates/xcrs/src/xcrs.rs | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index 3e40223..d112c89 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -957,21 +957,11 @@ impl AndroidDebugBridge { } pub fn start_androidkit(&self, serial: &str) -> Result<()> { - Command::new(&self.adb_path) - .args([ - "-s", - serial, - "shell", - "am", - "instrument", - "-w", - "-r", - "xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation", - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .with_context(|| format!("failed to start AndroidKit on {serial}"))?; + self.run_shell( + serial, + "nohup am instrument -w -r xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation >/dev/null 2>&1 &", + ) + .with_context(|| format!("failed to start AndroidKit on {serial}"))?; Ok(()) } From 074ea15f181432413d8c00b66526e886ded80db4 Mon Sep 17 00:00:00 2001 From: Seto Elkahfi <1797197+setoelkahfi@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:47:13 +0200 Subject: [PATCH 3/4] Wait for AndroidKit readiness after startup Co-Authored-By: siGit Code --- crates/xcrs/src/main.rs | 2 +- crates/xcrs/src/xcrs.rs | 63 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/crates/xcrs/src/main.rs b/crates/xcrs/src/main.rs index df2a233..0d2fad3 100644 --- a/crates/xcrs/src/main.rs +++ b/crates/xcrs/src/main.rs @@ -94,7 +94,7 @@ async fn main() -> Result<()> { println!("Installed XCRS AndroidKit on {serial}"); } Commands::AndroidKit(AndroidKitCommand::Start { serial }) => { - AndroidDebugBridge::new().start_androidkit(&serial)?; + AndroidDebugBridge::new().start_androidkit(&serial).await?; println!("Started XCRS AndroidKit on {serial}"); } Commands::AndroidKit(AndroidKitCommand::Status { serial, json }) => { diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index d112c89..5a4b58d 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -12,6 +12,10 @@ pub mod mcp; const IOS_SIMULATOR_DESTINATION_PREFIX: &str = "platform=iOS Simulator,id="; const CONTROLKIT_METHOD_NOT_FOUND: i64 = -32601; const CONTROLKIT_RUNNER_INFO_TIMEOUT: Duration = Duration::from_secs(2); +const ANDROIDKIT_CALL_TIMEOUT: Duration = Duration::from_secs(10); +const ANDROIDKIT_START_TIMEOUT: Duration = Duration::from_secs(10); +const ANDROIDKIT_START_PROBE_TIMEOUT: Duration = Duration::from_millis(500); +const ANDROIDKIT_START_RETRY_DELAY: Duration = Duration::from_millis(100); pub fn encode_base64(data: impl AsRef<[u8]>) -> String { use base64::{engine::general_purpose::STANDARD, Engine}; @@ -689,12 +693,14 @@ pub struct AndroidDebugBridge { #[derive(Debug, Clone)] pub struct AndroidKit { client: reqwest::Client, + adb_path: PathBuf, } impl Default for AndroidKit { fn default() -> Self { Self { client: reqwest::Client::new(), + adb_path: discover_adb_path(), } } } @@ -712,7 +718,25 @@ impl AndroidKit { method: &str, params: serde_json::Value, ) -> Result { - let bridge = AndroidDebugBridge::new(); + self.call_with_timeout(serial, method, params, ANDROIDKIT_CALL_TIMEOUT) + .await + } + + fn with_adb_path(adb_path: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + adb_path: adb_path.into(), + } + } + + async fn call_with_timeout( + &self, + serial: String, + method: &str, + params: serde_json::Value, + timeout: Duration, + ) -> Result { + let bridge = AndroidDebugBridge::with_path(self.adb_path.clone()); let local_port = tokio::task::spawn_blocking({ let serial = serial.clone(); move || bridge.forward_androidkit(&serial) @@ -723,7 +747,7 @@ impl AndroidKit { let response = self .client .post(format!("http://127.0.0.1:{local_port}/rpc")) - .timeout(Duration::from_secs(10)) + .timeout(timeout) .json(&serde_json::json!({ "jsonrpc": "2.0", "id": 1, @@ -734,7 +758,7 @@ impl AndroidKit { .await .with_context(|| format!("failed to connect to XCRS AndroidKit for {serial}")); - let cleanup_bridge = AndroidDebugBridge::new(); + let cleanup_bridge = AndroidDebugBridge::with_path(self.adb_path.clone()); let cleanup_serial = serial.clone(); let cleanup = tokio::task::spawn_blocking(move || { cleanup_bridge.remove_forward(&cleanup_serial, local_port) @@ -769,6 +793,32 @@ impl AndroidKit { .cloned() .ok_or_else(|| anyhow!("AndroidKit response for '{method}' did not contain a result")) } + + async fn wait_until_ready(&self, serial: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + ANDROIDKIT_START_TIMEOUT; + loop { + match self + .call_with_timeout( + serial.to_string(), + "device.info", + serde_json::json!({}), + ANDROIDKIT_START_PROBE_TIMEOUT, + ) + .await + { + Ok(_) => return Ok(()), + Err(error) if tokio::time::Instant::now() >= deadline => { + return Err(error).with_context(|| { + format!( + "AndroidKit did not become ready on {serial} within {} seconds", + ANDROIDKIT_START_TIMEOUT.as_secs() + ) + }); + } + Err(_) => tokio::time::sleep(ANDROIDKIT_START_RETRY_DELAY).await, + } + } + } } impl Default for AndroidDebugBridge { @@ -956,13 +1006,16 @@ impl AndroidDebugBridge { Ok(()) } - pub fn start_androidkit(&self, serial: &str) -> Result<()> { + pub async fn start_androidkit(&self, serial: &str) -> Result<()> { self.run_shell( serial, "nohup am instrument -w -r xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation >/dev/null 2>&1 &", ) .with_context(|| format!("failed to start AndroidKit on {serial}"))?; - Ok(()) + AndroidKit::with_adb_path(self.adb_path.clone()) + .wait_until_ready(serial) + .await + .with_context(|| format!("failed to start AndroidKit on {serial}")) } pub fn stop_androidkit(&self, serial: &str) -> Result<()> { From c351c92a9d1ce70d093a5136646687c26b1e7446 Mon Sep 17 00:00:00 2001 From: Seto Elkahfi Date: Thu, 10 Sep 2026 17:02:12 +0200 Subject: [PATCH 4/4] Update README.md --- crates/xcrs/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/xcrs/README.md b/crates/xcrs/README.md index 9616ed1..20be931 100644 --- a/crates/xcrs/README.md +++ b/crates/xcrs/README.md @@ -5,8 +5,6 @@ physical Apple devices, Android phones, and Android TV devices. XCRS combines Xcode, ControlKit, CoreDevice, and adb behind one target-aware automation workflow. -https://mobilenext.ai/docs/architecture#how-a-single-action-flows - MCP Registry name: `mcp-name: io.github.smbcloudXYZ/xcrs` ## Run the MCP server