diff --git a/crates/xcrs/README.md b/crates/xcrs/README.md index e2fb0dd..20be931 100644 --- a/crates/xcrs/README.md +++ b/crates/xcrs/README.md @@ -18,6 +18,10 @@ Android tools use `adb` from `ANDROID_SDK_ROOT`, `ANDROID_HOME`, the standard Android SDK locations, or `PATH`. Connect a device with USB debugging enabled and authorize the host before selecting it with `device_select`. +For Android accessibility-tree inspection and semantic `ui_tap`, install the +optional [XCRS AndroidKit](../../docs/androidkit.md) companion runner. Raw adb +actions remain available without it. + The same automation profile is available from the full smbCloud CLI: ```sh diff --git a/crates/xcrs/src/main.rs b/crates/xcrs/src/main.rs index 9f3b8ee..0d2fad3 100644 --- a/crates/xcrs/src/main.rs +++ b/crates/xcrs/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use std::path::PathBuf; -use xcrs::{IosAppTest, XcodeCommandLineTools}; +use xcrs::{AndroidDebugBridge, AndroidKit, IosAppTest, XcodeCommandLineTools}; #[derive(Debug, Parser)] #[command(name = "xcrs")] @@ -19,6 +19,8 @@ struct Cli { enum Commands { #[command(subcommand)] Simulator(SimulatorCommand), + #[command(subcommand)] + AndroidKit(AndroidKitCommand), IosAppTest { #[arg(long, conflicts_with = "simulator_udid")] simulator_name: Option, @@ -35,6 +37,30 @@ enum Commands { }, } +#[derive(Debug, Subcommand)] +enum AndroidKitCommand { + Install { + #[arg(long)] + serial: String, + #[arg(long)] + apk: PathBuf, + }, + Start { + #[arg(long)] + serial: String, + }, + Status { + #[arg(long)] + serial: String, + #[arg(long)] + json: bool, + }, + Stop { + #[arg(long)] + serial: String, + }, +} + #[derive(Debug, Subcommand)] enum SimulatorCommand { Find { @@ -63,6 +89,28 @@ async fn main() -> Result<()> { let tools = XcodeCommandLineTools::new(); match command { + Commands::AndroidKit(AndroidKitCommand::Install { serial, apk }) => { + AndroidDebugBridge::new().install_androidkit(&serial, &apk)?; + println!("Installed XCRS AndroidKit on {serial}"); + } + Commands::AndroidKit(AndroidKitCommand::Start { serial }) => { + AndroidDebugBridge::new().start_androidkit(&serial).await?; + println!("Started XCRS AndroidKit on {serial}"); + } + Commands::AndroidKit(AndroidKitCommand::Status { serial, json }) => { + let info = AndroidKit::new() + .call(serial.clone(), "device.info", serde_json::json!({})) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&info)?); + } else { + println!("XCRS AndroidKit is ready on {serial}"); + } + } + Commands::AndroidKit(AndroidKitCommand::Stop { serial }) => { + AndroidDebugBridge::new().stop_androidkit(&serial)?; + println!("Stopped XCRS AndroidKit on {serial}"); + } Commands::Simulator(SimulatorCommand::Find { name, json }) => { let simulator = tools.simctl().find_simulator_by_name(&name)?; if json { diff --git a/crates/xcrs/src/mcp.rs b/crates/xcrs/src/mcp.rs index 85a34aa..13636cd 100644 --- a/crates/xcrs/src/mcp.rs +++ b/crates/xcrs/src/mcp.rs @@ -314,6 +314,7 @@ pub fn android_capabilities() -> serde_json::Value { "app_install_launch": false, "ui_describe": false, "ui_element_list": false, + "ui_tap": false, "orientation_get": false, "orientation_set": false, "input_click": false, @@ -602,31 +603,22 @@ pub struct UiTargetArgs { /// Local ControlKit JSON-RPC port. Defaults to 12004. #[serde(default)] pub controlkit_port: Option, - /// Bundle identifier of the app whose accessibility hierarchy should be read. + /// Android device serial from `adb devices -l`. + #[serde(default)] + pub android_serial: Option, + /// Target-neutral app identifier: an Apple bundle identifier or Android package name. + #[serde(default)] + pub app_id: Option, + /// Backward-compatible alias for `app_id` on Apple targets. #[serde(default)] pub bundle_id: Option, } #[derive(Debug, Deserialize, JsonSchema)] pub struct UiTapArgs { - /// Exact Apple simulator name. Omit to use the target selected with - /// `device_select`. - #[serde(default)] - pub simulator_name: Option, - /// Apple simulator UDID. Omit to use the target selected with `device_select`. - #[serde(default)] - pub simulator_udid: Option, - /// ControlKit host of a physical device or remote runner. Omit for a local - /// simulator. - #[serde(default)] - pub host: Option, - /// Local ControlKit JSON-RPC port. Defaults to 12004. - #[serde(default)] - pub controlkit_port: Option, - /// Bundle identifier of the foreground app that owns the element. - #[serde(default)] - pub bundle_id: Option, - /// Accessibility label, identifier, or value of the element to activate. + #[serde(flatten)] + pub target: UiTargetArgs, + /// Exact accessible text, description, identifier, value, or hint to activate. pub element: String, } @@ -721,23 +713,29 @@ macro_rules! xcrs_mcp_tools { .and_then(|slot| slot.clone()) } - fn require_bundle_id( + fn require_app_id( tool_name: &str, + app_id: Option, bundle_id: Option, ) -> ::std::result::Result { - let bundle_id = bundle_id - .as_deref() - .map(str::trim) - .filter(|bundle_id| !bundle_id.is_empty()) - .ok_or_else(|| { + let app_id = app_id.as_deref().map(str::trim).filter(|value| !value.is_empty()); + let bundle_id = bundle_id.as_deref().map(str::trim).filter(|value| !value.is_empty()); + if let (Some(app_id), Some(bundle_id)) = (app_id, bundle_id) { + if app_id != bundle_id { + return Err(::rmcp::model::ErrorData::invalid_request( + format!("{tool_name} received conflicting app_id and bundle_id values."), + None, + )); + } + } + app_id.or(bundle_id).map(str::to_string).ok_or_else(|| { ::rmcp::model::ErrorData::invalid_request( format!( - "{tool_name} requires bundle_id. Pass the bundle identifier of the foreground Apple app." + "{tool_name} requires app_id. Pass the Apple bundle identifier or Android package name of the foreground app." ), None, ) - })?; - Ok(bundle_id.to_string()) + }) } /// Validate and tag a target from per-call fields, falling back to the @@ -971,7 +969,7 @@ macro_rules! xcrs_mcp_tools { name = $device_capabilities_name, title = "Get device capabilities", annotations(title = "Get device capabilities", read_only_hint = true, idempotent_hint = true), - description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, returns a static capability description reflecting exactly the adb actions this crate implements (screen capture, app launch/terminate, URL open, tap/text/swipe/button), since adb has no capability-discovery RPC. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out." + description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, starts from the adb actions this crate implements, then probes XCRS AndroidKit and enables semantic UI actions when its instrumentation runner is reachable. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device, and XCRS AndroidKit must be installed and started for semantic UI actions. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out." )] async fn device_capabilities( &self, @@ -992,11 +990,34 @@ macro_rules! xcrs_mcp_tools { args.android_serial, )?; match &target { - $crate::mcp::SelectedTarget::Android { .. } => { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let mut capabilities = $crate::mcp::android_capabilities(); + let androidkit = $crate::AndroidKit::new() + .call(device.serial.clone(), "device.info", ::serde_json::json!({})) + .await; + if let Some(object) = capabilities.as_object_mut() { + match androidkit { + Ok(info) => { + object.insert("ui_describe".to_string(), ::serde_json::json!(true)); + object.insert("ui_element_list".to_string(), ::serde_json::json!(true)); + object.insert("ui_tap".to_string(), ::serde_json::json!(true)); + object.insert("androidkit".to_string(), ::serde_json::json!({ "available": true, "info": info })); + } + Err(error) => { + object.insert("ui_tap".to_string(), ::serde_json::json!(false)); + object.insert("androidkit".to_string(), ::serde_json::json!({ + "available": false, + "setup": "Install and start XCRS AndroidKit with xcrs android-kit install/start.", + "error": error.to_string(), + })); + } + } + } Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "platform": "android", - "capabilities": $crate::mcp::android_capabilities(), + "capabilities": capabilities, }))?, ])) } @@ -1381,7 +1402,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_describe_name, title = "Describe UI", annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true), - description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." + description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground." )] async fn ui_describe( &self, @@ -1394,30 +1415,35 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { - let bundle_id = Self::require_bundle_id($ui_describe_name, args.bundle_id)?; - let target = Self::dispatch_apple_target( - $ui_describe_name, + let app_id = Self::require_app_id($ui_describe_name, args.app_id, args.bundle_id)?; + let target = Self::dispatch_target( args.simulator_name, args.simulator_udid, args.host, args.controlkit_port, + args.android_serial, )?; - let (simulator, controlkit) = Self::controlkit_for_target(&target)?; - let result = controlkit - .call( - "device.dump.ui", - ::serde_json::json!({ - "format": "json", - "bundleId": bundle_id, - }), - ) - .await - .map_err(|error| { - ::rmcp::model::ErrorData::internal_error(error.to_string(), None) - })?; + let (simulator, result) = match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let ui = $crate::AndroidKit::new() + .call(device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id })) + .await + .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (None, ui) + } + _ => { + let (simulator, controlkit) = Self::controlkit_for_target(&target)?; + let ui = controlkit.call("device.dump.ui", ::serde_json::json!({ + "format": "json", "bundleId": app_id, + })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (simulator, ui) + } + }; Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "simulator": simulator, + "target": target, "ui": result, }))?, ])) @@ -1427,7 +1453,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_element_list_name, title = "List UI elements", annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true), - description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." + description = "Purpose: list just the actionable accessibility elements of the foreground app with labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit; then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground." )] async fn ui_element_list( &self, @@ -1440,31 +1466,30 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { - let bundle_id = Self::require_bundle_id($ui_element_list_name, args.bundle_id)?; - let target = Self::dispatch_apple_target( - $ui_element_list_name, - args.simulator_name, - args.simulator_udid, - args.host, - args.controlkit_port, + let app_id = Self::require_app_id($ui_element_list_name, args.app_id, args.bundle_id)?; + let target = Self::dispatch_target( + args.simulator_name, args.simulator_udid, args.host, args.controlkit_port, args.android_serial, )?; - let (simulator, controlkit) = Self::controlkit_for_target(&target)?; - let ui = controlkit - .call( - "device.dump.ui", - ::serde_json::json!({ - "format": "json", - "bundleId": bundle_id, - }), - ) - .await - .map_err(|error| { - ::rmcp::model::ErrorData::internal_error(error.to_string(), None) - })?; - let elements = $crate::extract_controlkit_elements(&ui); + let (simulator, elements) = match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + let ui = $crate::AndroidKit::new().call( + device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id }), + ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (None, $crate::extract_androidkit_elements(&ui)) + } + _ => { + let (simulator, controlkit) = Self::controlkit_for_target(&target)?; + let ui = controlkit.call("device.dump.ui", ::serde_json::json!({ + "format": "json", "bundleId": app_id, + })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + (simulator, $crate::extract_controlkit_elements(&ui)) + } + }; Ok(::rmcp::model::CallToolResult::success(vec![ ::rmcp::model::ContentBlock::json(&::serde_json::json!({ "simulator": simulator, + "target": target, "elements": elements, }))?, ])) @@ -1474,53 +1499,45 @@ macro_rules! xcrs_mcp_tools { name = $ui_tap_name, title = "Tap UI element", annotations(title = "Tap UI element", read_only_hint = false, destructive_hint = false, idempotent_hint = false), - description = "Purpose: activate a named accessibility element in an Apple app as one action. When to use vs siblings: prefer this for a known button, link, or control; use ui_describe or ui_element_list when you first need to inspect the screen, and input_tap when you specifically need coordinate input. Behavior: forwards bundle_id and element to ControlKit `device.ui.tap`; the runner attaches to the foreground app, requires exactly one element matching its accessibility label, identifier, or value, and activates it with the platform interaction API. On tvOS, the matching element must already have focus and the runner presses Select. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.ui.tap`; bundle_id must identify the foreground app. Failure modes: errors if no Apple target can be resolved, the app is not foreground, no unique matching element is available, the tvOS element is not focused, or the runner is outdated or unavailable." + description = "Purpose: activate one uniquely identified accessible element in the foreground app. When to use vs siblings: use this for semantic interaction by label or identifier; use ui_describe or ui_element_list when you first need to inspect the screen, and input_tap only when you deliberately need raw screen coordinates. Behavior: accepts app_id (or Apple-compatible bundle_id) and an exact element value, and requires exactly one match. Apple targets go through ControlKit `device.ui.tap`, which attaches to the foreground app and activates the match with the platform interaction API; on tvOS the match must already have focus and the runner presses Select. Android targets use XCRS AndroidKit over adb forwarding. Prerequisites: the app is foreground and its platform runner is installed and reachable. Failure modes: returns an error when the app is not foreground, AndroidKit/ControlKit is unavailable or outdated, zero or multiple elements match, or a tvOS element is not focused." )] async fn ui_tap( &self, - ::rmcp::handler::server::wrapper::Parameters( - args, - ): ::rmcp::handler::server::wrapper::Parameters< - $crate::mcp::UiTapArgs, - >, - ) -> ::std::result::Result< - ::rmcp::model::CallToolResult, - ::rmcp::model::ErrorData, - > { - let bundle_id = Self::require_bundle_id($ui_tap_name, args.bundle_id)?; - let element = args.element.trim(); + ::rmcp::handler::server::wrapper::Parameters(args): ::rmcp::handler::server::wrapper::Parameters<$crate::mcp::UiTapArgs>, + ) -> ::std::result::Result<::rmcp::model::CallToolResult, ::rmcp::model::ErrorData> { + let app_id = Self::require_app_id($ui_tap_name, args.target.app_id, args.target.bundle_id)?; + let element = args.element.trim().to_string(); if element.is_empty() { return Err(::rmcp::model::ErrorData::invalid_request( - format!("{} requires a non-empty element", $ui_tap_name), + format!("{} requires a non-empty element.", $ui_tap_name), None, )); } - let target = Self::dispatch_apple_target( - $ui_tap_name, - args.simulator_name, - args.simulator_udid, - args.host, - args.controlkit_port, + let target = Self::dispatch_target( + args.target.simulator_name, + args.target.simulator_udid, + args.target.host, + args.target.controlkit_port, + args.target.android_serial, )?; - let (simulator, controlkit) = Self::controlkit_for_target(&target)?; - controlkit - .call( - "device.ui.tap", - ::serde_json::json!({ - "bundleId": bundle_id, - "element": element, - }), - ) - .await - .map_err(|error| { - ::rmcp::model::ErrorData::internal_error(error.to_string(), None) - })?; - Ok(::rmcp::model::CallToolResult::success(vec![ - ::rmcp::model::ContentBlock::text(format!( - "Activated {} on {}.", - element, + let label = match &target { + $crate::mcp::SelectedTarget::Android { serial } => { + let device = Self::android_device_for(serial.clone()).await?; + $crate::AndroidKit::new().call( + device.serial.clone(), "device.ui.tap", ::serde_json::json!({ "appId": app_id, "element": element }), + ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; + device.serial + } + _ => { + let (simulator, controlkit) = Self::controlkit_for_target(&target)?; + controlkit.call("device.ui.tap", ::serde_json::json!({ "bundleId": app_id, "element": element })) + .await + .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?; Self::target_label(&target, &simulator) - )), + } + }; + Ok(::rmcp::model::CallToolResult::success(vec![ + ::rmcp::model::ContentBlock::text(format!("Tapped '{}' on {}.", element, label)), ])) } @@ -2001,7 +2018,7 @@ impl ServerHandler for XcrsMcpServer { .with_instructions( "xcrs exposes a canonical cross-platform automation contract (device_list, \ device_select, device_capabilities, app_install_launch, screen_capture, \ - app_launch, app_terminate, url_open, ui_describe, ui_element_list, input_tap, \ + app_launch, app_terminate, url_open, ui_describe, ui_element_list, ui_tap, input_tap, \ input_text, input_swipe, input_button, input_click, input_spatial_tap, \ orientation_get, orientation_set) over Xcode/simctl/devicectl/ControlKit for \ Apple platforms and adb for Android.", @@ -2049,7 +2066,7 @@ mod tests { "orientation_set", ]; - /// Fully-qualified tool names this crate used before the 18-tool + /// Fully-qualified tool names this crate used before the canonical-tool /// consolidation (e.g. `xcrs_list_simulators`, `xcrs_use_target`, /// `xcrs_gesture`). None of these must reappear as a canonical name; a /// match here means a rename regressed back to pre-consolidation naming. @@ -2273,9 +2290,9 @@ mod tests { } #[test] - fn ui_tools_keep_bundle_id_optional_in_the_input_schema() { + fn ui_tools_keep_app_identifiers_optional_in_the_input_schema() { let tools = XcrsMcpServer::xcrs_tool_router().list_all(); - for tool_name in ["ui_describe", "ui_element_list"] { + for tool_name in ["ui_describe", "ui_element_list", "ui_tap"] { let tool = tools .iter() .find(|tool| tool.name.as_ref() == tool_name) @@ -2288,7 +2305,9 @@ mod tests { .unwrap_or_default(); assert!(tool.input_schema["properties"].get("bundle_id").is_some()); + assert!(tool.input_schema["properties"].get("app_id").is_some()); assert!(!required.contains(&serde_json::json!("bundle_id"))); + assert!(!required.contains(&serde_json::json!("app_id"))); } } @@ -2318,20 +2337,21 @@ mod tests { } #[test] - fn ui_target_args_accept_omitted_bundle_id_and_validate_it_explicitly() { + fn ui_target_args_accept_omitted_app_id_and_validate_it_explicitly() { let args: UiTargetArgs = serde_json::from_value(serde_json::json!({})).expect("arguments should deserialize"); assert!(args.bundle_id.is_none()); - let error = XcrsMcpServer::require_bundle_id("ui_describe", args.bundle_id) - .expect_err("missing bundle_id should fail validation"); - assert!(error.to_string().contains("ui_describe requires bundle_id")); + let error = XcrsMcpServer::require_app_id("ui_describe", args.app_id, args.bundle_id) + .expect_err("missing app_id should fail validation"); + assert!(error.to_string().contains("ui_describe requires app_id")); assert_eq!( - XcrsMcpServer::require_bundle_id( + XcrsMcpServer::require_app_id( "ui_describe", + Some(" com.example.app ".to_string()), Some(" com.example.app ".to_string()) ) - .expect("non-empty bundle_id should pass validation"), + .expect("matching app identifiers should pass validation"), "com.example.app" ); } diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index 2f4f5de..5a4b58d 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -12,6 +12,10 @@ pub mod mcp; const IOS_SIMULATOR_DESTINATION_PREFIX: &str = "platform=iOS Simulator,id="; const CONTROLKIT_METHOD_NOT_FOUND: i64 = -32601; const CONTROLKIT_RUNNER_INFO_TIMEOUT: Duration = Duration::from_secs(2); +const ANDROIDKIT_CALL_TIMEOUT: Duration = Duration::from_secs(10); +const ANDROIDKIT_START_TIMEOUT: Duration = Duration::from_secs(10); +const ANDROIDKIT_START_PROBE_TIMEOUT: Duration = Duration::from_millis(500); +const ANDROIDKIT_START_RETRY_DELAY: Duration = Duration::from_millis(100); pub fn encode_base64(data: impl AsRef<[u8]>) -> String { use base64::{engine::general_purpose::STANDARD, Engine}; @@ -25,6 +29,82 @@ pub fn extract_controlkit_elements(root: &serde_json::Value) -> Vec Vec { + let mut elements = Vec::new(); + if let Some(hierarchy) = root.get("hierarchy").and_then(serde_json::Value::as_array) { + for element in hierarchy { + collect_androidkit_elements(element, &mut elements); + } + } else { + collect_androidkit_elements(root, &mut elements); + } + elements +} + +fn collect_androidkit_elements(element: &serde_json::Value, elements: &mut Vec) { + let Some(object) = element.as_object() else { + return; + }; + + let rect = object.get("rect").and_then(serde_json::Value::as_object); + let has_visible_rect = object + .get("visible") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + && rect + .and_then(|rect| { + Some(( + rect.get("x")?.as_f64()?, + rect.get("y")?.as_f64()?, + rect.get("width")?.as_f64()?, + rect.get("height")?.as_f64()?, + )) + }) + .is_some_and(|(_, _, width, height)| width > 0.0 && height > 0.0); + let text = object + .get("text") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let description = object + .get("contentDescription") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let resource_id = object + .get("resourceId") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let hint = object + .get("hint") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + + if has_visible_rect + && [text, description, resource_id, hint] + .iter() + .any(|value| !value.is_empty()) + { + elements.push(serde_json::json!({ + "type": object.get("role").cloned().unwrap_or(serde_json::Value::Null), + "label": if !text.is_empty() { text } else { description }, + "name": resource_id, + "value": text, + "placeholderValue": hint, + "rawIdentifier": resource_id, + "rect": object.get("rect").cloned().unwrap_or(serde_json::Value::Null), + "enabled": object.get("enabled").cloned().unwrap_or(serde_json::Value::Bool(false)), + "selected": object.get("selected").cloned().unwrap_or(serde_json::Value::Bool(false)), + "hittable": object.get("clickable").cloned().unwrap_or(serde_json::Value::Bool(false)), + })); + } + + if let Some(children) = object.get("children").and_then(serde_json::Value::as_array) { + for child in children { + collect_androidkit_elements(child, elements); + } + } +} + fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec) { let object = match element.as_object() { Some(object) => object, @@ -609,6 +689,138 @@ pub struct AndroidDebugBridge { adb_path: PathBuf, } +/// Client for an XCRS AndroidKit instrumentation runner forwarded over adb. +#[derive(Debug, Clone)] +pub struct AndroidKit { + client: reqwest::Client, + adb_path: PathBuf, +} + +impl Default for AndroidKit { + fn default() -> Self { + Self { + client: reqwest::Client::new(), + adb_path: discover_adb_path(), + } + } +} + +impl AndroidKit { + pub const PACKAGE_NAME: &'static str = "xyz.smbcloud.xcrs.androidkit"; + + pub fn new() -> Self { + Self::default() + } + + pub async fn call( + &self, + serial: String, + method: &str, + params: serde_json::Value, + ) -> Result { + self.call_with_timeout(serial, method, params, ANDROIDKIT_CALL_TIMEOUT) + .await + } + + fn with_adb_path(adb_path: impl Into) -> Self { + Self { + client: reqwest::Client::new(), + adb_path: adb_path.into(), + } + } + + async fn call_with_timeout( + &self, + serial: String, + method: &str, + params: serde_json::Value, + timeout: Duration, + ) -> Result { + let bridge = AndroidDebugBridge::with_path(self.adb_path.clone()); + let local_port = tokio::task::spawn_blocking({ + let serial = serial.clone(); + move || bridge.forward_androidkit(&serial) + }) + .await + .context("AndroidKit forwarding task failed")??; + + let response = self + .client + .post(format!("http://127.0.0.1:{local_port}/rpc")) + .timeout(timeout) + .json(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": method, + "params": params, + })) + .send() + .await + .with_context(|| format!("failed to connect to XCRS AndroidKit for {serial}")); + + let cleanup_bridge = AndroidDebugBridge::with_path(self.adb_path.clone()); + let cleanup_serial = serial.clone(); + let cleanup = tokio::task::spawn_blocking(move || { + cleanup_bridge.remove_forward(&cleanup_serial, local_port) + }) + .await + .context("AndroidKit forwarding cleanup task failed")?; + cleanup.context("failed to remove AndroidKit adb forward")?; + + let response = response?; + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .context("failed to decode AndroidKit JSON-RPC response")?; + if !status.is_success() { + return Err(anyhow!("AndroidKit returned HTTP {status}: {body}")); + } + if let Some(error) = body.get("error") { + let code = error + .get("code") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + let message = error + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown JSON-RPC error"); + return Err(anyhow!( + "AndroidKit method '{method}' failed ({code}): {message}" + )); + } + body.get("result") + .cloned() + .ok_or_else(|| anyhow!("AndroidKit response for '{method}' did not contain a result")) + } + + async fn wait_until_ready(&self, serial: &str) -> Result<()> { + let deadline = tokio::time::Instant::now() + ANDROIDKIT_START_TIMEOUT; + loop { + match self + .call_with_timeout( + serial.to_string(), + "device.info", + serde_json::json!({}), + ANDROIDKIT_START_PROBE_TIMEOUT, + ) + .await + { + Ok(_) => return Ok(()), + Err(error) if tokio::time::Instant::now() >= deadline => { + return Err(error).with_context(|| { + format!( + "AndroidKit did not become ready on {serial} within {} seconds", + ANDROIDKIT_START_TIMEOUT.as_secs() + ) + }); + } + Err(_) => tokio::time::sleep(ANDROIDKIT_START_RETRY_DELAY).await, + } + } + } +} + impl Default for AndroidDebugBridge { fn default() -> Self { Self { @@ -772,6 +984,79 @@ impl AndroidDebugBridge { Ok(()) } + pub fn install_androidkit(&self, serial: &str, apk_path: &Path) -> Result<()> { + if !apk_path.is_file() { + return Err(anyhow!( + "AndroidKit APK does not exist: {}", + apk_path.display() + )); + } + run_command( + &self.adb_path, + [ + "-s", + serial, + "install", + "-r", + apk_path + .to_str() + .ok_or_else(|| anyhow!("AndroidKit APK path is not valid UTF-8"))?, + ], + )?; + Ok(()) + } + + pub async fn start_androidkit(&self, serial: &str) -> Result<()> { + self.run_shell( + serial, + "nohup am instrument -w -r xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation >/dev/null 2>&1 &", + ) + .with_context(|| format!("failed to start AndroidKit on {serial}"))?; + AndroidKit::with_adb_path(self.adb_path.clone()) + .wait_until_ready(serial) + .await + .with_context(|| format!("failed to start AndroidKit on {serial}")) + } + + pub fn stop_androidkit(&self, serial: &str) -> Result<()> { + self.run_shell( + serial, + &format!("am force-stop {}", AndroidKit::PACKAGE_NAME), + )?; + Ok(()) + } + + fn forward_androidkit(&self, serial: &str) -> Result { + let output = run_command( + &self.adb_path, + [ + "-s", + serial, + "forward", + "tcp:0", + "localabstract:xcrs-androidkit", + ], + )?; + output + .trim() + .parse::() + .with_context(|| format!("adb returned an invalid AndroidKit forward port: {output:?}")) + } + + fn remove_forward(&self, serial: &str, local_port: u16) -> Result<()> { + run_command( + &self.adb_path, + [ + "-s", + serial, + "forward", + "--remove", + &format!("tcp:{local_port}"), + ], + )?; + Ok(()) + } + fn run_shell(&self, serial: &str, command: &str) -> Result { run_command(&self.adb_path, ["-s", serial, "shell", command]) } @@ -1455,6 +1740,33 @@ mod tests { assert_eq!(elements[0]["hittable"], serde_json::json!(true)); } + #[test] + fn normalizes_visible_androidkit_elements() { + let hierarchy = serde_json::json!({ + "hierarchy": [{ + "role": "android.widget.Button", + "text": "Continue", + "hint": "", + "contentDescription": "", + "resourceId": "com.example:id/continue", + "enabled": true, + "clickable": true, + "selected": false, + "visible": true, + "rect": { "x": 10, "y": 20, "width": 100, "height": 40 } + }] + }); + + let elements = extract_androidkit_elements(&hierarchy); + + assert_eq!(elements.len(), 1); + assert_eq!(elements[0]["label"], serde_json::json!("Continue")); + assert_eq!( + elements[0]["rawIdentifier"], + serde_json::json!("com.example:id/continue") + ); + } + #[test] fn parses_device_tunnel_address() { let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n"; diff --git a/docs/androidkit.md b/docs/androidkit.md new file mode 100644 index 0000000..900c952 --- /dev/null +++ b/docs/androidkit.md @@ -0,0 +1,27 @@ +# XCRS AndroidKit + +XCRS AndroidKit is the optional Android companion runner for semantic E2E +automation. It is separate from adb: adb remains sufficient for screenshots, +coordinate input, app lifecycle, and system buttons. + +Build an AndroidKit APK from the public `xcrs-androidkit` repository, then +install and start it on the selected Android device: + +```sh +xcrs android-kit install --serial --apk +xcrs android-kit start --serial +xcrs android-kit status --serial +``` + +AndroidKit binds only to an Android local socket. xcrs creates and removes its +own `adb forward` connection for each RPC call; it does not expose a network +service on the device. + +When AndroidKit is available, `ui_describe`, `ui_element_list`, and `ui_tap` +accept `app_id` as the Android package name. `ui_tap` uses an exact accessible +text, content description, resource identifier, or hint and fails if it finds +zero or more than one visible enabled target. The app must be foreground. + +`bundle_id` remains accepted as a compatibility alias for Apple callers. New +cross-platform clients should send `app_id`; passing different `app_id` and +`bundle_id` values is invalid. diff --git a/docs/mcp.md b/docs/mcp.md index f681831..5d7e2f3 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -276,8 +276,8 @@ them by server. 2. Call `device_select` with the Apple simulator/runner details or Android adb serial. Later actions use that target by default. 3. Call `device_capabilities` before choosing an input or inspection action. -4. Use `screen_capture` or the Apple-only `ui_describe`/`ui_element_list` to - inspect the app, then drive it with the `input_*` tools. +4. Use `screen_capture` or `ui_describe`/`ui_element_list` to + inspect the app, then use semantic `ui_tap` or the raw `input_*` tools. | Tool | Purpose | | --- | --- | @@ -288,14 +288,15 @@ them by server. | `app_launch` / `app_terminate` | Start or stop an installed app on the selected target. | | `screen_capture` | Capture the selected target as a PNG image. | | `url_open` | Open an HTTP(S) URL or custom scheme where the target supports it. | -| `ui_describe` / `ui_element_list` | Inspect the Apple ControlKit accessibility hierarchy. | +| `ui_describe` / `ui_element_list` / `ui_tap` | Inspect or activate accessible elements through ControlKit (Apple) or AndroidKit (Android). | | `input_tap` / `input_text` / `input_swipe` / `input_button` | Drive shared touch, text, gesture, and button actions. | | `input_click` | Click a macOS target with pointer coordinates. | | `input_spatial_tap` | Perform a visionOS spatial tap. | | `orientation_get` / `orientation_set` | Read or update orientation where supported. | -See [ControlKit runners](./controlkit.md) for Apple runner setup. Android tools -require adb and an authorized device; Android TV app launches resolve the +See [ControlKit runners](./controlkit.md) for Apple runner setup and +[XCRS AndroidKit](./androidkit.md) for Android semantic UI automation. Android +tools require adb and an authorized device; Android TV app launches resolve the device's Leanback launcher activity. #### Automation migration in 0.5.0