diff --git a/android/samples/mobile-2/README.md b/android/samples/mobile-2/README.md index 4ecb83b60c..073938e1d7 100644 --- a/android/samples/mobile-2/README.md +++ b/android/samples/mobile-2/README.md @@ -19,6 +19,30 @@ An Android Jetpack Compose chat client that connects to a TypeAgent agent-server [Conversation persistence](#conversation-persistence)) - DevTunnel authentication via `X-Tunnel-Authorization` header - Build-time configuration via environment variables and `BuildConfig` +- Wear OS prompt handoff from the companion sample + +## Wear OS prompt handoff + +The companion project at [`../wearos`](../wearos/) recognizes speech on the +watch and opens this app with a `typeagentchat://main` deep link. The phone owns +the only TypeAgent WebSocket connection and submits the prompt after that +connection is ready. + +This is intentionally a fire-and-forget POC. The watch reports whether Android +handed the prompt to the paired phone, but TypeAgent responses remain in the +phone chat. + +External prompts fill the composer and wait for an explicit Send tap by default. +To enable automatic execution for controlled POC testing, build with: + +```powershell +.\gradlew.bat -Ptypeagent.wear.autoexecute=true assembleDebug +``` + +The deep link is `BROWSABLE`, as required by `RemoteActivityHelper`, and can +therefore be invoked by another app. The build flag is a demo switch, not an +authentication boundary. Use the Wear Data Layer before enabling automatic +execution in a production app. ## Conversation persistence @@ -174,6 +198,7 @@ The app connects automatically on launch. Tap **Retry** in the status bar if the - **Token storage**: `TYPEAGENT_TUNNEL_TOKEN` is compiled into `BuildConfig`. Do not distribute APKs built with a sensitive or long-lived token. - **Token transmission**: The token is sent only as an HTTP upgrade header and is never logged. +- **Wear prompt transport**: The POC deep link is externally reachable. Only enable automatic execution with `-Ptypeagent.wear.autoexecute=true` on controlled test devices. [devtunnel]: https://learn.microsoft.com/en-us/azure/developer/dev-tunnels/ [devtunnel-cli]: https://learn.microsoft.com/en-us/azure/developer/dev-tunnels/get-started diff --git a/android/samples/mobile-2/app/build.gradle.kts b/android/samples/mobile-2/app/build.gradle.kts index 491be6423c..cf701f6ae3 100644 --- a/android/samples/mobile-2/app/build.gradle.kts +++ b/android/samples/mobile-2/app/build.gradle.kts @@ -14,6 +14,14 @@ val tunnelTokenFromEnv = providers.environmentVariable("TYPEAGENT_TUNNEL_TOKEN") .orElse("") .get() .escapeForBuildConfig() +val wearPromptAutoExecute = providers.gradleProperty("typeagent.wear.autoexecute") + .orElse("false") + .get() + .also { value -> + require(value == "true" || value == "false") { + "typeagent.wear.autoexecute must be true or false" + } + } android { namespace = "com.example.typeagentchat" @@ -33,6 +41,7 @@ android { testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" buildConfigField("String", "TYPEAGENT_SERVER_URL", "\"$tunnelUrlFromEnv\"") buildConfigField("String", "TYPEAGENT_TUNNEL_TOKEN", "\"$tunnelTokenFromEnv\"") + buildConfigField("boolean", "WEAR_PROMPT_AUTOEXECUTE", wearPromptAutoExecute) } buildTypes { @@ -69,6 +78,7 @@ dependencies { implementation(libs.androidx.compose.ui.tooling.preview) implementation(libs.androidx.core.ktx) implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.lifecycle.viewmodel.ktx) implementation(libs.androidx.lifecycle.viewmodel.compose) implementation(libs.commonmark) implementation(libs.squareup.okhttp) diff --git a/android/samples/mobile-2/app/src/main/AndroidManifest.xml b/android/samples/mobile-2/app/src/main/AndroidManifest.xml index 2c5719a2d9..576c28ca46 100644 --- a/android/samples/mobile-2/app/src/main/AndroidManifest.xml +++ b/android/samples/mobile-2/app/src/main/AndroidManifest.xml @@ -139,6 +139,7 @@ android:name=".MainActivity" android:exported="true" android:label="@string/app_name" + android:launchMode="singleTask" android:theme="@style/Theme.TypeAgentChat" android:windowSoftInputMode="adjustResize"> @@ -146,6 +147,22 @@ + + + + + + + + + diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt index 6b95408142..b1d91dcfc3 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt @@ -14,9 +14,11 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.drop import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull /** * A client action that the agent asked the app to perform and that can only be @@ -150,6 +152,8 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { */ private val clientActionEvents = Channel(Channel.UNLIMITED) internal val clientActions: Flow = clientActionEvents.receiveAsFlow() + private val externalPromptEvents = Channel(Channel.UNLIMITED) + private val externalPromptDrafts = ExternalPromptDrafts() private var hasConnected = false @@ -303,6 +307,11 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { observeConversationForPersistence() observeConversationIdForPersistence() + viewModelScope.launch { + for (prompt in externalPromptEvents) { + deliverExternalPrompt(prompt) + } + } } /** @@ -436,7 +445,11 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { } fun onInputTextChange(text: String) { - _inputText.value = text + _inputText.value = if (text.isBlank()) { + externalPromptDrafts.currentRemoved("") + } else { + text + } } private val isConnected: Boolean @@ -447,7 +460,7 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { /** @return true when the message was handed to the socket and the input was cleared. */ fun submitMessage(): Boolean { - return sendText(_inputText.value) + return submitComposerText(_inputText.value) } /** @@ -464,7 +477,48 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { _inputText.value = merged return false } - return sendText(merged) + return submitComposerText(merged) + } + + fun submitExternalPrompt(prompt: String, autoExecute: Boolean) { + val text = prompt.trim() + if (text.isEmpty()) { + return + } + if (!autoExecute) { + parkInInput(text) + return + } + if (externalPromptEvents.trySend(ExternalPrompt(text)).isFailure) { + Log.w(TAG, "Could not queue external prompt: the chat screen is gone") + parkInInput(text) + } + } + + private suspend fun deliverExternalPrompt(prompt: ExternalPrompt) { + if ( + !awaitExternalPromptConnection() || + !webSocketManager.trySendExternalCommand(prompt.text) + ) { + parkInInput(prompt.text) + } + } + + private suspend fun awaitExternalPromptConnection(): Boolean { + if (isConnected) { + return true + } + val settled = withTimeoutOrNull(EXTERNAL_PROMPT_CONNECT_TIMEOUT_MILLIS) { + connectionStatus.first { + it.state == ConnectionStatus.State.CONNECTED || + it.state == ConnectionStatus.State.ERROR + } + } + return settled?.state == ConnectionStatus.State.CONNECTED + } + + private fun parkInInput(text: String) { + _inputText.value = externalPromptDrafts.park(_inputText.value, text) } fun respondToPendingYesNo(yes: Boolean): Boolean = webSocketManager.respondToPendingYesNo(yes) @@ -500,13 +554,19 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { } } - private fun sendText(text: String): Boolean { + private fun submitComposerText(text: String): Boolean { val message = text.trim() if (!isConnected || message.isBlank()) { return false } - webSocketManager.sendMessage(message) - _inputText.value = "" + if (externalPromptDrafts.isShowingExternalPrompt) { + if (!webSocketManager.trySendExternalCommand(message)) { + return false + } + } else { + webSocketManager.sendMessage(message) + } + _inputText.value = externalPromptDrafts.currentRemoved("") return true } @@ -515,6 +575,7 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { webSocketManager.setStaleConversationHandler(null) webSocketManager.disconnect() clientActionEvents.close() + externalPromptEvents.close() flushConversationToDisk() super.onCleared() } @@ -546,6 +607,7 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { private companion object { private const val TAG = "ChatViewModel" + private const val EXTERNAL_PROMPT_CONNECT_TIMEOUT_MILLIS = 15_000L /** * Long enough to collapse a burst of streamed display chunks into one @@ -556,6 +618,34 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { } } +private data class ExternalPrompt(val text: String) + +internal class ExternalPromptDrafts { + private val queuedPrompts = ArrayDeque() + + var isShowingExternalPrompt = false + private set + + fun park(currentText: String, prompt: String): String { + queuedPrompts.addLast(prompt) + return showNextIfAvailable(currentText) + } + + fun currentRemoved(currentText: String): String { + isShowingExternalPrompt = false + return showNextIfAvailable(currentText) + } + + private fun showNextIfAvailable(currentText: String): String { + if (isShowingExternalPrompt || currentText.isNotBlank()) { + return currentText + } + val nextPrompt = queuedPrompts.removeFirstOrNull() ?: return currentText + isShowingExternalPrompt = true + return nextPrompt + } +} + internal fun mergeSpeechInputText( currentText: String, recognizedText: String diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt index ebd93969c2..a9ec0cb80f 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt @@ -105,6 +105,9 @@ class MainActivity : ComponentActivity() { tunnelToken = tunnelToken, schemaContent = agentSchemaContent ) + if (savedInstanceState == null) { + handleExternalPrompt(intent) + } // Collected for the Activity's whole lifetime rather than only while // RESUMED. An agent-driven action has an `executeAction` RPC waiting on @@ -171,6 +174,37 @@ class MainActivity : ComponentActivity() { } } + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + setIntent(intent) + handleExternalPrompt(intent) + } + + private fun handleExternalPrompt(intent: Intent?) { + if (intent?.action != Intent.ACTION_VIEW) { + return + } + + when (val result = parseWearPrompt(intent.toWearLinkFields())) { + is WearPromptResult.Accepted -> { + val autoExecute = + BuildConfig.WEAR_PROMPT_AUTOEXECUTE && result.prompt.requestsExecute + Log.d( + TAG, + "External prompt accepted length=${result.prompt.text.length} " + + "autoExecute=$autoExecute" + ) + viewModel.submitExternalPrompt(result.prompt.text, autoExecute) + } + + is WearPromptResult.Rejected -> { + if (result.reason != WearPromptRejection.NOT_A_PROMPT_LINK) { + Log.w(TAG, "External prompt rejected: ${result.reason}") + } + } + } + } + // No onDestroy teardown: the socket is owned by ChatViewModel and released // in its onCleared. Disconnecting here would tear the connection down on // every rotation, theme or locale change. diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptIntent.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptIntent.kt new file mode 100644 index 0000000000..3883e14478 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptIntent.kt @@ -0,0 +1,15 @@ +package com.example.typeagentchat + +import android.content.Intent + +internal fun Intent.toWearLinkFields(): WearLinkFields { + val uri = data + val hierarchicalUri = uri?.takeIf { it.isHierarchical } + return WearLinkFields( + scheme = uri?.scheme, + host = uri?.host, + promptQuery = hierarchicalUri?.getQueryParameter(WEAR_PROMPT_PARAM), + executeQuery = hierarchicalUri?.getQueryParameter(WEAR_EXECUTE_PARAM), + promptExtra = getStringExtra(WEAR_PROMPT_PARAM) + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptLink.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptLink.kt new file mode 100644 index 0000000000..b629f037b8 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WearPromptLink.kt @@ -0,0 +1,59 @@ +package com.example.typeagentchat + +const val WEAR_LINK_SCHEME = "typeagentchat" +const val WEAR_LINK_HOST = "main" +const val WEAR_PROMPT_PARAM = "prompt" +const val WEAR_EXECUTE_PARAM = "execute" +// Keep this limit and link contract in sync with wearos/RemotePromptSender.kt. +const val WEAR_PROMPT_MAX_LENGTH = 1_000 + +data class WearLinkFields( + val scheme: String?, + val host: String?, + val promptQuery: String?, + val executeQuery: String?, + val promptExtra: String? +) + +data class WearPrompt( + val text: String, + val requestsExecute: Boolean +) + +enum class WearPromptRejection { + NOT_A_PROMPT_LINK, + NO_PROMPT, + TOO_LONG +} + +sealed interface WearPromptResult { + data class Accepted(val prompt: WearPrompt) : WearPromptResult + data class Rejected(val reason: WearPromptRejection) : WearPromptResult +} + +fun parseWearPrompt(fields: WearLinkFields): WearPromptResult { + if ( + !fields.scheme.equals(WEAR_LINK_SCHEME, ignoreCase = true) || + !fields.host.equals(WEAR_LINK_HOST, ignoreCase = true) + ) { + return WearPromptResult.Rejected(WearPromptRejection.NOT_A_PROMPT_LINK) + } + + val prompt = fields.promptExtra + ?.takeIf { it.isNotBlank() } + ?: fields.promptQuery + val text = prompt?.trim().orEmpty() + if (text.isEmpty()) { + return WearPromptResult.Rejected(WearPromptRejection.NO_PROMPT) + } + if (text.length > WEAR_PROMPT_MAX_LENGTH) { + return WearPromptResult.Rejected(WearPromptRejection.TOO_LONG) + } + + return WearPromptResult.Accepted( + WearPrompt( + text = text, + requestsExecute = fields.executeQuery.equals("true", ignoreCase = true) + ) + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt index a29de98c98..d913ee7236 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt @@ -287,7 +287,42 @@ class WebSocketManager internal constructor( return } - sendInvoke( + submitCommand(message, currentConversationId) + } + + /** + * Sends a new command from an external input without allowing it to answer + * an interaction that is waiting for the phone user. + * + * The decision and socket handoff share the same lock used by inbound + * interaction updates. The caller retains the prompt when this returns false. + */ + fun trySendExternalCommand(text: String): Boolean { + val message = text.trim() + if (message.isEmpty()) { + return false + } + + synchronized(lock) { + val currentConversationId = conversationId + if ( + webSocket == null || + currentConversationId.isNullOrBlank() || + pendingUserInteraction != null + ) { + return false + } + + if (!submitCommand(message, currentConversationId)) { + return false + } + appendUserMessage(message) + return true + } + } + + private fun submitCommand(message: String, currentConversationId: String): Boolean { + return sendInvoke( channelName = dispatcherChannelName(currentConversationId), methodName = "submitCommand", args = listOf(message), @@ -1175,11 +1210,11 @@ class WebSocketManager internal constructor( args: List, onResult: (Any?) -> Unit, onError: (String) -> Unit - ) { + ): Boolean { val socket = webSocket if (socket == null) { onError("WebSocket is not connected.") - return + return false } val callId = nextCallId.getAndIncrement() @@ -1208,7 +1243,9 @@ class WebSocketManager internal constructor( pendingInvokes.remove(callId) } onError("Failed to send RPC invoke for $methodName.") + return false } + return true } private fun sendRpcResult(channelName: String, callId: Int, result: Any?) { diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptDraftsTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptDraftsTest.kt new file mode 100644 index 0000000000..f99d77f543 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptDraftsTest.kt @@ -0,0 +1,50 @@ +package com.example.typeagentchat + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ExternalPromptDraftsTest { + + @Test + fun `parked prompts keep their boundaries`() { + val drafts = ExternalPromptDrafts() + + val first = drafts.park("", "first prompt") + val stillFirst = drafts.park(first, "second prompt") + val second = drafts.currentRemoved("") + val empty = drafts.currentRemoved("") + + assertEquals("first prompt", first) + assertEquals("first prompt", stillFirst) + assertEquals("second prompt", second) + assertEquals("", empty) + assertFalse(drafts.isShowingExternalPrompt) + } + + @Test + fun `external prompt waits behind an existing draft`() { + val drafts = ExternalPromptDrafts() + + val existing = drafts.park("phone draft", "watch prompt") + assertEquals("phone draft", existing) + assertFalse(drafts.isShowingExternalPrompt) + + val watchPrompt = drafts.currentRemoved("") + assertEquals("watch prompt", watchPrompt) + assertTrue(drafts.isShowingExternalPrompt) + } + + @Test + fun `edited external prompt remains marked and clearing advances the queue`() { + val drafts = ExternalPromptDrafts() + + drafts.park("", "watch prompt") + + assertEquals("edited watch prompt", drafts.park("edited watch prompt", "next prompt")) + assertTrue(drafts.isShowingExternalPrompt) + assertEquals("next prompt", drafts.currentRemoved("")) + assertTrue(drafts.isShowingExternalPrompt) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptSubmissionTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptSubmissionTest.kt new file mode 100644 index 0000000000..724a3fdc66 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ExternalPromptSubmissionTest.kt @@ -0,0 +1,78 @@ +package com.example.typeagentchat + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class ExternalPromptSubmissionTest { + + private val transport = TestWebSocketTransport() + private val manager = WebSocketManager(TestDeviceIdentity(), transport) + + @Test + fun `external prompt is submitted as a new command`() { + connectAndRegister() + + assertTrue(manager.trySendExternalCommand(" set a timer ")) + + val invoke = transport.takeInvoke("submitCommand") + assertEquals("set a timer", invoke.args.getString(0)) + assertEquals("set a timer", manager.messages.value.last().text) + } + + @Test + fun `external prompt cannot answer a pending confirmation`() { + connectAndRegister() + transport.deliverCall( + channelName = "clientio:$CONVERSATION_ID", + methodName = "requestChoice", + args = JSONArray() + .put(JSONObject().put("requestId", "request-1")) + .put("choice-1") + .put("yesNo") + .put("Continue?") + .put(JSONArray().put("Yes").put("No")) + ) + + assertFalse(manager.trySendExternalCommand("yes")) + assertNull(transport.nextInvokeOrNull()) + } + + @Test + fun `external prompt stays with caller until conversation is ready`() { + assertFalse(manager.trySendExternalCommand("hello")) + assertNull(transport.nextInvokeOrNull()) + } + + @Test + fun `external prompt stays with caller when socket rejects send`() { + connectAndRegister() + transport.rejectNextSend() + + assertFalse(manager.trySendExternalCommand("keep this prompt")) + assertFalse(manager.messages.value.any { it.text == "keep this prompt" }) + assertNull(transport.nextInvokeOrNull()) + } + + private fun connectAndRegister() { + manager.connect( + url = "ws://localhost:8080/", + schemaContent = "export type AndroidDeviceAction = never;" + ) + transport.open() + transport.takeInvoke("joinConversation").succeed( + JSONObject() + .put("conversationId", CONVERSATION_ID) + .put("connectionId", "connection-1") + ) + transport.takeInvoke("registerClientAgent").succeed() + } + + private companion object { + const val CONVERSATION_ID = "conversation-1" + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/RegistrationCollisionRecoveryTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/RegistrationCollisionRecoveryTest.kt index ced0c12610..4595b5be14 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/RegistrationCollisionRecoveryTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/RegistrationCollisionRecoveryTest.kt @@ -1,12 +1,5 @@ package com.example.typeagentchat -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import okhttp3.WebSocket -import okhttp3.WebSocketListener -import okio.ByteString -import org.json.JSONArray import org.json.JSONObject import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -21,8 +14,8 @@ import org.junit.Test */ class RegistrationCollisionRecoveryTest { - private val transport = FakeTransport() - private val manager = WebSocketManager(FakeDeviceIdentity(), transport) + private val transport = TestWebSocketTransport() + private val manager = WebSocketManager(TestDeviceIdentity(), transport) @Test fun `a collision evicts the stale registration and registers again`() { @@ -179,126 +172,6 @@ class RegistrationCollisionRecoveryTest { ) } - /** - * A [WebSocket.Factory] that hands back a socket which records the frames - * the manager sends and lets the test answer them, so the whole handshake - * runs over the real wire format without a server. - */ - private class FakeTransport : WebSocket.Factory { - private var listener: WebSocketListener? = null - private var socket: FakeWebSocket? = null - - override fun newWebSocket(request: Request, listener: WebSocketListener): WebSocket { - val created = FakeWebSocket() - this.listener = listener - this.socket = created - return created - } - - fun open() { - val current = requireNotNull(socket) { "connect() was not called" } - requireNotNull(listener).onOpen( - current, - Response.Builder() - .request(current.request()) - .protocol(Protocol.HTTP_1_1) - .code(101) - .message("Switching Protocols") - .build() - ) - } - - fun nextInvokeOrNull(): SentInvoke? { - val frame = requireNotNull(socket).sentFrames.removeFirstOrNull() ?: return null - val message = JSONObject(frame).getJSONObject("message") - return SentInvoke( - transport = this, - channelName = JSONObject(frame).getString("name"), - methodName = message.getString("name"), - callId = message.getInt("callId"), - args = message.getJSONArray("args") - ) - } - - fun takeInvoke(expectedMethodName: String): SentInvoke { - val invoke = requireNotNull(nextInvokeOrNull()) { - "expected an invoke of $expectedMethodName, but nothing was sent" - } - assertEquals(expectedMethodName, invoke.methodName) - return invoke - } - - fun deliver(channelName: String, message: JSONObject) { - val current = requireNotNull(socket) - requireNotNull(listener).onMessage( - current, - JSONObject() - .put("name", channelName) - .put("message", message) - .toString() - ) - } - } - - private class SentInvoke( - private val transport: FakeTransport, - val channelName: String, - val methodName: String, - val callId: Int, - val args: JSONArray - ) { - fun firstArg(): JSONObject = args.getJSONObject(0) - - fun succeed(result: Any? = null) { - transport.deliver( - channelName, - JSONObject() - .put("type", "invokeResult") - .put("callId", callId) - .putOpt("result", result) - ) - } - - fun failWith(error: String) { - transport.deliver( - channelName, - JSONObject() - .put("type", "invokeError") - .put("callId", callId) - .put("error", error) - ) - } - } - - private class FakeWebSocket : WebSocket { - val sentFrames = ArrayDeque() - - override fun request(): Request = - Request.Builder().url("http://localhost:8080/").build() - - override fun queueSize(): Long = 0 - - override fun send(text: String): Boolean { - sentFrames.addLast(text) - return true - } - - override fun send(bytes: ByteString): Boolean = true - - override fun close(code: Int, reason: String?): Boolean = true - - override fun cancel() = Unit - } - - /** - * Stands in for [StoredDeviceIdentity], which needs a `Context` these plain - * JVM tests do not have. - */ - private class FakeDeviceIdentity : DeviceIdentity { - override val instanceId = "device-under-test" - override val displayName = "Test Phone" - } - private companion object { const val CONVERSATION_ID = "conversation-1" } diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WearPromptLinkTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WearPromptLinkTest.kt new file mode 100644 index 0000000000..9b6debd2e9 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WearPromptLinkTest.kt @@ -0,0 +1,119 @@ +package com.example.typeagentchat + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class WearPromptLinkTest { + + @Test + fun `accepts a valid auto-execute prompt`() { + val result = parseWearPrompt(fields(promptQuery = "set a timer", executeQuery = "true")) + + assertTrue(result is WearPromptResult.Accepted) + result as WearPromptResult.Accepted + assertEquals("set a timer", result.prompt.text) + assertTrue(result.prompt.requestsExecute) + } + + @Test + fun `execute is opt-in and case insensitive`() { + val absent = accepted(fields(promptQuery = "hello")) + val mixedCase = accepted(fields(promptQuery = "hello", executeQuery = "TrUe")) + + assertFalse(absent.requestsExecute) + assertTrue(mixedCase.requestsExecute) + } + + @Test + fun `intent extra takes precedence over query`() { + val prompt = accepted( + fields( + promptQuery = "query prompt", + promptExtra = "extra prompt" + ) + ) + + assertEquals("extra prompt", prompt.text) + } + + @Test + fun `blank intent extra falls back to query`() { + val prompt = accepted(fields(promptQuery = "query prompt", promptExtra = " ")) + + assertEquals("query prompt", prompt.text) + } + + @Test + fun `trims accepted prompt`() { + val prompt = accepted(fields(promptQuery = " hello watch ")) + + assertEquals("hello watch", prompt.text) + } + + @Test + fun `accepts prompt at length limit`() { + val prompt = accepted(fields(promptQuery = "a".repeat(WEAR_PROMPT_MAX_LENGTH))) + + assertEquals(WEAR_PROMPT_MAX_LENGTH, prompt.text.length) + } + + @Test + fun `rejects prompt over length limit`() { + val result = parseWearPrompt( + fields(promptQuery = "a".repeat(WEAR_PROMPT_MAX_LENGTH + 1)) + ) + + assertRejected(WearPromptRejection.TOO_LONG, result) + } + + @Test + fun `rejects missing and blank prompts`() { + assertRejected(WearPromptRejection.NO_PROMPT, parseWearPrompt(fields())) + assertRejected( + WearPromptRejection.NO_PROMPT, + parseWearPrompt(fields(promptQuery = " ")) + ) + } + + @Test + fun `rejects wrong scheme or host`() { + assertRejected( + WearPromptRejection.NOT_A_PROMPT_LINK, + parseWearPrompt(fields(scheme = "https", promptQuery = "hello")) + ) + assertRejected( + WearPromptRejection.NOT_A_PROMPT_LINK, + parseWearPrompt(fields(host = "other", promptQuery = "hello")) + ) + } + + private fun fields( + scheme: String? = WEAR_LINK_SCHEME, + host: String? = WEAR_LINK_HOST, + promptQuery: String? = null, + executeQuery: String? = null, + promptExtra: String? = null + ) = WearLinkFields( + scheme = scheme, + host = host, + promptQuery = promptQuery, + executeQuery = executeQuery, + promptExtra = promptExtra + ) + + private fun accepted(fields: WearLinkFields): WearPrompt { + val result = parseWearPrompt(fields) + assertTrue("Expected accepted result, got $result", result is WearPromptResult.Accepted) + return (result as WearPromptResult.Accepted).prompt + } + + private fun assertRejected( + expected: WearPromptRejection, + actual: WearPromptResult + ) { + assertTrue("Expected rejected result, got $actual", actual is WearPromptResult.Rejected) + assertEquals(expected, (actual as WearPromptResult.Rejected).reason) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSocketTestTransport.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSocketTestTransport.kt new file mode 100644 index 0000000000..320a43d5c1 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/WebSocketTestTransport.kt @@ -0,0 +1,146 @@ +package com.example.typeagentchat + +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals + +internal class TestWebSocketTransport : WebSocket.Factory { + private var listener: WebSocketListener? = null + private var socket: TestWebSocket? = null + + override fun newWebSocket(request: Request, listener: WebSocketListener): WebSocket { + val created = TestWebSocket() + this.listener = listener + socket = created + return created + } + + fun open() { + val current = requireNotNull(socket) { "connect() was not called" } + requireNotNull(listener).onOpen( + current, + Response.Builder() + .request(current.request()) + .protocol(Protocol.HTTP_1_1) + .code(101) + .message("Switching Protocols") + .build() + ) + } + + fun nextInvokeOrNull(): SentInvoke? { + val frame = socket?.sentFrames?.removeFirstOrNull() ?: return null + val envelope = JSONObject(frame) + val message = envelope.getJSONObject("message") + return SentInvoke( + transport = this, + channelName = envelope.getString("name"), + methodName = message.getString("name"), + callId = message.getInt("callId"), + args = message.getJSONArray("args") + ) + } + + fun takeInvoke(expectedMethodName: String): SentInvoke { + val invoke = requireNotNull(nextInvokeOrNull()) { + "expected an invoke of $expectedMethodName, but nothing was sent" + } + assertEquals(expectedMethodName, invoke.methodName) + return invoke + } + + fun deliverCall(channelName: String, methodName: String, args: JSONArray) { + deliver( + channelName, + JSONObject() + .put("type", "call") + .put("name", methodName) + .put("args", args) + ) + } + + fun rejectNextSend() { + requireNotNull(socket).rejectNextSend = true + } + + fun deliver(channelName: String, message: JSONObject) { + val current = requireNotNull(socket) + requireNotNull(listener).onMessage( + current, + JSONObject() + .put("name", channelName) + .put("message", message) + .toString() + ) + } +} + +internal class SentInvoke( + private val transport: TestWebSocketTransport, + val channelName: String, + val methodName: String, + val callId: Int, + val args: JSONArray +) { + fun firstArg(): JSONObject = args.getJSONObject(0) + + fun succeed(result: Any? = null) { + transport.deliver( + channelName, + JSONObject() + .put("type", "invokeResult") + .put("callId", callId) + .putOpt("result", result) + ) + } + + fun failWith(error: String) { + transport.deliver( + channelName, + JSONObject() + .put("type", "invokeError") + .put("callId", callId) + .put("error", error) + ) + } +} + +private class TestWebSocket : WebSocket { + val sentFrames = ArrayDeque() + var rejectNextSend = false + + override fun request(): Request = + Request.Builder().url("http://localhost:8080/").build() + + override fun queueSize(): Long = 0 + + override fun send(text: String): Boolean { + if (rejectNextSend) { + rejectNextSend = false + return false + } + sentFrames.addLast(text) + return true + } + + override fun send(bytes: ByteString): Boolean = true + + override fun close(code: Int, reason: String?): Boolean = true + + override fun cancel() = Unit +} + +/** + * Stands in for [StoredDeviceIdentity], which needs a Context these JVM tests do + * not have. + */ +internal class TestDeviceIdentity : DeviceIdentity { + override val instanceId = "device-under-test" + override val displayName = "Test Phone" +} diff --git a/android/samples/mobile-2/gradle/libs.versions.toml b/android/samples/mobile-2/gradle/libs.versions.toml index f5dda036fe..3756dfa7b7 100644 --- a/android/samples/mobile-2/gradle/libs.versions.toml +++ b/android/samples/mobile-2/gradle/libs.versions.toml @@ -18,6 +18,7 @@ junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } androidx-espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espressoCore" } androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycleRuntimeKtx" } +androidx-lifecycle-viewmodel-ktx = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycleRuntimeKtx" } androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose", version.ref = "lifecycleRuntimeKtx" } androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" } androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" } diff --git a/android/samples/wearos/README.md b/android/samples/wearos/README.md index 4e01f57620..6b0aca3d80 100644 --- a/android/samples/wearos/README.md +++ b/android/samples/wearos/README.md @@ -1,3 +1,22 @@ -# WearOS Sample +# Wear OS Sample -This sample shows how to use the WearOS peripheral to invoke the TypeAgent on a paired mobile device. \ No newline at end of file +This sample recognizes a spoken prompt and sends it to the +[`mobile-2`](../mobile-2/) TypeAgent chat app on a paired phone. The phone owns +the TypeAgent connection; the watch is a thin input peripheral. + +## Run the POC + +1. Configure and run the `mobile-2` sample on the paired phone. +2. Open this folder in Android Studio and run the `app` module on a Wear OS 3+ + device or emulator paired with that phone. +3. Tap the speech-to-text button, speak a prompt, and confirm it in the system + recognizer. + +The watch displays whether Android accepted the remote launch. TypeAgent's +response is shown on the phone; this POC has no response path back to the watch. + +The phone accepts `typeagentchat://main` links and places them in the composer +for an explicit Send tap by default. `RemoteActivityHelper` requires a browsable +link, which is not an authentication boundary. Automatic execution can be +enabled with `-Ptypeagent.wear.autoexecute=true` only for controlled POC testing. +For production, use the Wear Data Layer. \ No newline at end of file diff --git a/android/samples/wearos/app/build.gradle.kts b/android/samples/wearos/app/build.gradle.kts index 67cf2b3dcd..631e5d610f 100644 --- a/android/samples/wearos/app/build.gradle.kts +++ b/android/samples/wearos/app/build.gradle.kts @@ -9,8 +9,8 @@ android { compileSdk = 34 defaultConfig { - applicationId = "com.microsoft.typeagent.wearos" - minSdk = 34 + applicationId = "com.example.typeagentchat" + minSdk = 30 targetSdk = 34 versionCode = 1 versionName = "1.0" @@ -53,6 +53,8 @@ dependencies { implementation(libs.constraintlayout.compose.android) implementation(libs.constraintlayout) implementation(libs.wear.remote.interactions) + implementation(libs.concurrent.futures.ktx) + implementation(libs.fragment.ktx) androidTestImplementation(platform(libs.compose.bom)) androidTestImplementation(libs.ui.test.junit4) debugImplementation(libs.ui.tooling) diff --git a/android/samples/wearos/app/src/main/AndroidManifest.xml b/android/samples/wearos/app/src/main/AndroidManifest.xml index 2c7e5d674d..5690805060 100644 --- a/android/samples/wearos/app/src/main/AndroidManifest.xml +++ b/android/samples/wearos/app/src/main/AndroidManifest.xml @@ -22,7 +22,7 @@ --> + android:value="false" /> + if (result.resultCode != Activity.RESULT_OK) { + speechToTextOverride = "" + return@registerForActivityResult } - } - override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { - super.onActivityResult(requestCode, resultCode, data) - - if (requestCode == REQUEST_CODE_SPEECH_INPUT) { - if (resultCode == RESULT_OK && data != null) { - - val res: ArrayList = - data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS) as ArrayList - - // Do we use a baked in shortcut/text or the one that was recognized? - if (speechToTextOverride.isNotBlank()) { - this.speechToTextText = this.speechToTextOverride - } else { - this.speechToTextText = Objects.requireNonNull(res)[0] - } + val recognizedText = speechToTextOverride.ifBlank { + result.data + ?.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS) + ?.firstOrNull() + .orEmpty() + } + speechToTextOverride = "" + if (recognizedText.isBlank()) { + return@registerForActivityResult + } - remoteLaunch() - } + lifecycleScope.launch { + mainState.onSpeechRecognized(recognizedText) } } - private fun remoteLaunch() { + private lateinit var mainState: MainViewModel - val remoteActivityHelper = RemoteActivityHelper(this, Executors.newSingleThreadExecutor()) + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) - val result = remoteActivityHelper.startRemoteActivity( - Intent(Intent.ACTION_VIEW) - .setData(Uri.parse("typeagent://main?execute=true&prompt=${this.speechToTextText}")) - .putExtra("prompt", this.speechToTextText) - .addCategory(Intent.CATEGORY_BROWSABLE), - null + val remoteActivityHelper = RemoteActivityHelper( + this, + ContextCompat.getMainExecutor(this) + ) + mainState = MainViewModel( + activity = this, + requestPermission = { + requestAudioPermission.launch(Manifest.permission.RECORD_AUDIO) + }, + requestSpeechRecognition = { overrideText -> + speechToTextOverride = overrideText + try { + recognizeSpeech.launch( + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra( + RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM + ) + putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault()) + putExtra( + RecognizerIntent.EXTRA_PROMPT, + getString(R.string.speech_to_text_prompt) + ) + } + ) + } catch (_: ActivityNotFoundException) { + speechToTextOverride = "" + mainState.onSpeechRecognitionUnavailable() + } + }, + remotePromptSender = RemotePromptSender(remoteActivityHelper) ) - println("wearsample $result") + setContent { + MainUI(mainState) + } } } diff --git a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainUI.kt b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainUI.kt index bf7d5b29ff..4d19915150 100644 --- a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainUI.kt +++ b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainUI.kt @@ -3,18 +3,9 @@ package com.microsoft.typeagent.wearos.presentation -import android.Manifest -import android.app.Activity -import android.content.Context -import android.content.ContextWrapper -import androidx.activity.compose.ManagedActivityResultLauncher -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts.RequestPermission import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect -import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLifecycleOwner import androidx.compose.ui.res.stringResource import androidx.lifecycle.DefaultLifecycleObserver @@ -28,30 +19,10 @@ import com.microsoft.typeagent.wearos.R import kotlinx.coroutines.launch @Composable -fun MainUI() { +fun MainUI(mainState: MainViewModel) { MaterialTheme { - lateinit var permissionsLauncher: ManagedActivityResultLauncher - - val context = LocalContext.current - val activity = context.findActivity() val scope = rememberCoroutineScope() - val mainState = remember(activity) { - MainViewModel( - activity = activity as MainActivity, - requestPermission = { - permissionsLauncher.launch(Manifest.permission.RECORD_AUDIO) - } - ) - } - - permissionsLauncher = rememberLauncherForActivityResult(RequestPermission()) { - // We ignore the direct result here, since we're going to check anyway. - scope.launch { - //mainState.permissionResultReturned() - } - } - val lifecycleOwner = LocalLifecycleOwner.current // Notify the state holder whenever we become stopped to reset the state @@ -104,7 +75,7 @@ fun MainUI() { mainState.onSTTClicked("e-mail Ted that I'm going to be late") } }, - (mainState.activity as MainActivity).speechToTextText + mainState.promptDeliveryStatus ) if (mainState.showPermissionRationale) { @@ -115,7 +86,7 @@ fun MainUI() { positiveButton = { Button( onClick = { - permissionsLauncher.launch(Manifest.permission.RECORD_AUDIO) + mainState.requestAudioPermission() mainState.showPermissionRationale = false } ) { @@ -143,15 +114,3 @@ fun MainUI() { } } } - -/** - * Find the closest Activity in a given Context. - */ -private tailrec fun Context.findActivity(): Activity = - when (this) { - is Activity -> this - is ContextWrapper -> baseContext.findActivity() - else -> throw IllegalStateException( - "findActivity() should be called in the context of an Activity" - ) - } diff --git a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainViewModel.kt b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainViewModel.kt index f48fc2d023..ab7d08cdbf 100644 --- a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainViewModel.kt +++ b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/MainViewModel.kt @@ -5,24 +5,19 @@ package com.microsoft.typeagent.wearos.presentation import android.Manifest import android.app.Activity -import android.content.Intent import android.content.pm.PackageManager import android.media.AudioDeviceInfo import android.media.AudioManager import android.media.MediaPlayer -import android.speech.RecognizerIntent -import android.util.Log import androidx.annotation.RequiresPermission import androidx.compose.foundation.MutatorMutex import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue -import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import androidx.core.content.getSystemService import com.microsoft.typeagent.wearos.R import java.time.Duration -import java.util.Locale import kotlin.coroutines.resume import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay @@ -34,10 +29,10 @@ import kotlinx.coroutines.suspendCancellableCoroutine */ class MainViewModel( val activity: MainActivity, - private val requestPermission: () -> Unit + private val requestPermission: () -> Unit, + private val requestSpeechRecognition: (overrideText: String) -> Unit, + private val remotePromptSender: RemotePromptSender ) { - private val REQUEST_CODE_SPEECH_INPUT = 1 - private val playbackStateMutatorMutex = MutatorMutex() var playbackState by mutableStateOf(PlaybackState.Ready) @@ -53,6 +48,9 @@ class MainViewModel( var showSpeakerNotSupported by mutableStateOf(false) + var promptDeliveryStatus by mutableStateOf("") + private set + private val soundRecorder = Recorder(activity, "audio.opus") suspend fun onStopped() { @@ -61,6 +59,10 @@ class MainViewModel( } } + fun requestAudioPermission() { + requestPermission() + } + suspend fun onMicClicked() { playbackStateMutatorMutex.mutate { when (playbackState) { @@ -160,7 +162,6 @@ class MainViewModel( } suspend fun onSTTClicked(reason: String) { - activity.speechToTextOverride = reason playbackStateMutatorMutex.mutate { when (playbackState) { is PlaybackState.Ready, @@ -170,37 +171,29 @@ class MainViewModel( PlaybackState.TakePicture, PlaybackState.EmailPicture, PlaybackState.SpeechToText -> { - speechToText() + requestSpeechRecognition(reason) } } } } - private fun speechToText() { - val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH) - intent.putExtra( - RecognizerIntent.EXTRA_LANGUAGE_MODEL, - RecognizerIntent.LANGUAGE_MODEL_FREE_FORM - ) - - intent.putExtra( - RecognizerIntent.EXTRA_LANGUAGE, - Locale.getDefault() - ) - - intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech to text") - - try { - ActivityCompat.startActivityForResult( - activity, - intent, - REQUEST_CODE_SPEECH_INPUT, - null - ) - } catch (e: Exception) { - Log.e("viewModel", e.toString()) + suspend fun onSpeechRecognized(text: String) { + promptDeliveryStatus = activity.getString(R.string.prompt_sending) + promptDeliveryStatus = when (remotePromptSender.send(text)) { + RemotePromptResult.HandedToPhone -> + activity.getString(R.string.prompt_handed_to_phone) + RemotePromptResult.PhoneUnreachable -> + activity.getString(R.string.phone_unavailable) + RemotePromptResult.PromptTooLong -> + activity.getString(R.string.prompt_too_long) + RemotePromptResult.Failed -> + activity.getString(R.string.prompt_send_failed) } } + + fun onSpeechRecognitionUnavailable() { + promptDeliveryStatus = activity.getString(R.string.speech_recognition_unavailable) + } } /** diff --git a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/RemotePromptSender.kt b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/RemotePromptSender.kt new file mode 100644 index 0000000000..bd7d445201 --- /dev/null +++ b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/RemotePromptSender.kt @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.microsoft.typeagent.wearos.presentation + +import android.content.Intent +import android.net.Uri +import android.util.Log +import androidx.concurrent.futures.await +import androidx.wear.remote.interactions.RemoteActivityHelper +import kotlinx.coroutines.CancellationException + +sealed interface RemotePromptResult { + data object HandedToPhone : RemotePromptResult + data object PhoneUnreachable : RemotePromptResult + data object PromptTooLong : RemotePromptResult + data object Failed : RemotePromptResult +} + +class RemotePromptSender( + private val remoteActivityHelper: RemoteActivityHelper +) { + suspend fun send(prompt: String): RemotePromptResult { + val text = prompt.trim() + if (text.isEmpty()) { + throw IllegalArgumentException("Prompt is empty") + } + if (text.length > WEAR_PROMPT_MAX_LENGTH) { + return RemotePromptResult.PromptTooLong + } + + val intent = Intent(Intent.ACTION_VIEW) + .setData( + Uri.Builder() + .scheme(WEAR_LINK_SCHEME) + .authority(WEAR_LINK_HOST) + .appendQueryParameter(WEAR_EXECUTE_PARAM, "true") + .appendQueryParameter(WEAR_PROMPT_PARAM, text) + .build() + ) + .putExtra(WEAR_PROMPT_PARAM, text) + .addCategory(Intent.CATEGORY_BROWSABLE) + + return try { + remoteActivityHelper.startRemoteActivity(intent, null).await() + RemotePromptResult.HandedToPhone + } catch (_: RemoteActivityHelper.RemoteIntentException) { + RemotePromptResult.PhoneUnreachable + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + Log.e(TAG, "Could not hand prompt to phone", error) + RemotePromptResult.Failed + } + } + + private companion object { + const val TAG = "RemotePromptSender" + // Keep this link contract in sync with mobile-2/WearPromptLink.kt. + const val WEAR_LINK_SCHEME = "typeagentchat" + const val WEAR_LINK_HOST = "main" + const val WEAR_PROMPT_PARAM = "prompt" + const val WEAR_EXECUTE_PARAM = "execute" + const val WEAR_PROMPT_MAX_LENGTH = 1_000 + } +} diff --git a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/SpeakerScreen.kt b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/SpeakerScreen.kt index a389c5dfe2..3732e34cb3 100644 --- a/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/SpeakerScreen.kt +++ b/android/samples/wearos/app/src/main/java/com/microsoft/typeagent/wearos/presentation/SpeakerScreen.kt @@ -64,12 +64,7 @@ fun SpeakerScreen( } // Speech to text - val isSTTVisible = - when (playbackState) { - PlaybackState.PlayingVoice, - is PlaybackState.PlayingVoice -> true - else -> false - } + val isSTTVisible = stt.isNotBlank() // We are using ConstraintLayout here to center the ControlDashboard, and align the progress // indicator to it. diff --git a/android/samples/wearos/app/src/main/res/values-round/strings.xml b/android/samples/wearos/app/src/main/res/values-round/strings.xml deleted file mode 100644 index 42f12297f4..0000000000 --- a/android/samples/wearos/app/src/main/res/values-round/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - From the Round world,\nHello, %1$s! - \ No newline at end of file diff --git a/android/samples/wearos/app/src/main/res/values/strings.xml b/android/samples/wearos/app/src/main/res/values/strings.xml index b750e8a99f..ea2f187b48 100644 --- a/android/samples/wearos/app/src/main/res/values/strings.xml +++ b/android/samples/wearos/app/src/main/res/values/strings.xml @@ -11,6 +11,13 @@ Stop Playing Recording Stop Playing Music Speech to Text in Progress + What should TypeAgent do? + Sending… + Handed to phone + Prompt too long + Could not hand prompt to phone + Phone unavailable + Speech recognition unavailable Take a picture Email Picture \ No newline at end of file diff --git a/android/samples/wearos/gradle/libs.versions.toml b/android/samples/wearos/gradle/libs.versions.toml index 7fa1122768..ef75f291e2 100644 --- a/android/samples/wearos/gradle/libs.versions.toml +++ b/android/samples/wearos/gradle/libs.versions.toml @@ -12,6 +12,8 @@ coreSplashscreen = "1.0.1" constraintlayoutComposeAndroid = "1.1.0" constraintlayout = "2.2.0" wearRemoteInteractions = "1.0.0" +concurrentFuturesKtx = "1.3.0" +fragmentKtx = "1.8.5" [libraries] material-icons-extended = { module = "androidx.compose.material:material-icons-extended", version.ref = "materialIconsExtended" } @@ -31,9 +33,10 @@ core-splashscreen = { group = "androidx.core", name = "core-splashscreen", versi constraintlayout-compose-android = { group = "androidx.constraintlayout", name = "constraintlayout-compose-android", version.ref = "constraintlayoutComposeAndroid" } constraintlayout = { group = "androidx.constraintlayout", name = "constraintlayout", version.ref = "constraintlayout" } wear-remote-interactions = { group = "androidx.wear", name = "wear-remote-interactions", version.ref = "wearRemoteInteractions" } +concurrent-futures-ktx = { group = "androidx.concurrent", name = "concurrent-futures-ktx", version.ref = "concurrentFuturesKtx" } +fragment-ktx = { group = "androidx.fragment", name = "fragment-ktx", version.ref = "fragmentKtx" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } -