feat(android): expose semantics to the TalkBack screen reader

README.md

@@ -257,7 +257,7 @@ The following table compares feature availability with the [previous Rive React
 | `useRiveFile()` hook              | ✅     | Convenient hook to load a Rive file                              |
 | `RiveView` error handling         | ✅     | Error handler for failed view operations                         |
 | `source` .riv file loading        | ✅     | Conveniently load .riv files from JS source                      |
-| Accessibility semantics           | ⚠️     | Editor-authored semantics → VoiceOver (iOS; Android in progress) |
+| Accessibility semantics           | ✅     | Editor-authored semantics → VoiceOver / TalkBack (new runtimes)  |
 | Animation selection               | ❌     | Animation playback not planned, use state machines               |
 | Renderer options                  | ❌     | Single renderer option available (Rive)                          |
 

android/src/new/java/com/margelo/nitro/rive/HybridRiveView.kt

@@ -8,7 +8,9 @@ import com.margelo.nitro.core.Promise
 import com.rive.BindData
 import com.rive.RiveReactNativeView
 import com.rive.ViewConfiguration
+import app.rive.ExperimentalRiveSemantics
 import app.rive.Fit as RiveFit
+import app.rive.RiveSemanticsMode
 import app.rive.Alignment as RiveAlignment
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
@@ -108,9 +110,16 @@ class HybridRiveView(val context: ThemedReactContext) : HybridRiveViewSpec() {
       )
     }
 
-  // Accepted for API parity; semantics support is pending in the upstream
-  // rive-android runtime (iOS-only for now).
+  @OptIn(ExperimentalRiveSemantics::class)
   override var semantics: Semantics? = null
+    set(value) {
+      field = value
+      view.semantics = when (value) {
+        Semantics.ON -> RiveSemanticsMode.On
+        Semantics.AUTOMATIC -> RiveSemanticsMode.Automatic
+        Semantics.OFF, null -> RiveSemanticsMode.Off
+      }
+    }
   override var dataBind: Variant_HybridViewModelInstanceSpec_DataBindMode_DataBindByName? = null
     set(value) {
       if (field != value) {

android/src/new/java/com/rive/RiveReactNativeView.kt

@@ -1,18 +1,25 @@
+@file:OptIn(ExperimentalRiveSemantics::class)
+
 package com.rive
 
 import android.annotation.SuppressLint
+import android.content.Context
 import android.graphics.SurfaceTexture
 import android.os.Build
 import android.util.Log
 import android.view.Choreographer
 import android.view.MotionEvent
 import android.view.TextureView
 import android.view.View
+import android.view.accessibility.AccessibilityManager
 import android.widget.FrameLayout
+import androidx.core.view.accessibility.AccessibilityNodeInfoCompat
 import com.facebook.react.bridge.UiThreadUtil
 import app.rive.Artboard
+import app.rive.ExperimentalRiveSemantics
 import app.rive.Fit
 import app.rive.RiveFile
+import app.rive.RiveSemanticsMode
 import app.rive.ViewModelInstance
 import app.rive.ViewModelSource
 import app.rive.core.ArtboardHandle
@@ -62,6 +69,20 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
     // (e.g. every 4th frame at 120Hz for a 30fps cap) instead of drifting past
     // it and halving the effective rate.
     private const val CAP_TOLERANCE_NS = 4_000_000L
+
+    // rive-android's TalkBack bridge calls androidx.core 1.17 APIs; an app that
+    // forces an older androidx.core crashes on the first accessibility query.
+    private val semanticsSupported: Boolean by lazy {
+      runCatching {
+        AccessibilityNodeInfoCompat::class.java
+          .getMethod("setSupplementalDescription", CharSequence::class.java)
+      }.isSuccess
+    }
+
+    private const val SEMANTICS_UNSUPPORTED_MESSAGE =
+      "Rive semantics need androidx.core 1.17.0 or newer (compileSdk 36), but the app " +
+        "resolved an older version; check for a forced androidx.core:core version in " +
+        "build.gradle. Semantics stay disabled."
   }
 
   // Render at most this many frames per second; null = every vsync.
@@ -73,6 +94,27 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
 
   var onError: ((String) -> Unit)? = null
 
+  var semantics: RiveSemanticsMode = RiveSemanticsMode.Off
+    set(value) {
+      if (field == value) return
+      field = value
+      resolveSemantics()
+    }
+
+  private val accessibilityManager =
+    context.getSystemService(Context.ACCESSIBILITY_SERVICE) as AccessibilityManager
+  private val accessibilityStateListener =
+    AccessibilityManager.AccessibilityStateChangeListener { resolveSemantics() }
+  private var observingAccessibilityState = false
+  private var semanticsEnabled = false
+  private var warnedSemanticsUnsupported = false
+  private var semanticsSyncJob: Job? = null
+  private val mainScope = CoroutineScope(Dispatchers.Main.immediate + SupervisorJob())
+
+  // Semantic input and viewport changes must be applied and drained even
+  // while paused or settled, which needs one zero-delta advance.
+  private var semanticFrameRequested = false
+
   private var settledJob: Job? = null
 
   // rive-runtime's command server emits a settle signal on every advance
@@ -121,7 +163,7 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
 
   private val viewScope = CoroutineScope(Dispatchers.Default + SupervisorJob())
 
-  private val textureView = TextureView(context).apply {
+  private val textureView = RiveSemanticsTextureView.create(context).apply {
     layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
     surfaceTextureListener = object : TextureView.SurfaceTextureListener {
       override fun onSurfaceTextureAvailable(st: SurfaceTexture, w: Int, h: Int) {
@@ -154,6 +196,7 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
         // RiveSurface.resize() is internal to the SDK, so only the artboard
         // is resized here (same behavior as before the 11.7.2 bump).
         resizeArtboardIfLayout()
+        if (semanticsEnabled) requestSemanticFrame()
       }
 
       override fun onSurfaceTextureUpdated(st: SurfaceTexture) {}
@@ -202,8 +245,14 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
 
       if (worker != null && art != null && sm != null && rs != null) {
         try {
-          if (!paused && !settled) {
-            worker.advanceStateMachine(sm, deltaTime)
+          val playing = !paused && !settled
+          val advanced = playing || semanticFrameRequested
+          if (advanced) {
+            worker.advanceStateMachine(sm, if (playing) deltaTime else Duration.ZERO)
+          }
+          semanticFrameRequested = false
+          if (advanced && semanticsEnabled && surfaceWidth > 0 && surfaceHeight > 0) {
+            worker.drainSemanticsDiff(sm, activeFit, surfaceWidth.toFloat(), surfaceHeight.toFloat())
           }
           worker.draw(art, sm, rs, activeFit)
           needsRedraw = false
@@ -255,6 +304,13 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
   }
 
   fun configure(config: ViewConfiguration, dataBindingChanged: Boolean, reload: Boolean = false, initialUpdate: Boolean = false) {
+    // Not reported from the semantics setter: props apply in declaration
+    // order, so onError is not wired yet when semantics is set.
+    if (semantics != RiveSemanticsMode.Off && !semanticsSupported && !warnedSemanticsUnsupported) {
+      warnedSemanticsUnsupported = true
+      Log.w(TAG, SEMANTICS_UNSUPPORTED_MESSAGE)
+      onError?.invoke(SEMANTICS_UNSUPPORTED_MESSAGE)
+    }
     riveWorker = config.riveWorker
     activeFit = config.fit
     needsRedraw = true
@@ -266,6 +322,7 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
     if (reload) {
       RiveErrorLogger.resetReportedErrors()
       RiveErrorLogger.addListener(errorListener)
+      if (semanticsEnabled) detachSemantics()
       stateMachineHandle?.let { old ->
         runCatching { config.riveWorker.deleteStateMachine(old) }
       }
@@ -289,6 +346,7 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
       }
       stateMachineHandle = newStateMachineHandle
       observeSettled(config.riveWorker, newStateMachineHandle)
+      if (semanticsEnabled) attachSemantics()
 
       if (surfaceTexture != null && riveSurface == null) {
         riveSurface = config.riveWorker.createRiveSurface(
@@ -467,6 +525,77 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
     }
   }
 
+  private fun resolveSemantics() {
+    val automatic = semantics == RiveSemanticsMode.Automatic
+    if (automatic && !observingAccessibilityState) {
+      accessibilityManager.addAccessibilityStateChangeListener(accessibilityStateListener)
+      observingAccessibilityState = true
+    } else if (!automatic && observingAccessibilityState) {
+      accessibilityManager.removeAccessibilityStateChangeListener(accessibilityStateListener)
+      observingAccessibilityState = false
+    }
+    val requested = when (semantics) {
+      RiveSemanticsMode.Off -> false
+      RiveSemanticsMode.On -> true
+      RiveSemanticsMode.Automatic -> accessibilityManager.isEnabled
+    }
+    val enabled = requested && semanticsSupported
+    if (enabled == semanticsEnabled) return
+    semanticsEnabled = enabled
+    if (enabled) attachSemantics() else detachSemantics()
+  }
+
+  // Core keeps producing semantics once enabled; detaching only stops
+  // draining them.
+  private fun attachSemantics() {
+    val worker = riveWorker ?: return
+    val sm = stateMachineHandle ?: return
+    try {
+      worker.enableSemantics(sm)
+    } catch (e: Exception) {
+      Log.e(TAG, "Failed to enable semantics", e)
+      return
+    }
+    val tree = worker.semanticTree(sm)
+    RiveSemanticsTextureView.install(
+      textureView,
+      tree,
+      { nodeId, action -> semanticInput { worker.fireSemanticAction(sm, nodeId, action) } },
+      { nodeId -> semanticInput { worker.requestSemanticFocus(sm, nodeId) } },
+      { semanticInput { worker.clearSemanticFocus(sm) } },
+    )
+    semanticsSyncJob?.cancel()
+    semanticsSyncJob = mainScope.launch {
+      tree.versionFlow.collect { RiveSemanticsTextureView.synchronize(textureView) }
+    }
+    requestSemanticFrame()
+  }
+
+  private fun detachSemantics() {
+    semanticsSyncJob?.cancel()
+    semanticsSyncJob = null
+    RiveSemanticsTextureView.clear(textureView)
+    val worker = riveWorker ?: return
+    val sm = stateMachineHandle ?: return
+    runCatching { worker.clearSemanticFocus(sm) }
+  }
+
+  private fun semanticInput(block: () -> Unit) {
+    try {
+      block()
+    } catch (e: Exception) {
+      Log.e(TAG, "Semantic input failed", e)
+      return
+    }
+    requestSemanticFrame()
+  }
+
+  private fun requestSemanticFrame() {
+    semanticFrameRequested = true
+    needsRedraw = true
+    settled = false
+  }
+
   fun play() {
     paused = false
     settled = false
@@ -526,6 +655,12 @@ class RiveReactNativeView(context: ThemedReactContext) : FrameLayout(context) {
     viewReadyDeferred.complete(false)
     settledJob?.cancel()
     viewScope.cancel()
+    if (semanticsEnabled) detachSemantics()
+    if (observingAccessibilityState) {
+      accessibilityManager.removeAccessibilityStateChangeListener(accessibilityStateListener)
+      observingAccessibilityState = false
+    }
+    mainScope.cancel()
     RiveErrorLogger.removeListener(errorListener)
     stopRenderLoop()
     // The command queue is FIFO, so deletes enqueued here run after any

android/src/new/java/com/rive/RiveSemanticsTextureView.java

@@ -0,0 +1,48 @@
+package com.rive;
+
+import android.content.Context;
+import android.view.TextureView;
+
+import app.rive.RiveTextureView;
+import app.rive.semantics.SemanticActionType;
+import app.rive.semantics.SemanticTreeModel;
+
+import kotlin.Unit;
+import kotlin.jvm.functions.Function0;
+import kotlin.jvm.functions.Function1;
+import kotlin.jvm.functions.Function2;
+
+/**
+ * Reaches {@code app.rive.RiveTextureView}, the SDK's TalkBack host, which is Kotlin-internal
+ * in rive-android 11.10 (only its Compose entry point uses it); Java is not bound by that.
+ */
+final class RiveSemanticsTextureView {
+  private RiveSemanticsTextureView() {}
+
+  static TextureView create(Context context) {
+    return new RiveTextureView(context);
+  }
+
+  static void install(
+      TextureView view,
+      SemanticTreeModel tree,
+      Function2<Integer, SemanticActionType, Unit> onSemanticAction,
+      Function1<Integer, Unit> onSemanticFocusRequested,
+      Function0<Unit> onSemanticFocusCleared) {
+    ((RiveTextureView) view)
+        .installSemantics(
+            tree,
+            onSemanticAction,
+            transition -> Unit.INSTANCE,
+            onSemanticFocusRequested,
+            onSemanticFocusCleared);
+  }
+
+  static boolean synchronize(TextureView view) {
+    return ((RiveTextureView) view).synchronizeSemantics();
+  }
+
+  static void clear(TextureView view) {
+    ((RiveTextureView) view).clearSemantics();
+  }
+}

example/android/build.gradle

@@ -2,7 +2,7 @@ buildscript {
     ext {
         buildToolsVersion = "35.0.0"
         minSdkVersion = 24
-        compileSdkVersion = 35
+        compileSdkVersion = 36
         targetSdkVersion = 35
         ndkVersion = "27.1.12297006"
         kotlinVersion = "2.1.20"
@@ -21,11 +21,3 @@ buildscript {
 
 apply plugin: "com.facebook.react.rootproject"
 
-allprojects {
-    configurations.all {
-        resolutionStrategy {
-            force 'androidx.core:core:1.15.0'
-            force 'androidx.core:core-ktx:1.15.0'
-        }
-    }
-}

example/src/demos/SemanticsExample.tsx

@@ -4,12 +4,12 @@ import { RiveView, useRiveFile, Semantics } from '@rive-app/react-native';
 import type { Metadata } from '../shared/metadata';
 
 /*
-  Semantics — editor-authored accessibility exposed to VoiceOver.
+  Semantics — editor-authored accessibility exposed to VoiceOver / TalkBack.
 
   tabtest.riv (from rive-runtime's semantics test assets) authors a tab bar
   with roles, labels and selection state. With semantics enabled, each tab
-  becomes an accessibility element: VoiceOver reads them and can activate
-  them. iOS new (default) backend only; see
+  becomes an accessibility element: the screen reader reads them and can
+  activate them. New (default) backends only; see
   https://rive.app/docs/runtimes/apple/semantics
 */
 
@@ -36,8 +36,8 @@ export default function SemanticsExample() {
   return (
     <View style={styles.container}>
       <Text style={styles.subtitle}>
-        With semantics On (or Automatic + VoiceOver running), the tabs below are
-        exposed to VoiceOver as selectable accessibility elements.
+        With semantics On (or Automatic + a screen reader running), the tabs
+        below are exposed to VoiceOver / TalkBack as selectable elements.
       </Text>
       <View style={styles.modeRow}>
         {MODES.map(({ label, value }) => (
@@ -71,9 +71,9 @@ export default function SemanticsExample() {
 }
 
 SemanticsExample.metadata = {
-  name: 'Semantics (VoiceOver)',
+  name: 'Semantics (VoiceOver / TalkBack)',
   description:
-    'Editor-authored accessibility semantics exposed to VoiceOver (iOS new backend)',
+    'Editor-authored accessibility semantics exposed to the screen reader (new backends)',
   order: 3,
 } satisfies Metadata;
 

package.json

@@ -70,8 +70,8 @@
   },
   "homepage": "https://github.com/rive-app/rive-nitro-react-native#readme",
   "runtimeVersions": {
-    "ios": "6.23.1",
-    "android": "11.9.1"
+    "ios": "6.24.0",
+    "android": "11.10.0"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/"