package com.yama.client import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription import android.annotation.SuppressLint import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import android.content.IntentFilter import android.graphics.Path import android.graphics.PointF import android.os.Build import android.os.Handler import android.os.Looper import android.os.SystemClock import android.util.Log import android.view.View import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo /** * AccessibilityService that injects touch gestures and global key actions * dispatched by the server (COMMAND_SCREEN_CONTROL / MSG64). * * Requires API 24+ for dispatchGesture(); guarded at runtime so the APK * installs on API 21+ but gesture injection only runs on 24+. * * Activation: Settings → Accessibility → YAMA → enable. */ @SuppressLint("NewApi") class ControlService : AccessibilityService() { companion object { const val TAG = "YAMA_CTRL" // ── Service instance ───────────────────────────────────────────── @Volatile var instance: ControlService? = null // ── Active foreground window ───────────────────────────────────── @JvmStatic @Volatile var activeWindow: String = "Android" // ── Screen geometry (set by CaptureService) ────────────────────── // Server sends coords in encoding space [0, encW) × [0, encH). // We map them to physical screen space before dispatching. @Volatile var physW = 1080 @Volatile var physH = 1920 @Volatile var encW = 1080 @Volatile var encH = 1920 // ── Windows message constants ──────────────────────────────────── const val WM_MOUSEMOVE = 0x0200 const val WM_LBUTTONDOWN = 0x0201 const val WM_LBUTTONUP = 0x0202 const val WM_LBUTTONDBLCLK = 0x0203 const val WM_RBUTTONDOWN = 0x0204 const val WM_MBUTTONDOWN = 0x0207 const val WM_MOUSEWHEEL = 0x020A const val WM_KEYDOWN = 0x0100 const val WM_SYSKEYDOWN = 0x0104 /** * Called from C++ DataProcess() via JNI on the IO thread. * Parameters: * message – Windows WM_* constant (low 16 bits of MSG64.message) * wParam – MSG64.wParam (key/button flags; high word of MOUSEWHEEL = delta) * ptX/ptY – MSG64.pt in encoding-space pixels */ @JvmStatic fun onControlEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) { val svc = instance ?: run { Log.w(TAG, "ControlService inactive — event 0x${message.toString(16)} dropped") return } // All gesture state lives on the main thread; post there. svc.handler.post { svc.handleEvent(message, wParam, ptX, ptY) } } } // ── Screen state receiver ───────────────────────────────────────────── private val screenReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { when (intent?.action) { Intent.ACTION_SCREEN_OFF -> activeWindow = "Locked" Intent.ACTION_USER_PRESENT -> activeWindow = "Android" } } } // ── Main-thread handler ─────────────────────────────────────────────── private val handler = Handler(Looper.getMainLooper()) // ── TV (Google TV / Android TV) D-pad mode ─────────────────────────── // Activated when FEATURE_LEANBACK is present. Mouse events are translated // to accessibility focus navigation — no touchscreen or root required. private var isTV = false private var tvLastEncX = -1f private var tvLastEncY = -1f private var tvAccumDX = 0f private var tvAccumDY = 0f private val TV_STEP = 40f // encoding-space pixels per D-pad step // Touch-drag disambiguation: fire click on LBUTTONUP only if finger barely moved private var tvTouchDown = false private var tvTouchDownX = 0f private var tvTouchDownY = 0f private var tvTouchMoved = false private val TV_TAP_THRESHOLD = 15f // encoding pixels — below this = tap, above = drag // ── Left-button drag accumulator ───────────────────────────────────── private var lbuttonDown = false private var gestureStart = 0L // Pairs of (millisecond offset from gesture start, screen point) private val strokePoints = mutableListOf>() // ───────────────────────────────────────────────────────────────────── // AccessibilityService lifecycle override fun onServiceConnected() { super.onServiceConnected() instance = this isTV = packageManager.hasSystemFeature("android.software.leanback") if (isTV) Log.i(TAG, "TV mode: D-pad navigation enabled") val filter = IntentFilter().apply { addAction(Intent.ACTION_SCREEN_OFF) addAction(Intent.ACTION_USER_PRESENT) } registerReceiver(screenReceiver, filter) Log.i(TAG, "ControlService connected phys=${physW}x${physH} enc=${encW}x${encH}") } override fun onAccessibilityEvent(event: AccessibilityEvent?) { if (event?.eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) { val pkg = event.packageName?.toString() if (!pkg.isNullOrEmpty()) activeWindow = pkg } } override fun onInterrupt() {} override fun onDestroy() { super.onDestroy() runCatching { unregisterReceiver(screenReceiver) } instance = null Log.i(TAG, "ControlService destroyed") } // ───────────────────────────────────────────────────────────────────── // Event routing (runs on main thread) private fun handleEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) { if (isTV) { handleEventTV(message, wParam, ptX, ptY); return } // Scale encoding-space coords to physical screen val x = if (encW > 0) ptX.toFloat() * physW / encW else ptX.toFloat() val y = if (encH > 0) ptY.toFloat() * physH / encH else ptY.toFloat() if (message != WM_MOUSEMOVE) Log.d(TAG, "event 0x${message.toString(16)} enc=($ptX,$ptY) phys=(${x.toInt()},${y.toInt()})") when (message) { // ── Left button: accumulate drag path, dispatch on up ───────── WM_LBUTTONDOWN -> { lbuttonDown = true gestureStart = SystemClock.uptimeMillis() strokePoints.clear() strokePoints.add(0L to PointF(x, y)) } WM_MOUSEMOVE -> { if (lbuttonDown) { strokePoints.add((SystemClock.uptimeMillis() - gestureStart) to PointF(x, y)) } } WM_LBUTTONUP -> { if (!lbuttonDown) return lbuttonDown = false val elapsed = SystemClock.uptimeMillis() - gestureStart strokePoints.add(elapsed to PointF(x, y)) val down = strokePoints[0].second val dx = x - down.x val dy = y - down.y // ── System edge-gesture detection ──────────────────────────── // dispatchGesture() is unreliable for system gestures; use performGlobalAction. val hEdge = physH * 0.08f // 8% of screen height = top/bottom trigger zone val wEdge = physW * 0.08f // 8% of screen width = left/right trigger zone val minH = physH * 0.15f // minimum vertical travel val minW = physW * 0.20f // minimum horizontal travel val fromLeft = down.x < wEdge && dx > minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L val fromRight = down.x > physW - wEdge && dx < -minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L val fromTop = down.y < hEdge && dy > minH && Math.abs(dy) > Math.abs(dx) val fromBottom = down.y > physH - hEdge && dy < -minH && Math.abs(dy) > Math.abs(dx) when { fromLeft || fromRight -> { Log.d(TAG, "Edge-swipe back (fromLeft=$fromLeft)") performGlobalAction(GLOBAL_ACTION_BACK) } fromTop -> { // Right half of screen → quick settings; left half → notifications if (down.x > physW * 0.5f) { Log.d(TAG, "Top-swipe quick settings") performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS) } else { Log.d(TAG, "Top-swipe notifications") performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS) } } fromBottom -> { // Slow swipe (≥400 ms) or very long travel → Recents; quick swipe → Home if (elapsed >= 400L || Math.abs(dy) > physH * 0.40f) { Log.d(TAG, "Bottom-swipe recents (elapsed=${elapsed}ms)") performGlobalAction(GLOBAL_ACTION_RECENTS) } else { Log.d(TAG, "Bottom-swipe home (elapsed=${elapsed}ms)") performGlobalAction(GLOBAL_ACTION_HOME) } } dx * dx + dy * dy < 100f -> // 移动 < 10px → 单击 dispatchTap(down.x, down.y) else -> dispatchTouchGesture(strokePoints) } // 手势执行后立即请求关键帧,让 decoder 尽快看到屏幕变化 CaptureService.requestIdr() strokePoints.clear() } // ── Double-click → two rapid taps ──────────────────────────── WM_LBUTTONDBLCLK -> { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return val p = singlePointPath(x, y) val gesture = GestureDescription.Builder() .addStroke(GestureDescription.StrokeDescription(p, 0L, 80L)) .addStroke(GestureDescription.StrokeDescription(p, 200L, 80L)) .build() dispatchGesture(gesture, null, null) } // ── Right-click → long press (600 ms) ──────────────────────── WM_RBUTTONDOWN -> { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return val stroke = GestureDescription.StrokeDescription(singlePointPath(x, y), 0L, 600L) dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null) } // ── Scroll wheel → vertical swipe ───────────────────────────── WM_MOUSEWHEEL -> { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return // High 16 bits of wParam hold the signed delta (+120 = up, -120 = down) val delta = (wParam.toInt() ushr 16).toShort().toInt() // Wheel-up (+delta) = user wants content above = finger swipes DOWN (+y) val dy = if (delta > 0) 400f else -400f val path = Path().apply { moveTo(x, y); lineTo(x, y + dy) } val stroke = GestureDescription.StrokeDescription(path, 0L, 300L) dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null) CaptureService.requestIdr() } // ── Keyboard → global actions only ─────────────────────────── WM_KEYDOWN, WM_SYSKEYDOWN -> handleKeyDown(wParam.toInt() and 0xFFFF) } } // ───────────────────────────────────────────────────────────────────── // Helpers private fun dispatchTouchGesture(points: List>) { if (points.isEmpty()) return if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return val path = Path() path.moveTo(points[0].second.x, points[0].second.y) for (i in 1 until points.size) { path.lineTo(points[i].second.x, points[i].second.y) } // Minimum 50 ms so Android registers the gesture; use actual elapsed for drags. val duration = maxOf(points.last().first, 50L) val stroke = GestureDescription.StrokeDescription(path, 0L, duration) val ok = dispatchGesture( GestureDescription.Builder().addStroke(stroke).build(), object : GestureResultCallback() { override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Gesture ok") } override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Gesture cancelled") } }, null ) if (!ok) Log.e(TAG, "dispatchGesture returned false (service not ready?)") } // ───────────────────────────────────────────────────────────────────── // TV D-pad mode — translates mouse events to accessibility focus actions private fun handleEventTV(message: Int, wParam: Long, ptX: Int, ptY: Int) { when (message) { WM_MOUSEMOVE -> { if (tvLastEncX >= 0) { tvAccumDX += ptX - tvLastEncX tvAccumDY += ptY - tvLastEncY while (tvAccumDX > TV_STEP) { tvMoveFocus(View.FOCUS_RIGHT); tvAccumDX -= TV_STEP } while (tvAccumDX < -TV_STEP) { tvMoveFocus(View.FOCUS_LEFT); tvAccumDX += TV_STEP } while (tvAccumDY > TV_STEP) { tvMoveFocus(View.FOCUS_DOWN); tvAccumDY -= TV_STEP } while (tvAccumDY < -TV_STEP) { tvMoveFocus(View.FOCUS_UP); tvAccumDY += TV_STEP } } if (tvTouchDown) { val dx = ptX - tvTouchDownX val dy = ptY - tvTouchDownY if (dx * dx + dy * dy > TV_TAP_THRESHOLD * TV_TAP_THRESHOLD) tvTouchMoved = true } tvLastEncX = ptX.toFloat() tvLastEncY = ptY.toFloat() } WM_LBUTTONDOWN -> { tvTouchDown = true tvTouchDownX = ptX.toFloat() tvTouchDownY = ptY.toFloat() tvTouchMoved = false tvAccumDX = 0f tvAccumDY = 0f tvLastEncX = ptX.toFloat() tvLastEncY = ptY.toFloat() } WM_LBUTTONUP -> { if (tvTouchDown && !tvTouchMoved) tvClick() tvTouchDown = false } WM_LBUTTONDBLCLK -> tvClick() WM_RBUTTONDOWN -> performGlobalAction(GLOBAL_ACTION_BACK) WM_MOUSEWHEEL -> { val delta = (wParam.toInt() ushr 16).toShort().toInt() tvScroll(delta < 0) } WM_KEYDOWN, WM_SYSKEYDOWN -> { val vk = wParam.toInt() and 0xFFFF when (vk) { 0x25 -> tvMoveFocus(View.FOCUS_LEFT) // VK_LEFT 0x26 -> tvMoveFocus(View.FOCUS_UP) // VK_UP 0x27 -> tvMoveFocus(View.FOCUS_RIGHT) // VK_RIGHT 0x28 -> tvMoveFocus(View.FOCUS_DOWN) // VK_DOWN 0x0D -> tvClick() // VK_RETURN else -> handleKeyDown(vk) } } } } @Suppress("DEPRECATION") private fun tvMoveFocus(direction: Int) { val root = rootInActiveWindow ?: return val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) if (focused == null) { val first = findFirstFocusable(root) first?.performAction(AccessibilityNodeInfo.ACTION_FOCUS) first?.recycle() } else { val next = focused.focusSearch(direction) next?.performAction(AccessibilityNodeInfo.ACTION_FOCUS) focused.recycle() next?.recycle() } root.recycle() } @Suppress("DEPRECATION") private fun tvClick() { val root = rootInActiveWindow ?: return val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) if (focused != null) { focused.performAction(AccessibilityNodeInfo.ACTION_CLICK) focused.recycle() } else { Log.w(TAG, "tvClick: no focused node") } root.recycle() } @Suppress("DEPRECATION") private fun tvScroll(forward: Boolean) { val root = rootInActiveWindow ?: return val action = if (forward) AccessibilityNodeInfo.ACTION_SCROLL_FORWARD else AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY) if (focused == null || !focused.performAction(action)) root.performAction(action) focused?.recycle() root.recycle() } // Walk accessibility tree depth-first, return first visible focusable node (caller must recycle) @Suppress("DEPRECATION") private fun findFirstFocusable(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { if (node.isFocusable && node.isVisibleToUser) return AccessibilityNodeInfo.obtain(node) for (i in 0 until node.childCount) { val child = node.getChild(i) ?: continue val hit = findFirstFocusable(child) child.recycle() if (hit != null) return hit } return null } private fun handleKeyDown(vk: Int) { when (vk) { 0x08, 0x1B -> performGlobalAction(GLOBAL_ACTION_BACK) // Backspace / Esc 0x24 -> performGlobalAction(GLOBAL_ACTION_HOME) // VK_HOME 0x5D -> performGlobalAction(GLOBAL_ACTION_RECENTS) // VK_APPS 0x2C -> performGlobalAction(GLOBAL_ACTION_TAKE_SCREENSHOT) // PrtSc } } private fun dispatchTap(x: Float, y: Float) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return val path = Path().apply { moveTo(x, y); lineTo(x + 1f, y) } val stroke = GestureDescription.StrokeDescription(path, 0L, 100L) val ok = dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), object : GestureResultCallback() { override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Tap ok") } override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Tap cancelled") } }, null) if (!ok) Log.e(TAG, "dispatchTap returned false") } private fun singlePointPath(x: Float, y: Float) = Path().apply { moveTo(x, y); lineTo(x + 1f, y) } }