425 lines
20 KiB
Kotlin
425 lines
20 KiB
Kotlin
package com.yama.client
|
||
|
||
import android.accessibilityservice.AccessibilityService
|
||
import android.accessibilityservice.GestureDescription
|
||
import android.annotation.SuppressLint
|
||
import android.content.BroadcastReceiver
|
||
import android.content.Context
|
||
import android.content.Intent
|
||
import android.content.IntentFilter
|
||
import android.graphics.Path
|
||
import android.graphics.PointF
|
||
import android.os.Build
|
||
import android.os.Handler
|
||
import android.os.Looper
|
||
import android.os.SystemClock
|
||
import android.util.Log
|
||
import android.view.View
|
||
import android.view.accessibility.AccessibilityEvent
|
||
import android.view.accessibility.AccessibilityNodeInfo
|
||
|
||
/**
|
||
* AccessibilityService that injects touch gestures and global key actions
|
||
* dispatched by the server (COMMAND_SCREEN_CONTROL / MSG64).
|
||
*
|
||
* Requires API 24+ for dispatchGesture(); guarded at runtime so the APK
|
||
* installs on API 21+ but gesture injection only runs on 24+.
|
||
*
|
||
* Activation: Settings → Accessibility → YAMA → enable.
|
||
*/
|
||
@SuppressLint("NewApi")
|
||
class ControlService : AccessibilityService() {
|
||
|
||
companion object {
|
||
const val TAG = "YAMA_CTRL"
|
||
|
||
// ── Service instance ─────────────────────────────────────────────
|
||
@Volatile var instance: ControlService? = null
|
||
|
||
// ── Active foreground window ─────────────────────────────────────
|
||
@JvmStatic
|
||
@Volatile var activeWindow: String = "Android"
|
||
|
||
// ── Screen geometry (set by CaptureService) ──────────────────────
|
||
// Server sends coords in encoding space [0, encW) × [0, encH).
|
||
// We map them to physical screen space before dispatching.
|
||
@Volatile var physW = 1080
|
||
@Volatile var physH = 1920
|
||
@Volatile var encW = 1080
|
||
@Volatile var encH = 1920
|
||
|
||
// ── Windows message constants ────────────────────────────────────
|
||
const val WM_MOUSEMOVE = 0x0200
|
||
const val WM_LBUTTONDOWN = 0x0201
|
||
const val WM_LBUTTONUP = 0x0202
|
||
const val WM_LBUTTONDBLCLK = 0x0203
|
||
const val WM_RBUTTONDOWN = 0x0204
|
||
const val WM_MBUTTONDOWN = 0x0207
|
||
const val WM_MOUSEWHEEL = 0x020A
|
||
const val WM_KEYDOWN = 0x0100
|
||
const val WM_SYSKEYDOWN = 0x0104
|
||
|
||
/**
|
||
* Called from C++ DataProcess() via JNI on the IO thread.
|
||
* Parameters:
|
||
* message – Windows WM_* constant (low 16 bits of MSG64.message)
|
||
* wParam – MSG64.wParam (key/button flags; high word of MOUSEWHEEL = delta)
|
||
* ptX/ptY – MSG64.pt in encoding-space pixels
|
||
*/
|
||
@JvmStatic
|
||
fun onControlEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) {
|
||
val svc = instance ?: run {
|
||
Log.w(TAG, "ControlService inactive — event 0x${message.toString(16)} dropped")
|
||
return
|
||
}
|
||
// All gesture state lives on the main thread; post there.
|
||
svc.handler.post { svc.handleEvent(message, wParam, ptX, ptY) }
|
||
}
|
||
}
|
||
|
||
// ── Screen state receiver ─────────────────────────────────────────────
|
||
private val screenReceiver = object : BroadcastReceiver() {
|
||
override fun onReceive(context: Context?, intent: Intent?) {
|
||
when (intent?.action) {
|
||
Intent.ACTION_SCREEN_OFF -> activeWindow = "Locked"
|
||
Intent.ACTION_USER_PRESENT -> activeWindow = "Android"
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── Main-thread handler ───────────────────────────────────────────────
|
||
private val handler = Handler(Looper.getMainLooper())
|
||
|
||
// ── TV (Google TV / Android TV) D-pad mode ───────────────────────────
|
||
// Activated when FEATURE_LEANBACK is present. Mouse events are translated
|
||
// to accessibility focus navigation — no touchscreen or root required.
|
||
private var isTV = false
|
||
private var tvLastEncX = -1f
|
||
private var tvLastEncY = -1f
|
||
private var tvAccumDX = 0f
|
||
private var tvAccumDY = 0f
|
||
private val TV_STEP = 40f // encoding-space pixels per D-pad step
|
||
// Touch-drag disambiguation: fire click on LBUTTONUP only if finger barely moved
|
||
private var tvTouchDown = false
|
||
private var tvTouchDownX = 0f
|
||
private var tvTouchDownY = 0f
|
||
private var tvTouchMoved = false
|
||
private val TV_TAP_THRESHOLD = 15f // encoding pixels — below this = tap, above = drag
|
||
|
||
// ── Left-button drag accumulator ─────────────────────────────────────
|
||
private var lbuttonDown = false
|
||
private var gestureStart = 0L
|
||
// Pairs of (millisecond offset from gesture start, screen point)
|
||
private val strokePoints = mutableListOf<Pair<Long, PointF>>()
|
||
|
||
// ─────────────────────────────────────────────────────────────────────
|
||
// AccessibilityService lifecycle
|
||
|
||
override fun onServiceConnected() {
|
||
super.onServiceConnected()
|
||
instance = this
|
||
isTV = packageManager.hasSystemFeature("android.software.leanback")
|
||
if (isTV) Log.i(TAG, "TV mode: D-pad navigation enabled")
|
||
val filter = IntentFilter().apply {
|
||
addAction(Intent.ACTION_SCREEN_OFF)
|
||
addAction(Intent.ACTION_USER_PRESENT)
|
||
}
|
||
registerReceiver(screenReceiver, filter)
|
||
Log.i(TAG, "ControlService connected phys=${physW}x${physH} enc=${encW}x${encH}")
|
||
}
|
||
|
||
override fun onAccessibilityEvent(event: AccessibilityEvent?) {
|
||
if (event?.eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
|
||
val pkg = event.packageName?.toString()
|
||
if (!pkg.isNullOrEmpty()) activeWindow = pkg
|
||
}
|
||
}
|
||
override fun onInterrupt() {}
|
||
|
||
override fun onDestroy() {
|
||
super.onDestroy()
|
||
runCatching { unregisterReceiver(screenReceiver) }
|
||
instance = null
|
||
Log.i(TAG, "ControlService destroyed")
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────
|
||
// Event routing (runs on main thread)
|
||
|
||
private fun handleEvent(message: Int, wParam: Long, ptX: Int, ptY: Int) {
|
||
if (isTV) { handleEventTV(message, wParam, ptX, ptY); return }
|
||
// Scale encoding-space coords to physical screen
|
||
val x = if (encW > 0) ptX.toFloat() * physW / encW else ptX.toFloat()
|
||
val y = if (encH > 0) ptY.toFloat() * physH / encH else ptY.toFloat()
|
||
if (message != WM_MOUSEMOVE)
|
||
Log.d(TAG, "event 0x${message.toString(16)} enc=($ptX,$ptY) phys=(${x.toInt()},${y.toInt()})")
|
||
|
||
when (message) {
|
||
|
||
// ── Left button: accumulate drag path, dispatch on up ─────────
|
||
WM_LBUTTONDOWN -> {
|
||
lbuttonDown = true
|
||
gestureStart = SystemClock.uptimeMillis()
|
||
strokePoints.clear()
|
||
strokePoints.add(0L to PointF(x, y))
|
||
}
|
||
WM_MOUSEMOVE -> {
|
||
if (lbuttonDown) {
|
||
strokePoints.add((SystemClock.uptimeMillis() - gestureStart) to PointF(x, y))
|
||
}
|
||
}
|
||
WM_LBUTTONUP -> {
|
||
if (!lbuttonDown) return
|
||
lbuttonDown = false
|
||
val elapsed = SystemClock.uptimeMillis() - gestureStart
|
||
strokePoints.add(elapsed to PointF(x, y))
|
||
val down = strokePoints[0].second
|
||
val dx = x - down.x
|
||
val dy = y - down.y
|
||
|
||
// ── System edge-gesture detection ────────────────────────────
|
||
// dispatchGesture() is unreliable for system gestures; use performGlobalAction.
|
||
val hEdge = physH * 0.08f // 8% of screen height = top/bottom trigger zone
|
||
val wEdge = physW * 0.08f // 8% of screen width = left/right trigger zone
|
||
val minH = physH * 0.15f // minimum vertical travel
|
||
val minW = physW * 0.20f // minimum horizontal travel
|
||
|
||
val fromLeft = down.x < wEdge && dx > minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L
|
||
val fromRight = down.x > physW - wEdge && dx < -minW && Math.abs(dx) > Math.abs(dy) * 1.5f && elapsed < 600L
|
||
val fromTop = down.y < hEdge && dy > minH && Math.abs(dy) > Math.abs(dx)
|
||
val fromBottom = down.y > physH - hEdge && dy < -minH && Math.abs(dy) > Math.abs(dx)
|
||
|
||
when {
|
||
fromLeft || fromRight -> {
|
||
Log.d(TAG, "Edge-swipe back (fromLeft=$fromLeft)")
|
||
performGlobalAction(GLOBAL_ACTION_BACK)
|
||
}
|
||
fromTop -> {
|
||
// Right half of screen → quick settings; left half → notifications
|
||
if (down.x > physW * 0.5f) {
|
||
Log.d(TAG, "Top-swipe quick settings")
|
||
performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS)
|
||
} else {
|
||
Log.d(TAG, "Top-swipe notifications")
|
||
performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS)
|
||
}
|
||
}
|
||
fromBottom -> {
|
||
// Slow swipe (≥400 ms) or very long travel → Recents; quick swipe → Home
|
||
if (elapsed >= 400L || Math.abs(dy) > physH * 0.40f) {
|
||
Log.d(TAG, "Bottom-swipe recents (elapsed=${elapsed}ms)")
|
||
performGlobalAction(GLOBAL_ACTION_RECENTS)
|
||
} else {
|
||
Log.d(TAG, "Bottom-swipe home (elapsed=${elapsed}ms)")
|
||
performGlobalAction(GLOBAL_ACTION_HOME)
|
||
}
|
||
}
|
||
dx * dx + dy * dy < 100f -> // 移动 < 10px → 单击
|
||
dispatchTap(down.x, down.y)
|
||
else ->
|
||
dispatchTouchGesture(strokePoints)
|
||
}
|
||
// 手势执行后立即请求关键帧，让 decoder 尽快看到屏幕变化
|
||
CaptureService.requestIdr()
|
||
strokePoints.clear()
|
||
}
|
||
|
||
// ── Double-click → two rapid taps ────────────────────────────
|
||
WM_LBUTTONDBLCLK -> {
|
||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||
val p = singlePointPath(x, y)
|
||
val gesture = GestureDescription.Builder()
|
||
.addStroke(GestureDescription.StrokeDescription(p, 0L, 80L))
|
||
.addStroke(GestureDescription.StrokeDescription(p, 200L, 80L))
|
||
.build()
|
||
dispatchGesture(gesture, null, null)
|
||
}
|
||
|
||
// ── Right-click → long press (600 ms) ────────────────────────
|
||
WM_RBUTTONDOWN -> {
|
||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||
val stroke = GestureDescription.StrokeDescription(singlePointPath(x, y), 0L, 600L)
|
||
dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null)
|
||
}
|
||
|
||
// ── Scroll wheel → vertical swipe ─────────────────────────────
|
||
WM_MOUSEWHEEL -> {
|
||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||
// High 16 bits of wParam hold the signed delta (+120 = up, -120 = down)
|
||
val delta = (wParam.toInt() ushr 16).toShort().toInt()
|
||
// Wheel-up (+delta) = user wants content above = finger swipes DOWN (+y)
|
||
val dy = if (delta > 0) 400f else -400f
|
||
val path = Path().apply { moveTo(x, y); lineTo(x, y + dy) }
|
||
val stroke = GestureDescription.StrokeDescription(path, 0L, 300L)
|
||
dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(), null, null)
|
||
CaptureService.requestIdr()
|
||
}
|
||
|
||
// ── Keyboard → global actions only ───────────────────────────
|
||
WM_KEYDOWN, WM_SYSKEYDOWN -> handleKeyDown(wParam.toInt() and 0xFFFF)
|
||
}
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────
|
||
// Helpers
|
||
|
||
private fun dispatchTouchGesture(points: List<Pair<Long, PointF>>) {
|
||
if (points.isEmpty()) return
|
||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||
|
||
val path = Path()
|
||
path.moveTo(points[0].second.x, points[0].second.y)
|
||
for (i in 1 until points.size) {
|
||
path.lineTo(points[i].second.x, points[i].second.y)
|
||
}
|
||
// Minimum 50 ms so Android registers the gesture; use actual elapsed for drags.
|
||
val duration = maxOf(points.last().first, 50L)
|
||
val stroke = GestureDescription.StrokeDescription(path, 0L, duration)
|
||
val ok = dispatchGesture(
|
||
GestureDescription.Builder().addStroke(stroke).build(),
|
||
object : GestureResultCallback() {
|
||
override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Gesture ok") }
|
||
override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Gesture cancelled") }
|
||
},
|
||
null
|
||
)
|
||
if (!ok) Log.e(TAG, "dispatchGesture returned false (service not ready?)")
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────
|
||
// TV D-pad mode — translates mouse events to accessibility focus actions
|
||
|
||
private fun handleEventTV(message: Int, wParam: Long, ptX: Int, ptY: Int) {
|
||
when (message) {
|
||
WM_MOUSEMOVE -> {
|
||
if (tvLastEncX >= 0) {
|
||
tvAccumDX += ptX - tvLastEncX
|
||
tvAccumDY += ptY - tvLastEncY
|
||
while (tvAccumDX > TV_STEP) { tvMoveFocus(View.FOCUS_RIGHT); tvAccumDX -= TV_STEP }
|
||
while (tvAccumDX < -TV_STEP) { tvMoveFocus(View.FOCUS_LEFT); tvAccumDX += TV_STEP }
|
||
while (tvAccumDY > TV_STEP) { tvMoveFocus(View.FOCUS_DOWN); tvAccumDY -= TV_STEP }
|
||
while (tvAccumDY < -TV_STEP) { tvMoveFocus(View.FOCUS_UP); tvAccumDY += TV_STEP }
|
||
}
|
||
if (tvTouchDown) {
|
||
val dx = ptX - tvTouchDownX
|
||
val dy = ptY - tvTouchDownY
|
||
if (dx * dx + dy * dy > TV_TAP_THRESHOLD * TV_TAP_THRESHOLD) tvTouchMoved = true
|
||
}
|
||
tvLastEncX = ptX.toFloat()
|
||
tvLastEncY = ptY.toFloat()
|
||
}
|
||
WM_LBUTTONDOWN -> {
|
||
tvTouchDown = true
|
||
tvTouchDownX = ptX.toFloat()
|
||
tvTouchDownY = ptY.toFloat()
|
||
tvTouchMoved = false
|
||
tvAccumDX = 0f
|
||
tvAccumDY = 0f
|
||
tvLastEncX = ptX.toFloat()
|
||
tvLastEncY = ptY.toFloat()
|
||
}
|
||
WM_LBUTTONUP -> {
|
||
if (tvTouchDown && !tvTouchMoved) tvClick()
|
||
tvTouchDown = false
|
||
}
|
||
WM_LBUTTONDBLCLK -> tvClick()
|
||
WM_RBUTTONDOWN -> performGlobalAction(GLOBAL_ACTION_BACK)
|
||
WM_MOUSEWHEEL -> {
|
||
val delta = (wParam.toInt() ushr 16).toShort().toInt()
|
||
tvScroll(delta < 0)
|
||
}
|
||
WM_KEYDOWN, WM_SYSKEYDOWN -> {
|
||
val vk = wParam.toInt() and 0xFFFF
|
||
when (vk) {
|
||
0x25 -> tvMoveFocus(View.FOCUS_LEFT) // VK_LEFT
|
||
0x26 -> tvMoveFocus(View.FOCUS_UP) // VK_UP
|
||
0x27 -> tvMoveFocus(View.FOCUS_RIGHT) // VK_RIGHT
|
||
0x28 -> tvMoveFocus(View.FOCUS_DOWN) // VK_DOWN
|
||
0x0D -> tvClick() // VK_RETURN
|
||
else -> handleKeyDown(vk)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
@Suppress("DEPRECATION")
|
||
private fun tvMoveFocus(direction: Int) {
|
||
val root = rootInActiveWindow ?: return
|
||
val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
|
||
?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)
|
||
if (focused == null) {
|
||
val first = findFirstFocusable(root)
|
||
first?.performAction(AccessibilityNodeInfo.ACTION_FOCUS)
|
||
first?.recycle()
|
||
} else {
|
||
val next = focused.focusSearch(direction)
|
||
next?.performAction(AccessibilityNodeInfo.ACTION_FOCUS)
|
||
focused.recycle()
|
||
next?.recycle()
|
||
}
|
||
root.recycle()
|
||
}
|
||
|
||
@Suppress("DEPRECATION")
|
||
private fun tvClick() {
|
||
val root = rootInActiveWindow ?: return
|
||
val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
|
||
?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)
|
||
if (focused != null) {
|
||
focused.performAction(AccessibilityNodeInfo.ACTION_CLICK)
|
||
focused.recycle()
|
||
} else {
|
||
Log.w(TAG, "tvClick: no focused node")
|
||
}
|
||
root.recycle()
|
||
}
|
||
|
||
@Suppress("DEPRECATION")
|
||
private fun tvScroll(forward: Boolean) {
|
||
val root = rootInActiveWindow ?: return
|
||
val action = if (forward) AccessibilityNodeInfo.ACTION_SCROLL_FORWARD
|
||
else AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD
|
||
val focused = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
|
||
?: root.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)
|
||
if (focused == null || !focused.performAction(action)) root.performAction(action)
|
||
focused?.recycle()
|
||
root.recycle()
|
||
}
|
||
|
||
// Walk accessibility tree depth-first, return first visible focusable node (caller must recycle)
|
||
@Suppress("DEPRECATION")
|
||
private fun findFirstFocusable(node: AccessibilityNodeInfo): AccessibilityNodeInfo? {
|
||
if (node.isFocusable && node.isVisibleToUser) return AccessibilityNodeInfo.obtain(node)
|
||
for (i in 0 until node.childCount) {
|
||
val child = node.getChild(i) ?: continue
|
||
val hit = findFirstFocusable(child)
|
||
child.recycle()
|
||
if (hit != null) return hit
|
||
}
|
||
return null
|
||
}
|
||
|
||
private fun handleKeyDown(vk: Int) {
|
||
when (vk) {
|
||
0x08, 0x1B -> performGlobalAction(GLOBAL_ACTION_BACK) // Backspace / Esc
|
||
0x24 -> performGlobalAction(GLOBAL_ACTION_HOME) // VK_HOME
|
||
0x5D -> performGlobalAction(GLOBAL_ACTION_RECENTS) // VK_APPS
|
||
0x2C -> performGlobalAction(GLOBAL_ACTION_TAKE_SCREENSHOT) // PrtSc
|
||
}
|
||
}
|
||
|
||
private fun dispatchTap(x: Float, y: Float) {
|
||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return
|
||
val path = Path().apply { moveTo(x, y); lineTo(x + 1f, y) }
|
||
val stroke = GestureDescription.StrokeDescription(path, 0L, 100L)
|
||
val ok = dispatchGesture(GestureDescription.Builder().addStroke(stroke).build(),
|
||
object : GestureResultCallback() {
|
||
override fun onCompleted(g: GestureDescription?) { Log.d(TAG, "Tap ok") }
|
||
override fun onCancelled(g: GestureDescription?) { Log.w(TAG, "Tap cancelled") }
|
||
}, null)
|
||
if (!ok) Log.e(TAG, "dispatchTap returned false")
|
||
}
|
||
|
||
private fun singlePointPath(x: Float, y: Float) = Path().apply { moveTo(x, y); lineTo(x + 1f, y) }
|
||
}
|